Initial commit: Jarvis Memory system

skills/mem-redis/scripts/cron_backup.py

@@ -0,0 +1,204 @@
+#!/usr/bin/env python3
+"""
+Daily Cron: Process Redis buffer → Qdrant → Clear Redis.
+
+This script runs once daily (via cron) to move buffered conversation
+turns from Redis to durable Qdrant storage. Only clears Redis after
+successful Qdrant write.
+
+Usage: python3 cron_backup.py [--user-id rob] [--dry-run]
+"""
+
+import os
+import sys
+import json
+import redis
+import argparse
+from datetime import datetime, timezone
+from pathlib import Path
+
+# Add qdrant-memory to path (portable)
+from pathlib import Path as _Path
+WORKSPACE = _Path(os.getenv("OPENCLAW_WORKSPACE", str(_Path.home() / ".openclaw" / "workspace")))
+sys.path.insert(0, str(WORKSPACE / "skills" / "qdrant-memory" / "scripts"))
+
+try:
+    from auto_store import store_conversation_turn
+    QDRANT_AVAILABLE = True
+except ImportError:
+    QDRANT_AVAILABLE = False
+    print("Warning: Qdrant storage not available, will simulate", file=sys.stderr)
+
+# Config
+REDIS_HOST = os.getenv("REDIS_HOST", "127.0.0.1")
+REDIS_PORT = int(os.getenv("REDIS_PORT", "6379"))
+USER_ID = os.getenv("USER_ID", "yourname")
+
+def get_redis_items(user_id):
+    """Get all items from Redis list."""
+    try:
+        r = redis.Redis(host=REDIS_HOST, port=REDIS_PORT, decode_responses=True)
+        key = f"mem:{user_id}"
+        
+        # Get all items (0 to -1 = entire list)
+        items = r.lrange(key, 0, -1)
+        
+        # Parse JSON
+        turns = []
+        for item in items:
+            try:
+                turn = json.loads(item)
+                turns.append(turn)
+            except json.JSONDecodeError:
+                continue
+        
+        return turns, key
+    except Exception as e:
+        print(f"Error reading from Redis: {e}", file=sys.stderr)
+        return None, None
+
+def store_to_qdrant(turns, user_id):
+    """Store turns to Qdrant with file fallback."""
+    if not QDRANT_AVAILABLE:
+        print("[DRY RUN] Would store to Qdrant:", file=sys.stderr)
+        for turn in turns[:3]:
+            print(f"  - Turn {turn.get('turn', '?')}: {turn.get('role', '?')}", file=sys.stderr)
+        if len(turns) > 3:
+            print(f"  ... and {len(turns) - 3} more", file=sys.stderr)
+        return True
+    
+    # Ensure chronological order (older -> newer)
+    try:
+        turns_sorted = sorted(turns, key=lambda t: (t.get('timestamp', ''), t.get('turn', 0)))
+    except Exception:
+        turns_sorted = turns
+
+    user_turns = [t for t in turns_sorted if t.get('role') == 'user']
+    if not user_turns:
+        return True
+
+    success_count = 0
+    attempted = 0
+
+    for i, turn in enumerate(turns_sorted):
+        if turn.get('role') != 'user':
+            continue
+        attempted += 1
+        try:
+            # Pair with the next assistant message in chronological order (best effort)
+            ai_response = ""
+            j = i + 1
+            while j < len(turns_sorted):
+                if turns_sorted[j].get('role') == 'assistant':
+                    ai_response = turns_sorted[j].get('content', '')
+                    break
+                if turns_sorted[j].get('role') == 'user':
+                    break
+                j += 1
+
+            result = store_conversation_turn(
+                user_message=turn.get('content', ''),
+                ai_response=ai_response,
+                user_id=user_id,
+                turn_number=turn.get('turn', i),
+                conversation_id=f"mem-buffer-{turn.get('timestamp', 'unknown')[:10]}"
+            )
+
+            # store_conversation_turn returns success/skipped; treat skipped as ok
+            if result.get('success') or result.get('skipped'):
+                success_count += 1
+        except Exception as e:
+            print(f"Error storing user turn {turn.get('turn', '?')}: {e}", file=sys.stderr)
+
+    # Only consider Qdrant storage successful if we stored/skipped ALL user turns.
+    return attempted > 0 and success_count == attempted
+
+def store_to_file(turns, user_id):
+    """Fallback: Store turns to JSONL file."""
+    from datetime import datetime
+    
+    workspace = Path(os.getenv("OPENCLAW_WORKSPACE", str(Path.home() / ".openclaw" / "workspace")))
+    backup_dir = workspace / "memory" / "redis-backups"
+    backup_dir.mkdir(exist_ok=True)
+    
+    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+    filename = backup_dir / f"mem-backup-{user_id}-{timestamp}.jsonl"
+    
+    try:
+        with open(filename, 'w') as f:
+            for turn in turns:
+                f.write(json.dumps(turn) + '\n')
+        print(f"✅ Backed up {len(turns)} turns to file: {filename}")
+        return True
+    except Exception as e:
+        print(f"❌ File backup failed: {e}", file=sys.stderr)
+        return False
+
+def clear_redis(key):
+    """Clear Redis list after successful backup."""
+    try:
+        r = redis.Redis(host=REDIS_HOST, port=REDIS_PORT, decode_responses=True)
+        r.delete(key)
+        return True
+    except Exception as e:
+        print(f"Error clearing Redis: {e}", file=sys.stderr)
+        return False
+
+def main():
+    parser = argparse.ArgumentParser(description="Backup Redis mem buffer to Qdrant")
+    parser.add_argument("--user-id", default=USER_ID, help="User ID")
+    parser.add_argument("--dry-run", action="store_true", help="Don't actually clear Redis")
+    args = parser.parse_args()
+    
+    # Get items from Redis
+    turns, key = get_redis_items(args.user_id)
+    
+    if turns is None:
+        print("❌ Failed to read from Redis")
+        sys.exit(1)
+    
+    if not turns:
+        print(f"No items in Redis buffer (mem:{args.user_id})")
+        sys.exit(0)
+    
+    print(f"Found {len(turns)} turns in Redis buffer")
+    
+    # Try Qdrant first
+    qdrant_success = False
+    if not args.dry_run:
+        qdrant_success = store_to_qdrant(turns, args.user_id)
+        if qdrant_success:
+            print(f"✅ Stored Redis buffer to Qdrant (all user turns)")
+        else:
+            print("⚠️ Qdrant storage incomplete; will NOT clear Redis", file=sys.stderr)
+    else:
+        print("[DRY RUN] Would attempt Qdrant storage")
+        qdrant_success = True  # Dry run pretends success
+    
+    # If Qdrant failed/incomplete, try file backup (still do NOT clear Redis unless user chooses)
+    file_success = False
+    if not qdrant_success:
+        print("⚠️ Qdrant storage failed/incomplete, writing file backup (Redis preserved)...")
+        file_success = store_to_file(turns, args.user_id)
+        if not file_success:
+            print("❌ Both Qdrant and file backup failed - Redis buffer preserved")
+            sys.exit(1)
+        # Exit non-zero so monitoring can alert; keep Redis for re-try.
+        sys.exit(1)
+    
+    # Clear Redis (only if not dry-run)
+    if args.dry_run:
+        print("[DRY RUN] Would clear Redis buffer")
+        sys.exit(0)
+    
+    if clear_redis(key):
+        print(f"✅ Cleared Redis buffer (mem:{args.user_id})")
+    else:
+        print(f"⚠️ Backup succeeded but failed to clear Redis - may duplicate on next run")
+        sys.exit(1)
+    
+    backup_type = "Qdrant" if qdrant_success else "file"
+    print(f"\n🎉 Successfully backed up {len(turns)} turns to {backup_type} long-term memory")
+
+if __name__ == "__main__":
+    main()

skills/mem-redis/scripts/cron_capture.py

@@ -0,0 +1,230 @@
+#!/usr/bin/env python3
+"""
+Cron Capture: Append NEW session transcript messages to Redis (no LLM / no heartbeat).
+
+Goal: minimize token spend by capturing context out-of-band.
+
+- Tracks per-session file offsets (byte position) in a JSON state file.
+- No-ops if the transcript file hasn't changed since last run.
+- Stores user/assistant visible text to Redis (chronological order via RPUSH).
+- Optionally stores model "thinking" separately (disabled by default) so it can be
+  queried only when explicitly needed.
+
+Usage:
+  python3 cron_capture.py [--user-id rob] [--include-thinking]
+
+Suggested cron (every 5 minutes):
+  */5 * * * * cd ~/.openclaw/workspace && python3 skills/mem-redis/scripts/cron_capture.py --user-id $USER
+
+Env:
+  OPENCLAW_WORKSPACE: override workspace path (default: ~/.openclaw/workspace)
+  OPENCLAW_SESSIONS_DIR: override sessions dir (default: ~/.openclaw/agents/main/sessions)
+  REDIS_HOST / REDIS_PORT / USER_ID
+"""
+
+import argparse
+import json
+import os
+import sys
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Tuple
+
+REDIS_HOST = os.getenv("REDIS_HOST", "127.0.0.1")
+REDIS_PORT = int(os.getenv("REDIS_PORT", "6379"))
+USER_ID = os.getenv("USER_ID", "yourname")
+
+DEFAULT_WORKSPACE = Path(os.getenv("OPENCLAW_WORKSPACE", str(Path.home() / ".openclaw" / "workspace")))
+DEFAULT_SESSIONS_DIR = Path(os.getenv("OPENCLAW_SESSIONS_DIR", str(Path.home() / ".openclaw" / "agents" / "main" / "sessions")))
+
+STATE_FILE = DEFAULT_WORKSPACE / ".mem_capture_state.json"
+
+
+@dataclass
+class ParsedMessage:
+    role: str  # user|assistant
+    text: str
+    thinking: Optional[str]
+    timestamp: str
+    session_id: str
+
+
+def _now_iso() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def find_latest_transcript(sessions_dir: Path) -> Optional[Path]:
+    files = list(sessions_dir.glob("*.jsonl"))
+    if not files:
+        return None
+    return max(files, key=lambda p: p.stat().st_mtime)
+
+
+def load_state() -> Dict[str, Any]:
+    if not STATE_FILE.exists():
+        return {}
+    try:
+        return json.loads(STATE_FILE.read_text())
+    except Exception:
+        return {}
+
+
+def save_state(state: Dict[str, Any]) -> None:
+    try:
+        STATE_FILE.write_text(json.dumps(state, indent=2, sort_keys=True))
+    except Exception as e:
+        print(f"[cron_capture] Warning: could not write state: {e}", file=sys.stderr)
+
+
+def extract_text_and_thinking(content: Any) -> Tuple[str, Optional[str]]:
+    """Extract visible text and optional thinking from OpenClaw message content."""
+    if isinstance(content, str):
+        return content, None
+
+    text_parts: List[str] = []
+    thinking_parts: List[str] = []
+
+    if isinstance(content, list):
+        for item in content:
+            if not isinstance(item, dict):
+                continue
+            if "text" in item and isinstance(item["text"], str):
+                text_parts.append(item["text"])
+            if "thinking" in item and isinstance(item["thinking"], str):
+                thinking_parts.append(item["thinking"])
+
+    text = "".join(text_parts).strip()
+    thinking = "\n".join(thinking_parts).strip() if thinking_parts else None
+    return text, thinking
+
+
+def parse_new_messages(transcript_path: Path, start_offset: int, include_thinking: bool) -> Tuple[List[ParsedMessage], int]:
+    """Parse messages from transcript_path starting at byte offset."""
+    session_id = transcript_path.stem
+    msgs: List[ParsedMessage] = []
+
+    with transcript_path.open("rb") as f:
+        f.seek(start_offset)
+        while True:
+            line = f.readline()
+            if not line:
+                break
+            try:
+                entry = json.loads(line.decode("utf-8", errors="replace").strip())
+            except Exception:
+                continue
+
+            if entry.get("type") != "message" or "message" not in entry:
+                continue
+            msg = entry.get("message") or {}
+            role = msg.get("role")
+            if role not in ("user", "assistant"):
+                continue
+
+            # Skip tool results explicitly
+            if role == "toolResult":
+                continue
+
+            text, thinking = extract_text_and_thinking(msg.get("content"))
+            if not text and not (include_thinking and thinking):
+                continue
+
+            msgs.append(
+                ParsedMessage(
+                    role=role,
+                    text=text[:8000],
+                    thinking=(thinking[:16000] if (include_thinking and thinking) else None),
+                    timestamp=entry.get("timestamp") or _now_iso(),
+                    session_id=session_id,
+                )
+            )
+
+        end_offset = f.tell()
+
+    return msgs, end_offset
+
+
+def append_to_redis(user_id: str, messages: List[ParsedMessage]) -> int:
+    if not messages:
+        return 0
+
+    import redis  # lazy import so --dry-run works without deps
+    r = redis.Redis(host=REDIS_HOST, port=REDIS_PORT, decode_responses=True)
+
+    key = f"mem:{user_id}"
+    thinking_key = f"mem_thinking:{user_id}"
+
+    # RPUSH keeps chronological order.
+    for m in messages:
+        payload: Dict[str, Any] = {
+            "role": m.role,
+            "content": m.text,
+            "timestamp": m.timestamp,
+            "user_id": user_id,
+            "session": m.session_id,
+        }
+        r.rpush(key, json.dumps(payload))
+
+        if m.thinking:
+            t_payload = {
+                "role": m.role,
+                "thinking": m.thinking,
+                "timestamp": m.timestamp,
+                "user_id": user_id,
+                "session": m.session_id,
+            }
+            r.rpush(thinking_key, json.dumps(t_payload))
+
+    return len(messages)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Cron capture: append new transcript messages to Redis")
+    parser.add_argument("--user-id", default=USER_ID)
+    parser.add_argument("--include-thinking", action="store_true", help="Store thinking into mem_thinking:<user>")
+    parser.add_argument("--sessions-dir", default=str(DEFAULT_SESSIONS_DIR))
+    parser.add_argument("--dry-run", action="store_true", help="Parse + update state, but do not write to Redis")
+    args = parser.parse_args()
+
+    sessions_dir = Path(args.sessions_dir)
+    transcript = find_latest_transcript(sessions_dir)
+    if not transcript:
+        print("[cron_capture] No session transcripts found")
+        return
+
+    st = load_state()
+    key = str(transcript)
+    info = st.get(key, {})
+    last_offset = int(info.get("offset", 0))
+    last_size = int(info.get("size", 0))
+
+    cur_size = transcript.stat().st_size
+    if cur_size == last_size and last_offset > 0:
+        print("[cron_capture] No changes")
+        return
+
+    messages, end_offset = parse_new_messages(transcript, last_offset, include_thinking=args.include_thinking)
+    if not messages:
+        # Still update size/offset so we don't re-read noise lines.
+        st[key] = {"offset": end_offset, "size": cur_size, "updated_at": _now_iso()}
+        save_state(st)
+        print("[cron_capture] No new user/assistant messages")
+        return
+
+    if args.dry_run:
+        st[key] = {"offset": end_offset, "size": cur_size, "updated_at": _now_iso()}
+        save_state(st)
+        print(f"[cron_capture] DRY RUN: would append {len(messages)} messages to Redis mem:{args.user_id}")
+        return
+
+    count = append_to_redis(args.user_id, messages)
+
+    st[key] = {"offset": end_offset, "size": cur_size, "updated_at": _now_iso()}
+    save_state(st)
+
+    print(f"[cron_capture] Appended {count} messages to Redis mem:{args.user_id}")
+
+
+if __name__ == "__main__":
+    main()

skills/mem-redis/scripts/hb_append.py

@@ -0,0 +1,161 @@
+#!/usr/bin/env python3
+"""
+Heartbeat: Append new conversation turns to Redis short-term buffer.
+
+This script runs during heartbeat to capture recent conversation context
+before it gets compacted away. Stores in Redis until daily cron backs up to Qdrant.
+
+Usage: python3 hb_append.py [--user-id rob]
+"""
+
+import os
+import sys
+import json
+import redis
+import argparse
+from datetime import datetime, timezone
+from pathlib import Path
+
+# Config
+REDIS_HOST = os.getenv("REDIS_HOST", "127.0.0.1")
+REDIS_PORT = int(os.getenv("REDIS_PORT", "6379"))
+USER_ID = os.getenv("USER_ID", "yourname")
+
+# Paths (portable)
+WORKSPACE = Path(os.getenv("OPENCLAW_WORKSPACE", str(Path.home() / ".openclaw" / "workspace")))
+MEMORY_DIR = WORKSPACE / "memory"
+SESSIONS_DIR = Path(os.getenv("OPENCLAW_SESSIONS_DIR", str(Path.home() / ".openclaw" / "agents" / "main" / "sessions")))
+STATE_FILE = WORKSPACE / ".mem_last_turn"
+
+def get_session_transcript():
+    """Find the current session JSONL file."""
+    files = list(SESSIONS_DIR.glob("*.jsonl"))
+    if not files:
+        return None
+    # Get most recently modified
+    return max(files, key=lambda p: p.stat().st_mtime)
+
+def parse_turns_since(last_turn_num):
+    """Extract conversation turns since last processed."""
+    transcript_file = get_session_transcript()
+    if not transcript_file or not transcript_file.exists():
+        return []
+    
+    turns = []
+    turn_counter = last_turn_num
+    try:
+        with open(transcript_file, 'r') as f:
+            for line in f:
+                line = line.strip()
+                if not line:
+                    continue
+                try:
+                    entry = json.loads(line)
+                    # OpenClaw format: {"type": "message", "message": {"role": "...", ...}}
+                    if entry.get('type') == 'message' and 'message' in entry:
+                        msg = entry['message']
+                        role = msg.get('role')
+                        
+                        # Skip tool results for memory storage
+                        if role == 'toolResult':
+                            continue
+                            
+                        # Get content from message content array or string
+                        content = ""
+                        if isinstance(msg.get('content'), list):
+                            # Extract text from content array
+                            for item in msg['content']:
+                                if isinstance(item, dict):
+                                    if 'text' in item:
+                                        content += item['text']
+                                    # Intentionally do NOT store model thinking in the main buffer.
+                                    # If you need thinking, use cron_capture.py --include-thinking to store it
+                                    # separately under mem_thinking:<user_id>.
+                                    elif 'thinking' in item:
+                                        pass
+                        elif isinstance(msg.get('content'), str):
+                            content = msg['content']
+                        
+                        if content and role in ('user', 'assistant'):
+                            turn_counter += 1
+                            turns.append({
+                                'turn': turn_counter,
+                                'role': role,
+                                'content': content[:2000],
+                                'timestamp': entry.get('timestamp', datetime.now(timezone.utc).isoformat()),
+                                'user_id': USER_ID,
+                                'session': str(transcript_file.name).replace('.jsonl', '')
+                            })
+                except json.JSONDecodeError:
+                    continue
+    except Exception as e:
+        print(f"Error reading transcript: {e}", file=sys.stderr)
+        return []
+    
+    return turns
+
+def get_last_turn():
+    """Get last turn number from state file."""
+    if STATE_FILE.exists():
+        try:
+            with open(STATE_FILE) as f:
+                return int(f.read().strip())
+        except:
+            pass
+    return 0
+
+def save_last_turn(turn_num):
+    """Save last turn number to state file."""
+    try:
+        with open(STATE_FILE, 'w') as f:
+            f.write(str(turn_num))
+    except Exception as e:
+        print(f"Warning: Could not save state: {e}", file=sys.stderr)
+
+def append_to_redis(turns, user_id):
+    """Append turns to Redis list."""
+    if not turns:
+        return 0
+    
+    try:
+        r = redis.Redis(host=REDIS_HOST, port=REDIS_PORT, decode_responses=True)
+        key = f"mem:{user_id}"
+        
+        # Add all turns to list (LPUSH puts newest at front)
+        for turn in turns:
+            r.lpush(key, json.dumps(turn))
+        
+        return len(turns)
+    except Exception as e:
+        print(f"Error writing to Redis: {e}", file=sys.stderr)
+        return 0
+
+def main():
+    parser = argparse.ArgumentParser(description="Append new turns to Redis mem buffer")
+    parser.add_argument("--user-id", default=USER_ID, help="User ID for key naming")
+    args = parser.parse_args()
+    
+    # Get last processed turn
+    last_turn = get_last_turn()
+    
+    # Get new turns
+    new_turns = parse_turns_since(last_turn)
+    
+    if not new_turns:
+        print(f"No new turns since turn {last_turn}")
+        sys.exit(0)
+    
+    # Append to Redis
+    count = append_to_redis(new_turns, args.user_id)
+    
+    if count > 0:
+        # Update last turn tracker
+        max_turn = max(t['turn'] for t in new_turns)
+        save_last_turn(max_turn)
+        print(f"✅ Appended {count} turns to Redis (mem:{args.user_id})")
+    else:
+        print("❌ Failed to append to Redis")
+        sys.exit(1)
+
+if __name__ == "__main__":
+    main()

skills/mem-redis/scripts/mem_retrieve.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+"""
+Manual Retrieval: Get recent conversation turns from Redis buffer.
+
+Use this when context has been compacted or you need to recall recent details.
+
+Usage: python3 mem_retrieve.py [--limit 20] [--user-id rob]
+"""
+
+import os
+import sys
+import json
+import redis
+import argparse
+from datetime import datetime, timezone
+
+# Config
+REDIS_HOST = os.getenv("REDIS_HOST", "127.0.0.1")
+REDIS_PORT = int(os.getenv("REDIS_PORT", "6379"))
+USER_ID = os.getenv("USER_ID", "yourname")
+
+def get_recent_turns(user_id, limit=20):
+    """Get recent turns from Redis buffer."""
+    try:
+        r = redis.Redis(host=REDIS_HOST, port=REDIS_PORT, decode_responses=True)
+        key = f"mem:{user_id}"
+        
+        # Get most recent N items (0 to limit-1)
+        items = r.lrange(key, 0, limit - 1)
+        
+        # Parse and reverse (so oldest first)
+        turns = []
+        for item in items:
+            try:
+                turn = json.loads(item)
+                turns.append(turn)
+            except json.JSONDecodeError:
+                continue
+        
+        # Reverse to chronological order
+        turns.reverse()
+        
+        return turns
+    except Exception as e:
+        print(f"Error reading from Redis: {e}", file=sys.stderr)
+        return []
+
+def format_turn(turn):
+    """Format a turn for display."""
+    role = turn.get('role', 'unknown')
+    content = turn.get('content', '')
+    turn_num = turn.get('turn', '?')
+    
+    # Truncate long content
+    if len(content) > 500:
+        content = content[:500] + "..."
+    
+    role_icon = "👤" if role == 'user' else "🤖"
+    return f"{role_icon} Turn {turn_num} ({role}):\n{content}\n"
+
+def main():
+    parser = argparse.ArgumentParser(description="Retrieve recent turns from mem buffer")
+    parser.add_argument("--user-id", default=USER_ID, help="User ID")
+    parser.add_argument("--limit", type=int, default=20, help="Number of turns to retrieve")
+    args = parser.parse_args()
+    
+    # Get turns
+    turns = get_recent_turns(args.user_id, args.limit)
+    
+    if not turns:
+        print(f"No recent turns in memory buffer (mem:{args.user_id})")
+        print("\nPossible reasons:")
+        print("  - Heartbeat hasn't run yet")
+        print("  - Cron already backed up and cleared Redis")
+        print("  - Redis connection issue")
+        sys.exit(0)
+    
+    # Display
+    print(f"=== Recent {len(turns)} Turn(s) from Memory Buffer ===\n")
+    for turn in turns:
+        print(format_turn(turn))
+    
+    print(f"\nBuffer key: mem:{args.user_id}")
+    print("Note: These turns are also in Redis until daily cron backs them up to Qdrant.")
+
+if __name__ == "__main__":
+    main()

skills/mem-redis/scripts/save_mem.py

@@ -0,0 +1,149 @@
+#!/usr/bin/env python3
+"""
+Save all conversation context to Redis (not just new turns).
+
+Unlike hb_append.py which only saves NEW turns since last run,
+this script saves ALL context from the session (or resets and saves fresh).
+
+Usage: python3 save_mem.py [--user-id rob] [--reset]
+"""
+
+import os
+import sys
+import json
+import redis
+import argparse
+from datetime import datetime, timezone
+from pathlib import Path
+
+# Config
+REDIS_HOST = os.getenv("REDIS_HOST", "127.0.0.1")
+REDIS_PORT = int(os.getenv("REDIS_PORT", "6379"))
+USER_ID = os.getenv("USER_ID", "yourname")
+
+# Paths (portable)
+WORKSPACE = Path(os.getenv("OPENCLAW_WORKSPACE", str(Path.home() / ".openclaw" / "workspace")))
+SESSIONS_DIR = Path(os.getenv("OPENCLAW_SESSIONS_DIR", str(Path.home() / ".openclaw" / "agents" / "main" / "sessions")))
+STATE_FILE = WORKSPACE / ".mem_last_turn"
+
+def get_session_transcript():
+    """Find the current session JSONL file."""
+    files = list(SESSIONS_DIR.glob("*.jsonl"))
+    if not files:
+        return None
+    return max(files, key=lambda p: p.stat().st_mtime)
+
+def parse_all_turns():
+    """Extract ALL conversation turns from current session."""
+    transcript_file = get_session_transcript()
+    if not transcript_file or not transcript_file.exists():
+        return []
+    
+    turns = []
+    turn_counter = 0
+    try:
+        with open(transcript_file, 'r') as f:
+            for line in f:
+                line = line.strip()
+                if not line:
+                    continue
+                try:
+                    entry = json.loads(line)
+                    if entry.get('type') == 'message' and 'message' in entry:
+                        msg = entry['message']
+                        role = msg.get('role')
+                        
+                        if role == 'toolResult':
+                            continue
+                            
+                        content = ""
+                        if isinstance(msg.get('content'), list):
+                            for item in msg['content']:
+                                if isinstance(item, dict):
+                                    if 'text' in item:
+                                        content += item['text']
+                                    # Do not mix thinking into the main content buffer.
+                                    elif 'thinking' in item:
+                                        pass
+                        elif isinstance(msg.get('content'), str):
+                            content = msg['content']
+                        
+                        if content and role in ('user', 'assistant'):
+                            turn_counter += 1
+                            turns.append({
+                                'turn': turn_counter,
+                                'role': role,
+                                'content': content[:2000],
+                                'timestamp': entry.get('timestamp', datetime.now(timezone.utc).isoformat()),
+                                'user_id': USER_ID,
+                                'session': str(transcript_file.name).replace('.jsonl', '')
+                            })
+                except json.JSONDecodeError:
+                    continue
+    except Exception as e:
+        print(f"Error reading transcript: {e}", file=sys.stderr)
+        return []
+    
+    return turns
+
+def save_to_redis(turns, user_id, reset=False):
+    """Save turns to Redis. If reset, clear existing first."""
+    if not turns:
+        return 0
+    
+    try:
+        r = redis.Redis(host=REDIS_HOST, port=REDIS_PORT, decode_responses=True)
+        key = f"mem:{user_id}"
+        
+        # Clear existing if reset
+        if reset:
+            r.delete(key)
+            print(f"Cleared existing Redis buffer ({key})")
+        
+        # Add all turns (LPUSH puts newest at front, so we reverse to keep order)
+        for turn in reversed(turns):
+            r.lpush(key, json.dumps(turn))
+        
+        return len(turns)
+    except Exception as e:
+        print(f"Error writing to Redis: {e}", file=sys.stderr)
+        return 0
+
+def update_state(last_turn_num):
+    """Update last turn tracker."""
+    try:
+        with open(STATE_FILE, 'w') as f:
+            f.write(str(last_turn_num))
+    except Exception as e:
+        print(f"Warning: Could not save state: {e}", file=sys.stderr)
+
+def main():
+    parser = argparse.ArgumentParser(description="Save all conversation context to Redis")
+    parser.add_argument("--user-id", default=USER_ID, help="User ID for key naming")
+    parser.add_argument("--reset", action="store_true", help="Clear existing buffer first")
+    args = parser.parse_args()
+    
+    # Get all turns
+    turns = parse_all_turns()
+    
+    if not turns:
+        print("No conversation turns found in session")
+        sys.exit(0)
+    
+    # Save to Redis
+    count = save_to_redis(turns, args.user_id, reset=args.reset)
+    
+    if count > 0:
+        # Update state to track last turn
+        max_turn = max(t['turn'] for t in turns)
+        update_state(max_turn)
+        
+        action = "Reset and saved" if args.reset else "Saved"
+        print(f"✅ {action} {count} turns to Redis (mem:{args.user_id})")
+        print(f"   State updated to turn {max_turn}")
+    else:
+        print("❌ Failed to save to Redis")
+        sys.exit(1)
+
+if __name__ == "__main__":
+    main()

skills/mem-redis/scripts/search_mem.py

@@ -0,0 +1,242 @@
+#!/usr/bin/env python3
+"""
+Search memory: First Redis (exact), then Qdrant (semantic).
+
+Usage: python3 search_mem.py "your search query" [--limit 10] [--user-id rob]
+
+Searches:
+1. Redis (mem:{user_id}) - exact text match in recent buffer
+2. Qdrant (kimi_memories) - semantic similarity search
+"""
+
+import os
+import sys
+import json
+import redis
+import argparse
+from pathlib import Path
+from datetime import datetime
+
+# Config
+REDIS_HOST = os.getenv("REDIS_HOST", "10.0.0.36")
+REDIS_PORT = int(os.getenv("REDIS_PORT", "6379"))
+USER_ID = os.getenv("USER_ID", "yourname")
+
+QDRANT_URL = os.getenv("QDRANT_URL", "http://10.0.0.40:6333")
+OLLAMA_URL = os.getenv("OLLAMA_URL", "http://10.0.0.10:11434/v1")
+
+def search_redis(query, user_id, limit=20):
+    """Search Redis buffer for exact text matches."""
+    try:
+        r = redis.Redis(host=REDIS_HOST, port=REDIS_PORT, decode_responses=True)
+        key = f"mem:{user_id}"
+        
+        # Get all items from list
+        items = r.lrange(key, 0, -1)
+        if not items:
+            return []
+        
+        query_lower = query.lower()
+        matches = []
+        
+        for item in items:
+            try:
+                turn = json.loads(item)
+                content = turn.get('content', '').lower()
+                if query_lower in content:
+                    matches.append({
+                        'source': 'redis',
+                        'turn': turn.get('turn'),
+                        'role': turn.get('role'),
+                        'content': turn.get('content'),
+                        'timestamp': turn.get('timestamp'),
+                        'score': 'exact'
+                    })
+            except json.JSONDecodeError:
+                continue
+        
+        # Sort by turn number descending (newest first)
+        matches.sort(key=lambda x: x.get('turn', 0), reverse=True)
+        return matches[:limit]
+    except Exception as e:
+        print(f"Redis search error: {e}", file=sys.stderr)
+        return []
+
+def get_embedding(text):
+    """Get embedding from Ollama."""
+    import urllib.request
+    
+    payload = json.dumps({
+        "model": "snowflake-arctic-embed2",
+        "input": text
+    }).encode()
+    
+    req = urllib.request.Request(
+        f"{OLLAMA_URL}/embeddings",
+        data=payload,
+        headers={"Content-Type": "application/json"},
+        method="POST"
+    )
+    
+    try:
+        with urllib.request.urlopen(req, timeout=30) as resp:
+            result = json.loads(resp.read().decode())
+            return result.get('data', [{}])[0].get('embedding')
+    except Exception as e:
+        print(f"Embedding error: {e}", file=sys.stderr)
+        return None
+
+def search_qdrant(query, user_id, limit=10):
+    """Search Qdrant for semantic similarity."""
+    import urllib.request
+    
+    embedding = get_embedding(query)
+    if not embedding:
+        return []
+    
+    payload = json.dumps({
+        "vector": embedding,
+        "limit": limit,
+        "with_payload": True,
+        "filter": {
+            "must": [
+                {"key": "user_id", "match": {"value": user_id}}
+            ]
+        }
+    }).encode()
+    
+    req = urllib.request.Request(
+        f"{QDRANT_URL}/collections/kimi_memories/points/search",
+        data=payload,
+        headers={"Content-Type": "application/json"},
+        method="POST"
+    )
+    
+    try:
+        with urllib.request.urlopen(req, timeout=30) as resp:
+            result = json.loads(resp.read().decode())
+            points = result.get('result', [])
+            
+            matches = []
+            for point in points:
+                payload = point.get('payload', {})
+                matches.append({
+                    'source': 'qdrant',
+                    'score': round(point.get('score', 0), 3),
+                    'turn': payload.get('turn_number'),
+                    'role': payload.get('role'),
+                    'content': payload.get('user_message') or payload.get('content', ''),
+                    'ai_response': payload.get('ai_response', ''),
+                    'timestamp': payload.get('timestamp'),
+                    'conversation_id': payload.get('conversation_id')
+                })
+            return matches
+    except Exception as e:
+        print(f"Qdrant search error: {e}", file=sys.stderr)
+        return []
+
+def format_result(result, index):
+    """Format a single search result."""
+    source = result.get('source', 'unknown')
+    role = result.get('role', 'unknown')
+    turn = result.get('turn', '?')
+    score = result.get('score', '?')
+    
+    content = result.get('content', '')
+    if len(content) > 200:
+        content = content[:200] + "..."
+    
+    # Role emoji
+    role_emoji = "👤" if role == "user" else "🤖"
+    
+    # Source indicator
+    source_icon = "🔴" if source == "redis" else "🔵"
+    
+    lines = [
+        f"{source_icon} [{index}] Turn {turn} ({role}):",
+        f"   {role_emoji} {content}"
+    ]
+    
+    if source == "qdrant" and result.get('ai_response'):
+        ai_resp = result['ai_response'][:150]
+        if len(result['ai_response']) > 150:
+            ai_resp += "..."
+        lines.append(f"   💬 AI: {ai_resp}")
+    
+    if score != 'exact':
+        lines.append(f"   📊 Score: {score}")
+    else:
+        lines.append(f"   📊 Match: exact (Redis)")
+    
+    return "\n".join(lines)
+
+def main():
+    parser = argparse.ArgumentParser(description="Search memory: Redis first, then Qdrant")
+    parser.add_argument("query", help="Search query")
+    parser.add_argument("--limit", type=int, default=10, help="Results per source (default: 10)")
+    parser.add_argument("--user-id", default=USER_ID, help="User ID")
+    parser.add_argument("--redis-only", action="store_true", help="Only search Redis")
+    parser.add_argument("--qdrant-only", action="store_true", help="Only search Qdrant")
+    args = parser.parse_args()
+    
+    print(f"🔍 Searching for: \"{args.query}\"\n")
+    
+    all_results = []
+    
+    # Search Redis first (unless qdrant-only)
+    if not args.qdrant_only:
+        print("📍 Searching Redis (exact match)...")
+        redis_results = search_redis(args.query, args.user_id, limit=args.limit)
+        if redis_results:
+            print(f"✅ Found {len(redis_results)} matches in Redis\n")
+            all_results.extend(redis_results)
+        else:
+            print("❌ No exact matches in Redis\n")
+    
+    # Search Qdrant (unless redis-only)
+    if not args.redis_only:
+        print("🧠 Searching Qdrant (semantic similarity)...")
+        qdrant_results = search_qdrant(args.query, args.user_id, limit=args.limit)
+        if qdrant_results:
+            print(f"✅ Found {len(qdrant_results)} matches in Qdrant\n")
+            all_results.extend(qdrant_results)
+        else:
+            print("❌ No semantic matches in Qdrant\n")
+    
+    # Display results
+    if not all_results:
+        print("No results found in either Redis or Qdrant.")
+        sys.exit(0)
+    
+    print(f"=== Search Results ({len(all_results)} total) ===\n")
+    
+    # Sort: Redis first (chronological), then Qdrant (by score)
+    redis_sorted = [r for r in all_results if r['source'] == 'redis']
+    qdrant_sorted = sorted(
+        [r for r in all_results if r['source'] == 'qdrant'],
+        key=lambda x: x.get('score', 0),
+        reverse=True
+    )
+    
+    # Display Redis results first
+    if redis_sorted:
+        print("🔴 FROM REDIS (Recent Buffer):\n")
+        for i, result in enumerate(redis_sorted, 1):
+            print(format_result(result, i))
+            print()
+    
+    # Then Qdrant results
+    if qdrant_sorted:
+        print("🔵 FROM QDRANT (Long-term Memory):\n")
+        for i, result in enumerate(qdrant_sorted, len(redis_sorted) + 1):
+            print(format_result(result, i))
+            print()
+    
+    print(f"=== {len(all_results)} results ===")
+    if redis_sorted:
+        print(f"  🔴 Redis: {len(redis_sorted)} (exact, recent)")
+    if qdrant_sorted:
+        print(f"  🔵 Qdrant: {len(qdrant_sorted)} (semantic, long-term)")
+
+if __name__ == "__main__":
+    main()