jarvis-memory/skills/mem-redis/scripts/cron_backup.py

#!/usr/bin/env python3
"""
Daily Cron: Process Redis buffer → Qdrant → Clear Redis.

This script runs once daily (via cron) to move buffered conversation
turns from Redis to durable Qdrant storage. Only clears Redis after
successful Qdrant write.

Usage: python3 cron_backup.py [--user-id rob] [--dry-run]
"""

import os
import sys
import json
import redis
import argparse
from datetime import datetime, timezone
from pathlib import Path

# Add qdrant-memory to path (portable)
from pathlib import Path as _Path
WORKSPACE = _Path(os.getenv("OPENCLAW_WORKSPACE", str(_Path.home() / ".openclaw" / "workspace")))
sys.path.insert(0, str(WORKSPACE / "skills" / "qdrant-memory" / "scripts"))

try:
    from auto_store import store_conversation_turn
    QDRANT_AVAILABLE = True
except ImportError:
    QDRANT_AVAILABLE = False
    print("Warning: Qdrant storage not available, will simulate", file=sys.stderr)

# Config
REDIS_HOST = os.getenv("REDIS_HOST", "127.0.0.1")
REDIS_PORT = int(os.getenv("REDIS_PORT", "6379"))
USER_ID = os.getenv("USER_ID", "yourname")

def get_redis_items(user_id):
    """Get all items from Redis list."""
    try:
        r = redis.Redis(host=REDIS_HOST, port=REDIS_PORT, decode_responses=True)
        key = f"mem:{user_id}"
        
        # Get all items (0 to -1 = entire list)
        items = r.lrange(key, 0, -1)
        
        # Parse JSON
        turns = []
        for item in items:
            try:
                turn = json.loads(item)
                turns.append(turn)
            except json.JSONDecodeError:
                continue
        
        return turns, key
    except Exception as e:
        print(f"Error reading from Redis: {e}", file=sys.stderr)
        return None, None

def store_to_qdrant(turns, user_id):
    """Store turns to Qdrant with file fallback."""
    if not QDRANT_AVAILABLE:
        print("[DRY RUN] Would store to Qdrant:", file=sys.stderr)
        for turn in turns[:3]:
            print(f"  - Turn {turn.get('turn', '?')}: {turn.get('role', '?')}", file=sys.stderr)
        if len(turns) > 3:
            print(f"  ... and {len(turns) - 3} more", file=sys.stderr)
        return True
    
    # Ensure chronological order (older -> newer)
    try:
        turns_sorted = sorted(turns, key=lambda t: (t.get('timestamp', ''), t.get('turn', 0)))
    except Exception:
        turns_sorted = turns

    user_turns = [t for t in turns_sorted if t.get('role') == 'user']
    if not user_turns:
        return True

    success_count = 0
    attempted = 0

    for i, turn in enumerate(turns_sorted):
        if turn.get('role') != 'user':
            continue
        attempted += 1
        try:
            # Pair with the next assistant message in chronological order (best effort)
            ai_response = ""
            j = i + 1
            while j < len(turns_sorted):
                if turns_sorted[j].get('role') == 'assistant':
                    ai_response = turns_sorted[j].get('content', '')
                    break
                if turns_sorted[j].get('role') == 'user':
                    break
                j += 1

            result = store_conversation_turn(
                user_message=turn.get('content', ''),
                ai_response=ai_response,
                user_id=user_id,
                turn_number=turn.get('turn', i),
                conversation_id=f"mem-buffer-{turn.get('timestamp', 'unknown')[:10]}"
            )

            # store_conversation_turn returns success/skipped; treat skipped as ok
            if result.get('success') or result.get('skipped'):
                success_count += 1
        except Exception as e:
            print(f"Error storing user turn {turn.get('turn', '?')}: {e}", file=sys.stderr)

    # Only consider Qdrant storage successful if we stored/skipped ALL user turns.
    return attempted > 0 and success_count == attempted

def store_to_file(turns, user_id):
    """Fallback: Store turns to JSONL file."""
    from datetime import datetime
    
    workspace = Path(os.getenv("OPENCLAW_WORKSPACE", str(Path.home() / ".openclaw" / "workspace")))
    backup_dir = workspace / "memory" / "redis-backups"
    backup_dir.mkdir(exist_ok=True)
    
    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
    filename = backup_dir / f"mem-backup-{user_id}-{timestamp}.jsonl"
    
    try:
        with open(filename, 'w') as f:
            for turn in turns:
                f.write(json.dumps(turn) + '\n')
        print(f"✅ Backed up {len(turns)} turns to file: {filename}")
        return True
    except Exception as e:
        print(f"❌ File backup failed: {e}", file=sys.stderr)
        return False

def clear_redis(key):
    """Clear Redis list after successful backup."""
    try:
        r = redis.Redis(host=REDIS_HOST, port=REDIS_PORT, decode_responses=True)
        r.delete(key)
        return True
    except Exception as e:
        print(f"Error clearing Redis: {e}", file=sys.stderr)
        return False

def main():
    parser = argparse.ArgumentParser(description="Backup Redis mem buffer to Qdrant")
    parser.add_argument("--user-id", default=USER_ID, help="User ID")
    parser.add_argument("--dry-run", action="store_true", help="Don't actually clear Redis")
    args = parser.parse_args()
    
    # Get items from Redis
    turns, key = get_redis_items(args.user_id)
    
    if turns is None:
        print("❌ Failed to read from Redis")
        sys.exit(1)
    
    if not turns:
        print(f"No items in Redis buffer (mem:{args.user_id})")
        sys.exit(0)
    
    print(f"Found {len(turns)} turns in Redis buffer")
    
    # Try Qdrant first
    qdrant_success = False
    if not args.dry_run:
        qdrant_success = store_to_qdrant(turns, args.user_id)
        if qdrant_success:
            print(f"✅ Stored Redis buffer to Qdrant (all user turns)")
        else:
            print("⚠️ Qdrant storage incomplete; will NOT clear Redis", file=sys.stderr)
    else:
        print("[DRY RUN] Would attempt Qdrant storage")
        qdrant_success = True  # Dry run pretends success
    
    # If Qdrant failed/incomplete, try file backup (still do NOT clear Redis unless user chooses)
    file_success = False
    if not qdrant_success:
        print("⚠️ Qdrant storage failed/incomplete, writing file backup (Redis preserved)...")
        file_success = store_to_file(turns, args.user_id)
        if not file_success:
            print("❌ Both Qdrant and file backup failed - Redis buffer preserved")
            sys.exit(1)
        # Exit non-zero so monitoring can alert; keep Redis for re-try.
        sys.exit(1)
    
    # Clear Redis (only if not dry-run)
    if args.dry_run:
        print("[DRY RUN] Would clear Redis buffer")
        sys.exit(0)
    
    if clear_redis(key):
        print(f"✅ Cleared Redis buffer (mem:{args.user_id})")
    else:
        print(f"⚠️ Backup succeeded but failed to clear Redis - may duplicate on next run")
        sys.exit(1)
    
    backup_type = "Qdrant" if qdrant_success else "file"
    print(f"\n🎉 Successfully backed up {len(turns)} turns to {backup_type} long-term memory")

if __name__ == "__main__":
    main()