Initial commit: Jarvis Memory system
This commit is contained in:
204
skills/mem-redis/scripts/cron_backup.py
Executable file
204
skills/mem-redis/scripts/cron_backup.py
Executable file
@@ -0,0 +1,204 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Daily Cron: Process Redis buffer → Qdrant → Clear Redis.
|
||||
|
||||
This script runs once daily (via cron) to move buffered conversation
|
||||
turns from Redis to durable Qdrant storage. Only clears Redis after
|
||||
successful Qdrant write.
|
||||
|
||||
Usage: python3 cron_backup.py [--user-id rob] [--dry-run]
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import redis
|
||||
import argparse
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
# Add qdrant-memory to path (portable)
|
||||
from pathlib import Path as _Path
|
||||
WORKSPACE = _Path(os.getenv("OPENCLAW_WORKSPACE", str(_Path.home() / ".openclaw" / "workspace")))
|
||||
sys.path.insert(0, str(WORKSPACE / "skills" / "qdrant-memory" / "scripts"))
|
||||
|
||||
try:
|
||||
from auto_store import store_conversation_turn
|
||||
QDRANT_AVAILABLE = True
|
||||
except ImportError:
|
||||
QDRANT_AVAILABLE = False
|
||||
print("Warning: Qdrant storage not available, will simulate", file=sys.stderr)
|
||||
|
||||
# Config
|
||||
REDIS_HOST = os.getenv("REDIS_HOST", "127.0.0.1")
|
||||
REDIS_PORT = int(os.getenv("REDIS_PORT", "6379"))
|
||||
USER_ID = os.getenv("USER_ID", "yourname")
|
||||
|
||||
def get_redis_items(user_id):
|
||||
"""Get all items from Redis list."""
|
||||
try:
|
||||
r = redis.Redis(host=REDIS_HOST, port=REDIS_PORT, decode_responses=True)
|
||||
key = f"mem:{user_id}"
|
||||
|
||||
# Get all items (0 to -1 = entire list)
|
||||
items = r.lrange(key, 0, -1)
|
||||
|
||||
# Parse JSON
|
||||
turns = []
|
||||
for item in items:
|
||||
try:
|
||||
turn = json.loads(item)
|
||||
turns.append(turn)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
|
||||
return turns, key
|
||||
except Exception as e:
|
||||
print(f"Error reading from Redis: {e}", file=sys.stderr)
|
||||
return None, None
|
||||
|
||||
def store_to_qdrant(turns, user_id):
|
||||
"""Store turns to Qdrant with file fallback."""
|
||||
if not QDRANT_AVAILABLE:
|
||||
print("[DRY RUN] Would store to Qdrant:", file=sys.stderr)
|
||||
for turn in turns[:3]:
|
||||
print(f" - Turn {turn.get('turn', '?')}: {turn.get('role', '?')}", file=sys.stderr)
|
||||
if len(turns) > 3:
|
||||
print(f" ... and {len(turns) - 3} more", file=sys.stderr)
|
||||
return True
|
||||
|
||||
# Ensure chronological order (older -> newer)
|
||||
try:
|
||||
turns_sorted = sorted(turns, key=lambda t: (t.get('timestamp', ''), t.get('turn', 0)))
|
||||
except Exception:
|
||||
turns_sorted = turns
|
||||
|
||||
user_turns = [t for t in turns_sorted if t.get('role') == 'user']
|
||||
if not user_turns:
|
||||
return True
|
||||
|
||||
success_count = 0
|
||||
attempted = 0
|
||||
|
||||
for i, turn in enumerate(turns_sorted):
|
||||
if turn.get('role') != 'user':
|
||||
continue
|
||||
attempted += 1
|
||||
try:
|
||||
# Pair with the next assistant message in chronological order (best effort)
|
||||
ai_response = ""
|
||||
j = i + 1
|
||||
while j < len(turns_sorted):
|
||||
if turns_sorted[j].get('role') == 'assistant':
|
||||
ai_response = turns_sorted[j].get('content', '')
|
||||
break
|
||||
if turns_sorted[j].get('role') == 'user':
|
||||
break
|
||||
j += 1
|
||||
|
||||
result = store_conversation_turn(
|
||||
user_message=turn.get('content', ''),
|
||||
ai_response=ai_response,
|
||||
user_id=user_id,
|
||||
turn_number=turn.get('turn', i),
|
||||
conversation_id=f"mem-buffer-{turn.get('timestamp', 'unknown')[:10]}"
|
||||
)
|
||||
|
||||
# store_conversation_turn returns success/skipped; treat skipped as ok
|
||||
if result.get('success') or result.get('skipped'):
|
||||
success_count += 1
|
||||
except Exception as e:
|
||||
print(f"Error storing user turn {turn.get('turn', '?')}: {e}", file=sys.stderr)
|
||||
|
||||
# Only consider Qdrant storage successful if we stored/skipped ALL user turns.
|
||||
return attempted > 0 and success_count == attempted
|
||||
|
||||
def store_to_file(turns, user_id):
|
||||
"""Fallback: Store turns to JSONL file."""
|
||||
from datetime import datetime
|
||||
|
||||
workspace = Path(os.getenv("OPENCLAW_WORKSPACE", str(Path.home() / ".openclaw" / "workspace")))
|
||||
backup_dir = workspace / "memory" / "redis-backups"
|
||||
backup_dir.mkdir(exist_ok=True)
|
||||
|
||||
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
filename = backup_dir / f"mem-backup-{user_id}-{timestamp}.jsonl"
|
||||
|
||||
try:
|
||||
with open(filename, 'w') as f:
|
||||
for turn in turns:
|
||||
f.write(json.dumps(turn) + '\n')
|
||||
print(f"✅ Backed up {len(turns)} turns to file: {filename}")
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"❌ File backup failed: {e}", file=sys.stderr)
|
||||
return False
|
||||
|
||||
def clear_redis(key):
|
||||
"""Clear Redis list after successful backup."""
|
||||
try:
|
||||
r = redis.Redis(host=REDIS_HOST, port=REDIS_PORT, decode_responses=True)
|
||||
r.delete(key)
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"Error clearing Redis: {e}", file=sys.stderr)
|
||||
return False
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Backup Redis mem buffer to Qdrant")
|
||||
parser.add_argument("--user-id", default=USER_ID, help="User ID")
|
||||
parser.add_argument("--dry-run", action="store_true", help="Don't actually clear Redis")
|
||||
args = parser.parse_args()
|
||||
|
||||
# Get items from Redis
|
||||
turns, key = get_redis_items(args.user_id)
|
||||
|
||||
if turns is None:
|
||||
print("❌ Failed to read from Redis")
|
||||
sys.exit(1)
|
||||
|
||||
if not turns:
|
||||
print(f"No items in Redis buffer (mem:{args.user_id})")
|
||||
sys.exit(0)
|
||||
|
||||
print(f"Found {len(turns)} turns in Redis buffer")
|
||||
|
||||
# Try Qdrant first
|
||||
qdrant_success = False
|
||||
if not args.dry_run:
|
||||
qdrant_success = store_to_qdrant(turns, args.user_id)
|
||||
if qdrant_success:
|
||||
print(f"✅ Stored Redis buffer to Qdrant (all user turns)")
|
||||
else:
|
||||
print("⚠️ Qdrant storage incomplete; will NOT clear Redis", file=sys.stderr)
|
||||
else:
|
||||
print("[DRY RUN] Would attempt Qdrant storage")
|
||||
qdrant_success = True # Dry run pretends success
|
||||
|
||||
# If Qdrant failed/incomplete, try file backup (still do NOT clear Redis unless user chooses)
|
||||
file_success = False
|
||||
if not qdrant_success:
|
||||
print("⚠️ Qdrant storage failed/incomplete, writing file backup (Redis preserved)...")
|
||||
file_success = store_to_file(turns, args.user_id)
|
||||
if not file_success:
|
||||
print("❌ Both Qdrant and file backup failed - Redis buffer preserved")
|
||||
sys.exit(1)
|
||||
# Exit non-zero so monitoring can alert; keep Redis for re-try.
|
||||
sys.exit(1)
|
||||
|
||||
# Clear Redis (only if not dry-run)
|
||||
if args.dry_run:
|
||||
print("[DRY RUN] Would clear Redis buffer")
|
||||
sys.exit(0)
|
||||
|
||||
if clear_redis(key):
|
||||
print(f"✅ Cleared Redis buffer (mem:{args.user_id})")
|
||||
else:
|
||||
print(f"⚠️ Backup succeeded but failed to clear Redis - may duplicate on next run")
|
||||
sys.exit(1)
|
||||
|
||||
backup_type = "Qdrant" if qdrant_success else "file"
|
||||
print(f"\n🎉 Successfully backed up {len(turns)} turns to {backup_type} long-term memory")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user