Initial commit: Jarvis Memory system

skills/task-queue/SKILL.md

@@ -0,0 +1,33 @@
+# Task Queue Skill
+
+Redis-based task queue for background jobs.
+
+## What It Does
+
+Queues and executes tasks via heartbeat worker.
+
+## Commands
+
+```bash
+# Add a task
+python3 scripts/add_task.py "Check disk space"
+
+# List tasks
+python3 scripts/list_tasks.py
+
+# Execute (runs on heartbeat)
+python3 scripts/heartbeat_worker.py
+```
+
+## Heartbeat Integration
+
+Add to HEARTBEAT.md:
+```bash
+python3 /path/to/skills/task-queue/scripts/heartbeat_worker.py
+```
+
+## Files
+
+- `add_task.py` - Add task to queue
+- `list_tasks.py` - View queue status
+- `heartbeat_worker.py` - Execute pending tasks

skills/task-queue/scripts/add_task.py

@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+"""
+Add a task to the queue.
+Usage: python3 add_task.py "Task description" [options]
+"""
+
+import redis
+import sys
+import time
+import os
+import argparse
+
+REDIS_HOST = os.environ.get("REDIS_HOST", "10.0.0.36")
+REDIS_PORT = int(os.environ.get("REDIS_PORT", 6379))
+REDIS_PASSWORD = os.environ.get("REDIS_PASSWORD", None)
+
+def get_redis():
+    return redis.Redis(
+        host=REDIS_HOST, 
+        port=REDIS_PORT, 
+        password=REDIS_PASSWORD,
+        decode_responses=True
+    )
+
+def generate_task_id():
+    return f"task_{int(time.time())}_{os.urandom(4).hex()[:8]}"
+
+def add_task(description, task_type="default", priority="medium", created_by="Kimi", message=None, command=None):
+    r = get_redis()
+    
+    task_id = generate_task_id()
+    timestamp = str(int(time.time()))
+    
+    # Build task data
+    task_data = {
+        "id": task_id,
+        "description": description,
+        "type": task_type,
+        "status": "pending",
+        "created_at": timestamp,
+        "created_by": created_by,
+        "priority": priority,
+        "started_at": "",
+        "completed_at": "",
+        "result": ""
+    }
+    
+    # Add type-specific fields
+    if task_type == "notify" and message:
+        task_data["message"] = message
+    elif task_type == "command" and command:
+        task_data["command"] = command
+    
+    # Store task details
+    r.hset(f"task:{task_id}", mapping=task_data)
+    
+    # Add to pending queue
+    # For priority: high=lpush (front), others=rpush (back)
+    if priority == "high":
+        r.lpush("tasks:pending", task_id)
+    else:
+        r.rpush("tasks:pending", task_id)
+    
+    print(f"[ADDED] {task_id}: {description} ({priority}, {task_type})")
+    return task_id
+
+def main():
+    parser = argparse.ArgumentParser(description="Add a task to the queue")
+    parser.add_argument("description", help="Task description")
+    parser.add_argument("--type", choices=["default", "notify", "command"], 
+                        default="default", help="Task type")
+    parser.add_argument("--priority", choices=["high", "medium", "low"], 
+                        default="medium", help="Task priority")
+    parser.add_argument("--by", default="Kimi", help="Who created the task")
+    parser.add_argument("--message", help="Message to send (for notify type)")
+    parser.add_argument("--command", help="Shell command to run (for command type)")
+    
+    args = parser.parse_args()
+    
+    task_id = add_task(
+        args.description, 
+        args.type, 
+        args.priority, 
+        args.by,
+        args.message,
+        args.command
+    )
+    print(f"Task ID: {task_id}")
+
+if __name__ == "__main__":
+    main()

skills/task-queue/scripts/heartbeat_worker.py

@@ -0,0 +1,443 @@
+#!/usr/bin/env python3
+"""
+Heartbeat worker - GPT-powered task execution.
+Sends tasks to Ollama for command generation, executes via SSH.
+"""
+
+import redis
+import json
+import time
+import os
+import sys
+import subprocess
+import requests
+from datetime import datetime
+
+REDIS_HOST = os.environ.get("REDIS_HOST", "127.0.0.1")
+REDIS_PORT = int(os.environ.get("REDIS_PORT", 6379))
+REDIS_PASSWORD = os.environ.get("REDIS_PASSWORD", None)
+OLLAMA_URL = os.environ.get("OLLAMA_URL", "http://127.0.0.1:11434")
+TASK_LLM_MODEL = os.environ.get("TASK_LLM_MODEL", "kimi-k2.5:cloud")
+DEFAULT_TARGET_HOST = os.environ.get("TASK_SSH_HOST", "")
+DEFAULT_SSH_USER = os.environ.get("TASK_SSH_USER", "")
+DEFAULT_SUDO_PASS = os.environ.get("TASK_SUDO_PASS", "")
+
+def get_redis():
+    return redis.Redis(
+        host=REDIS_HOST,
+        port=REDIS_PORT,
+        password=REDIS_PASSWORD,
+        decode_responses=True
+    )
+
+def generate_task_id():
+    return f"task_{int(time.time())}_{os.urandom(4).hex()}"
+
+def check_active_task(r):
+    """Check if there's already an active task."""
+    active = r.lrange("tasks:active", 0, -1)
+    if active:
+        task_id = active[0]
+        task = r.hgetall(f"task:{task_id}")
+        started_at = int(task.get("started_at", 0))
+        elapsed = time.time() - started_at
+        print(f"[BUSY] Task {task_id} active for {elapsed:.0f}s")
+        return True
+    return False
+
+def get_pending_task(r):
+    """Pop a task from pending queue."""
+    task_id = r.rpop("tasks:pending")
+    if task_id:
+        return task_id
+    return None
+
+def clean_json_content(content):
+    """Strip markdown code blocks if present."""
+    cleaned = content.strip()
+    if cleaned.startswith("```json"):
+        cleaned = cleaned[7:]
+    elif cleaned.startswith("```"):
+        cleaned = cleaned[3:]
+    if cleaned.endswith("```"):
+        cleaned = cleaned[:-3]
+    return cleaned.strip()
+
+def ask_gpt_for_commands(task_description, target_host=None, ssh_user=None, sudo_pass=None):
+    """
+    Send task to Ollama/GPT to generate SSH commands.
+    Returns dict with commands, expected results, and explanation.
+    """
+    target_host = target_host or DEFAULT_TARGET_HOST
+    ssh_user = ssh_user or DEFAULT_SSH_USER
+    sudo_pass = sudo_pass if sudo_pass is not None else DEFAULT_SUDO_PASS
+
+    if not target_host or not ssh_user:
+        raise ValueError("TASK_SSH_HOST and TASK_SSH_USER must be set (or passed explicitly)")
+
+    sudo_line = (
+        f"Sudo password: {sudo_pass}"
+        if sudo_pass
+        else "Sudo password: (not provided; avoid sudo unless absolutely necessary)"
+    )
+
+    system_prompt = f"""You have SSH access to {ssh_user}@{target_host}
+{sudo_line}
+
+Your job is to generate shell commands to complete the given task.
+Respond ONLY with valid JSON in this format:
+{{
+  "commands": [
+    "ssh -t {ssh_user}@{target_host} 'sudo apt update'",
+    "ssh -t {ssh_user}@{target_host} 'sudo apt install -y docker.io'"
+  ],
+  "expected_results": [
+    "apt updated successfully",
+    "docker installed and running"
+  ],
+  "explanation": "Updating packages and installing Docker"
+}}
+
+Rules:
+- Commands should use ssh -t (allocates TTY for sudo) to execute on the remote host
+- Use sudo only when needed
+- Keep commands safe and idempotent where possible
+- If task is unclear, ask for clarification in explanation
+
+For Docker-related tasks:
+- Search Docker Hub for official images (docker.io/library/ or verified publishers)
+- Prefer latest stable versions
+- Use official images over community when available
+- Verify image exists before trying to pull
+- Map volumes as specified in the task (e.g., -v /root/html:/usr/share/nginx/html)
+"""
+
+    user_prompt = f"Task: {task_description}\n\nGenerate the commands to complete this task."
+
+    try:
+        response = requests.post(
+            f"{OLLAMA_URL}/api/chat",
+            json={
+                "model": TASK_LLM_MODEL,
+                "messages": [
+                    {"role": "system", "content": system_prompt},
+                    {"role": "user", "content": user_prompt}
+                ],
+                "stream": False,
+                "format": "json"
+            },
+            timeout=120
+        )
+        response.raise_for_status()
+        
+        result = response.json()
+        content = result.get("message", {}).get("content", "{}")
+        
+        # Parse the JSON response
+        try:
+            cleaned = clean_json_content(content)
+            gpt_plan = json.loads(cleaned)
+            return gpt_plan
+        except json.JSONDecodeError:
+            # If GPT didn't return valid JSON, wrap the raw response
+            return {
+                "commands": [],
+                "expected_results": [],
+                "explanation": f"GPT response: {content[:200]}",
+                "parse_error": "GPT did not return valid JSON"
+            }
+            
+    except Exception as e:
+        return {
+            "commands": [],
+            "expected_results": [],
+            "explanation": f"Failed to get commands from GPT: {e}",
+            "error": str(e)
+        }
+
+def execute_ssh_command_with_sudo(command, sudo_pass, timeout=300):
+    """
+    Execute an SSH command with sudo password handling.
+    Uses -t flag for TTY allocation and handles sudo password prompt.
+    """
+    try:
+        # Ensure command has -t flag for TTY
+        if not "-t" in command and command.startswith("ssh "):
+            command = command.replace("ssh ", "ssh -t ", 1)
+        
+        # Use expect-like approach with subprocess
+        # Send password when prompted
+        import pty
+        import select
+        import termios
+        import tty
+        
+        master_fd, slave_fd = pty.openpty()
+        
+        process = subprocess.Popen(
+            command,
+            shell=True,
+            stdin=slave_fd,
+            stdout=slave_fd,
+            stderr=slave_fd,
+            preexec_fn=os.setsid
+        )
+        
+        os.close(slave_fd)
+        
+        output = []
+        password_sent = False
+        start_time = time.time()
+        
+        while process.poll() is None:
+            if time.time() - start_time > timeout:
+                process.kill()
+                return {
+                    "success": False,
+                    "stdout": "".join(output),
+                    "stderr": "Command timed out",
+                    "exit_code": -1
+                }
+            
+            ready, _, _ = select.select([master_fd], [], [], 0.1)
+            if ready:
+                try:
+                    data = os.read(master_fd, 1024).decode()
+                    output.append(data)
+                    
+                    # Check for sudo password prompt
+                    if "password:" in data.lower() or "password for" in data.lower():
+                        if not password_sent:
+                            os.write(master_fd, (sudo_pass + "\n").encode())
+                            password_sent = True
+                            time.sleep(0.5)
+                except OSError:
+                    break
+        
+        os.close(master_fd)
+        
+        stdout = "".join(output)
+        return {
+            "success": process.returncode == 0,
+            "stdout": stdout,
+            "stderr": "" if process.returncode == 0 else stdout,
+            "exit_code": process.returncode
+        }
+        
+    except Exception as e:
+        return {
+            "success": False,
+            "stdout": "",
+            "stderr": str(e),
+            "exit_code": -1
+        }
+
+def execute_ssh_command_simple(command, timeout=300):
+    """
+    Execute an SSH command without sudo (simple version).
+    """
+    try:
+        result = subprocess.run(
+            command,
+            shell=True,
+            capture_output=True,
+            text=True,
+            timeout=timeout
+        )
+        return {
+            "success": result.returncode == 0,
+            "stdout": result.stdout,
+            "stderr": result.stderr,
+            "exit_code": result.returncode
+        }
+    except subprocess.TimeoutExpired:
+        return {
+            "success": False,
+            "stdout": "",
+            "stderr": "Command timed out",
+            "exit_code": -1
+        }
+    except Exception as e:
+        return {
+            "success": False,
+            "stdout": "",
+            "stderr": str(e),
+            "exit_code": -1
+        }
+
+def execute_task_with_gpt(task):
+    """
+    Execute task using GPT to generate commands, then run via SSH.
+    """
+    task_description = task.get("description", "No description")
+    target_host = task.get("target_host", "10.0.0.38")
+    ssh_user = task.get("ssh_user", "n8n")
+    sudo_pass = task.get("sudo_pass", "passw0rd")
+    
+    print(f"[GPT] Generating commands for: {task_description}")
+    
+    # Get commands from GPT
+    gpt_plan = ask_gpt_for_commands(task_description, target_host, ssh_user, sudo_pass)
+    
+    if not gpt_plan.get("commands"):
+        comments = f"GPT failed to generate commands: {gpt_plan.get('explanation', 'Unknown error')}"
+        return {
+            "success": False,
+            "gpt_plan": gpt_plan,
+            "execution_results": [],
+            "comments": comments
+        }
+    
+    print(f"[GPT] Plan: {gpt_plan.get('explanation', 'No explanation')}")
+    print(f"[EXEC] Running {len(gpt_plan['commands'])} commands...")
+    
+    # Execute each command
+    execution_results = []
+    any_failed = False
+    
+    for i, cmd in enumerate(gpt_plan["commands"]):
+        print(f"[CMD {i+1}] {cmd[:80]}...")
+        
+        # Check if command uses sudo
+        if "sudo" in cmd.lower():
+            result = execute_ssh_command_with_sudo(cmd, sudo_pass)
+        else:
+            result = execute_ssh_command_simple(cmd)
+            
+        execution_results.append({
+            "command": cmd,
+            "result": result
+        })
+        
+        if not result["success"]:
+            any_failed = True
+            print(f"[FAIL] Exit code {result['exit_code']}: {result['stderr'][:100]}")
+        else:
+            print(f"[OK] Success")
+    
+    # Build comments field
+    if any_failed:
+        failed_cmds = [r for r in execution_results if not r["result"]["success"]]
+        comments = f"ERRORS ({len(failed_cmds)} failed):\n"
+        for r in failed_cmds:
+            comments += f"- Command: {r['command'][:60]}...\n"
+            comments += f"  Error: {r['result']['stderr'][:200]}\n"
+    else:
+        comments = "OK"
+    
+    return {
+        "success": not any_failed,
+        "gpt_plan": gpt_plan,
+        "execution_results": execution_results,
+        "comments": comments
+    }
+
+def execute_simple_task(task):
+    """
+    Execute simple tasks (notify, command) without GPT.
+    """
+    task_type = task.get("type", "default")
+    description = task.get("description", "No description")
+    sudo_pass = task.get("sudo_pass", "passw0rd")
+    
+    if task_type == "notify":
+        # For now, just log it (messaging handled elsewhere)
+        return {
+            "success": True,
+            "result": f"Notification: {task.get('message', description)}",
+            "comments": "OK"
+        }
+    
+    elif task_type == "command":
+        # Execute shell command directly
+        command = task.get("command", "")
+        if command:
+            if "sudo" in command.lower():
+                result = execute_ssh_command_with_sudo(command, sudo_pass)
+            else:
+                result = execute_ssh_command_simple(command)
+            comments = "OK" if result["success"] else f"Error: {result['stderr'][:500]}"
+            return {
+                "success": result["success"],
+                "result": result["stdout"][:500],
+                "comments": comments
+            }
+        else:
+            return {
+                "success": False,
+                "result": "No command specified",
+                "comments": "ERROR: No command provided"
+            }
+    
+    else:
+        # Default: use GPT
+        return execute_task_with_gpt(task)
+
+def mark_completed(r, task_id, result_data):
+    """Mark task as completed with full result data."""
+    r.hset(f"task:{task_id}", mapping={
+        "status": "completed" if result_data["success"] else "failed",
+        "completed_at": str(int(time.time())),
+        "result": json.dumps(result_data.get("result", "")),
+        "comments": result_data.get("comments", "")
+    })
+    r.lrem("tasks:active", 0, task_id)
+    r.lpush("tasks:completed", task_id)
+    
+    status = "DONE" if result_data["success"] else "FAILED"
+    print(f"[{status}] {task_id}")
+    if result_data.get("comments") and result_data["comments"] != "OK":
+        print(f"[COMMENTS] {result_data['comments'][:200]}")
+
+def mark_failed(r, task_id, error):
+    """Mark task as failed."""
+    r.hset(f"task:{task_id}", mapping={
+        "status": "failed",
+        "completed_at": str(int(time.time())),
+        "result": f"Error: {error}",
+        "comments": f"Worker error: {error}"
+    })
+    r.lrem("tasks:active", 0, task_id)
+    r.lpush("tasks:completed", task_id)
+    print(f"[FAILED] {task_id}: {error}")
+
+def main():
+    r = get_redis()
+    
+    # Check if already busy
+    if check_active_task(r):
+        sys.exit(0)
+    
+    # Get next pending task
+    task_id = get_pending_task(r)
+    if not task_id:
+        print("[IDLE] No pending tasks")
+        sys.exit(0)
+    
+    # Load task details
+    task = r.hgetall(f"task:{task_id}")
+    if not task:
+        print(f"[ERROR] Task {task_id} not found")
+        sys.exit(1)
+    
+    # Move to active
+    r.hset(f"task:{task_id}", mapping={
+        "status": "active",
+        "started_at": str(int(time.time()))
+    })
+    r.lpush("tasks:active", task_id)
+    
+    print(f"[START] {task_id}: {task.get('description', 'No description')}")
+    
+    try:
+        # Execute the task
+        result_data = execute_simple_task(task)
+        mark_completed(r, task_id, result_data)
+        print(f"[WAKE] Task complete - check comments field for status")
+        
+    except Exception as e:
+        mark_failed(r, task_id, str(e))
+        sys.exit(1)
+
+if __name__ == "__main__":
+    main()

skills/task-queue/scripts/list_tasks.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+"""
+List tasks in the queue - pending, active, and recent completed.
+"""
+
+import redis
+import os
+from datetime import datetime
+
+REDIS_HOST = os.environ.get("REDIS_HOST", "10.0.0.36")
+REDIS_PORT = int(os.environ.get("REDIS_PORT", 6379))
+
+def get_redis():
+    return redis.Redis(host=REDIS_HOST, port=REDIS_PORT, decode_responses=True)
+
+def format_time(timestamp):
+    if not timestamp or timestamp == "0":
+        return "-"
+    try:
+        dt = datetime.fromtimestamp(int(timestamp))
+        return dt.strftime("%H:%M:%S")
+    except:
+        return timestamp
+
+def show_tasks(r, key, title, status_filter=None, limit=10):
+    task_ids = r.lrange(key, 0, limit - 1)
+    
+    if not task_ids:
+        print(f"\n{title}: (empty)")
+        return
+    
+    print(f"\n{title}:")
+    print("-" * 80)
+    
+    for task_id in task_ids:
+        task = r.hgetall(f"task:{task_id}")
+        if not task:
+            print(f"  {task_id}: [missing data]")
+            continue
+        
+        status = task.get("status", "?")
+        desc = task.get("description", "no description")[:50]
+        priority = task.get("priority", "medium")
+        created = format_time(task.get("created_at"))
+        
+        if status_filter and status != status_filter:
+            continue
+            
+        print(f"  [{status:10}] {task_id} | {priority:6} | {created} | {desc}")
+
+def main():
+    r = get_redis()
+    
+    print("=" * 80)
+    print("TASK QUEUE STATUS")
+    print("=" * 80)
+    
+    # Show counts
+    pending_count = r.llen("tasks:pending")
+    active_count = r.llen("tasks:active")
+    completed_count = r.llen("tasks:completed")
+    
+    print(f"\nCounts: {pending_count} pending | {active_count} active | {completed_count} completed")
+    
+    # Show pending
+    show_tasks(r, "tasks:pending", "PENDING TASKS", limit=10)
+    
+    # Show active
+    show_tasks(r, "tasks:active", "ACTIVE TASKS")
+    
+    # Show recent completed
+    show_tasks(r, "tasks:completed", "RECENT COMPLETED (last 10)", limit=10)
+    
+    print("\n" + "=" * 80)
+
+if __name__ == "__main__":
+    main()