forked from SpeedyFoxAi/jarvis-memory
Initial commit: Jarvis Memory system
This commit is contained in:
463
install.sh
Executable file
463
install.sh
Executable file
@@ -0,0 +1,463 @@
|
||||
#!/bin/bash
|
||||
# OpenClaw Jarvis-Like Memory System - Installation Script
|
||||
# This script sets up the complete memory system from scratch
|
||||
|
||||
set -e
|
||||
|
||||
# Colors for output
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
RED='\033[0;31m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Configuration
|
||||
WORKSPACE_DIR="${WORKSPACE_DIR:-$HOME/.openclaw/workspace}"
|
||||
USER_ID="${USER_ID:-$(whoami)}"
|
||||
REDIS_HOST="${REDIS_HOST:-127.0.0.1}"
|
||||
REDIS_PORT="${REDIS_PORT:-6379}"
|
||||
QDRANT_URL="${QDRANT_URL:-http://127.0.0.1:6333}"
|
||||
OLLAMA_URL="${OLLAMA_URL:-http://127.0.0.1:11434}"
|
||||
|
||||
# Optional toggles (avoid touching host config during tests)
|
||||
SKIP_CRON="${SKIP_CRON:-0}"
|
||||
SKIP_HEARTBEAT="${SKIP_HEARTBEAT:-0}"
|
||||
SKIP_QDRANT_INIT="${SKIP_QDRANT_INIT:-0}"
|
||||
|
||||
# If Redis/Qdrant/Ollama aren't reachable, attempt to start them via docker compose (recommended).
|
||||
START_DOCKER="${START_DOCKER:-1}"
|
||||
|
||||
# Backup directory
|
||||
BACKUP_DIR="$WORKSPACE_DIR/.backups"
|
||||
TIMESTAMP=$(date +%Y%m%d_%H%M%S)
|
||||
BACKUP_PREFIX="$BACKUP_DIR/install_${TIMESTAMP}"
|
||||
|
||||
echo "═══════════════════════════════════════════════════════════════"
|
||||
echo " OpenClaw Jarvis-Like Memory System - Installer"
|
||||
echo "═══════════════════════════════════════════════════════════════"
|
||||
echo ""
|
||||
echo -e "${BLUE}Backup Location: $BACKUP_DIR${NC}"
|
||||
echo ""
|
||||
|
||||
# Create backup directory
|
||||
mkdir -p "$BACKUP_DIR"
|
||||
|
||||
# Function to backup a file before modifying
|
||||
backup_file() {
|
||||
local file="$1"
|
||||
local backup_name="$2"
|
||||
if [ -f "$file" ]; then
|
||||
cp "$file" "$backup_name"
|
||||
echo -e "${GREEN} ✓ Backed up: $(basename $file) → $(basename $backup_name)${NC}"
|
||||
return 0
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
|
||||
# Helpers
|
||||
have_cmd() { command -v "$1" >/dev/null 2>&1; }
|
||||
|
||||
need_sudo() {
|
||||
if [ "$(id -u)" -eq 0 ]; then
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
run_root() {
|
||||
if need_sudo; then
|
||||
sudo "$@"
|
||||
else
|
||||
"$@"
|
||||
fi
|
||||
}
|
||||
|
||||
install_pkg_debian() {
|
||||
local pkgs=("$@")
|
||||
run_root apt-get update -y
|
||||
run_root apt-get install -y "${pkgs[@]}"
|
||||
}
|
||||
|
||||
install_docker_debian() {
|
||||
# Prefer distro packages for speed/simplicity.
|
||||
install_pkg_debian ca-certificates curl gnupg lsb-release
|
||||
if ! have_cmd docker; then
|
||||
install_pkg_debian docker.io
|
||||
run_root systemctl enable --now docker >/dev/null 2>&1 || true
|
||||
fi
|
||||
# docker compose v2 plugin (preferred)
|
||||
if ! docker compose version >/dev/null 2>&1; then
|
||||
install_pkg_debian docker-compose-plugin || true
|
||||
fi
|
||||
# fallback: docker-compose v1
|
||||
if ! docker compose version >/dev/null 2>&1 && ! have_cmd docker-compose; then
|
||||
install_pkg_debian docker-compose || true
|
||||
fi
|
||||
}
|
||||
|
||||
# Step 1: Check/install system dependencies
|
||||
echo -e "${YELLOW}[1/10] Checking system dependencies...${NC}"
|
||||
|
||||
if ! have_cmd curl; then
|
||||
echo " • Installing curl"
|
||||
if have_cmd apt-get; then
|
||||
install_pkg_debian curl
|
||||
else
|
||||
echo -e "${RED} ✗ Missing curl and no supported package manager detected.${NC}"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# Useful for quick testing (optional)
|
||||
if ! have_cmd redis-cli; then
|
||||
echo " • Installing redis-cli (redis-tools)"
|
||||
if have_cmd apt-get; then
|
||||
install_pkg_debian redis-tools
|
||||
fi
|
||||
fi
|
||||
|
||||
if ! have_cmd python3; then
|
||||
echo " • Installing python3"
|
||||
if have_cmd apt-get; then
|
||||
install_pkg_debian python3 python3-pip python3-venv
|
||||
else
|
||||
echo -e "${RED} ✗ Python 3 not found. Please install Python 3.8+${NC}"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if ! have_cmd pip3; then
|
||||
echo " • Installing pip3"
|
||||
if have_cmd apt-get; then
|
||||
install_pkg_debian python3-pip python3-venv
|
||||
else
|
||||
echo -e "${RED} ✗ pip3 not found. Please install python3-pip${NC}"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# Docker is optional but recommended (used by docker-compose.yml)
|
||||
if ! have_cmd docker; then
|
||||
echo " • Docker not found — installing"
|
||||
if have_cmd apt-get; then
|
||||
install_docker_debian
|
||||
else
|
||||
echo -e "${RED} ✗ Docker not found and no supported package manager detected.${NC}"
|
||||
echo " Install Docker manually, then re-run install.sh"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
echo " ✓ Docker found"
|
||||
fi
|
||||
|
||||
# Docker compose (v2 plugin preferred)
|
||||
if docker compose version >/dev/null 2>&1; then
|
||||
echo " ✓ docker compose (v2) found"
|
||||
elif have_cmd docker-compose; then
|
||||
echo " ✓ docker-compose (v1) found"
|
||||
else
|
||||
echo " • docker compose not found — attempting install"
|
||||
if have_cmd apt-get; then
|
||||
install_docker_debian
|
||||
fi
|
||||
fi
|
||||
|
||||
PYTHON_VERSION=$(python3 --version 2>&1 | awk '{print $2}')
|
||||
echo " ✓ Python $PYTHON_VERSION found"
|
||||
|
||||
# Step 2: Create directory structure + copy blueprint files
|
||||
echo -e "${YELLOW}[2/10] Creating directory structure...${NC}"
|
||||
mkdir -p "$WORKSPACE_DIR"/{skills/{mem-redis,qdrant-memory,task-queue}/scripts,memory,MEMORY_DEF,docs,config}
|
||||
touch "$WORKSPACE_DIR/memory/.gitkeep"
|
||||
|
||||
# Copy scripts/docs from this repo into workspace (portable install)
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
cp -r "$SCRIPT_DIR/skills"/* "$WORKSPACE_DIR/skills/" 2>/dev/null || true
|
||||
cp -r "$SCRIPT_DIR/docs"/* "$WORKSPACE_DIR/docs/" 2>/dev/null || true
|
||||
cp -r "$SCRIPT_DIR/config"/* "$WORKSPACE_DIR/config/" 2>/dev/null || true
|
||||
|
||||
echo " ✓ Directories created and files copied"
|
||||
|
||||
# Step 3: Install Python dependencies (PEP 668-safe)
|
||||
echo -e "${YELLOW}[3/10] Installing Python dependencies...${NC}"
|
||||
REQ_FILE="$(dirname "$0")/requirements.txt"
|
||||
PYTHON_BIN="python3"
|
||||
|
||||
pip_user_install() {
|
||||
if [ -f "$REQ_FILE" ]; then
|
||||
pip3 install --user -r "$REQ_FILE"
|
||||
else
|
||||
pip3 install --user redis qdrant-client requests urllib3
|
||||
fi
|
||||
}
|
||||
|
||||
pip_venv_install() {
|
||||
local venv_dir="$WORKSPACE_DIR/.venv"
|
||||
python3 -m venv "$venv_dir"
|
||||
"$venv_dir/bin/pip" install --upgrade pip setuptools wheel >/dev/null 2>&1 || true
|
||||
if [ -f "$REQ_FILE" ]; then
|
||||
"$venv_dir/bin/pip" install -r "$REQ_FILE"
|
||||
else
|
||||
"$venv_dir/bin/pip" install redis qdrant-client requests urllib3
|
||||
fi
|
||||
PYTHON_BIN="$venv_dir/bin/python"
|
||||
}
|
||||
|
||||
# Try user install first (fast path)
|
||||
if pip_user_install >/dev/null 2>&1; then
|
||||
echo " ✓ Dependencies installed (pip --user)"
|
||||
else
|
||||
echo " ℹ️ pip --user install blocked (PEP 668). Creating venv in $WORKSPACE_DIR/.venv"
|
||||
pip_venv_install
|
||||
echo " ✓ Dependencies installed (venv)"
|
||||
fi
|
||||
|
||||
# Step 4: Test infrastructure connectivity
|
||||
echo -e "${YELLOW}[4/10] Testing infrastructure...${NC}"
|
||||
|
||||
docker_compose_up() {
|
||||
local compose_dir="$(cd "$(dirname "$0")" && pwd)"
|
||||
if docker compose version >/dev/null 2>&1; then
|
||||
run_root docker compose -f "$compose_dir/docker-compose.yml" up -d
|
||||
elif have_cmd docker-compose; then
|
||||
run_root docker-compose -f "$compose_dir/docker-compose.yml" up -d
|
||||
fi
|
||||
}
|
||||
|
||||
redis_ok=0
|
||||
qdrant_ok=0
|
||||
ollama_ok=0
|
||||
|
||||
# Test Redis
|
||||
if "$PYTHON_BIN" -c "import redis; r=redis.Redis(host='$REDIS_HOST', port=$REDIS_PORT); r.ping()" 2>/dev/null; then
|
||||
redis_ok=1
|
||||
fi
|
||||
|
||||
# Test Qdrant
|
||||
if curl -s "$QDRANT_URL/collections" >/dev/null 2>&1; then
|
||||
qdrant_ok=1
|
||||
fi
|
||||
|
||||
# Test Ollama
|
||||
if curl -s "$OLLAMA_URL/api/tags" >/dev/null 2>&1; then
|
||||
ollama_ok=1
|
||||
fi
|
||||
|
||||
if [ "$START_DOCKER" = "1" ] && { [ $redis_ok -eq 0 ] || [ $qdrant_ok -eq 0 ] || [ $ollama_ok -eq 0 ]; }; then
|
||||
echo " ℹ️ Infrastructure not reachable; attempting to start via docker compose"
|
||||
docker_compose_up || true
|
||||
sleep 2
|
||||
if "$PYTHON_BIN" -c "import redis; r=redis.Redis(host='$REDIS_HOST', port=$REDIS_PORT); r.ping()" 2>/dev/null; then redis_ok=1; fi
|
||||
if curl -s "$QDRANT_URL/collections" >/dev/null 2>&1; then qdrant_ok=1; fi
|
||||
if curl -s "$OLLAMA_URL/api/tags" >/dev/null 2>&1; then ollama_ok=1; fi
|
||||
fi
|
||||
|
||||
if [ $redis_ok -eq 1 ]; then
|
||||
echo " ✓ Redis connection OK"
|
||||
else
|
||||
echo -e "${RED} ✗ Redis connection failed ($REDIS_HOST:$REDIS_PORT)${NC}"
|
||||
fi
|
||||
|
||||
if [ $qdrant_ok -eq 1 ]; then
|
||||
echo " ✓ Qdrant connection OK"
|
||||
else
|
||||
echo -e "${RED} ✗ Qdrant connection failed ($QDRANT_URL)${NC}"
|
||||
fi
|
||||
|
||||
if [ $ollama_ok -eq 1 ]; then
|
||||
echo " ✓ Ollama connection OK"
|
||||
else
|
||||
echo -e "${YELLOW} ⚠️ Ollama not reachable ($OLLAMA_URL)${NC}"
|
||||
echo " Embeddings will fail until Ollama is running with snowflake-arctic-embed2."
|
||||
fi
|
||||
|
||||
# Step 5: Backup existing files before modifying
|
||||
echo ""
|
||||
echo -e "${YELLOW}[5/10] Creating backups of existing files...${NC}"
|
||||
BACKUP_COUNT=0
|
||||
|
||||
# Backup existing crontab
|
||||
if crontab -l 2>/dev/null >/dev/null; then
|
||||
crontab -l > "${BACKUP_PREFIX}_crontab.bak.rush" 2>/dev/null
|
||||
echo -e "${GREEN} ✓ Backed up crontab → .backups/install_${TIMESTAMP}_crontab.bak.rush${NC}"
|
||||
((BACKUP_COUNT++))
|
||||
else
|
||||
echo " ℹ️ No existing crontab to backup"
|
||||
fi
|
||||
|
||||
# Backup existing HEARTBEAT.md
|
||||
if backup_file "$WORKSPACE_DIR/HEARTBEAT.md" "${BACKUP_PREFIX}_HEARTBEAT.md.bak.rush"; then
|
||||
((BACKUP_COUNT++))
|
||||
fi
|
||||
|
||||
# Backup existing .memory_env
|
||||
if backup_file "$WORKSPACE_DIR/.memory_env" "${BACKUP_PREFIX}_memory_env.bak.rush"; then
|
||||
((BACKUP_COUNT++))
|
||||
fi
|
||||
|
||||
if [ $BACKUP_COUNT -eq 0 ]; then
|
||||
echo " ℹ️ No existing files to backup (fresh install)"
|
||||
else
|
||||
echo -e "${GREEN} ✓ $BACKUP_COUNT file(s) backed up to $BACKUP_DIR${NC}"
|
||||
fi
|
||||
|
||||
# Step 6: Create environment configuration
|
||||
echo ""
|
||||
echo -e "${YELLOW}[6/10] Creating environment configuration...${NC}"
|
||||
cat > "$WORKSPACE_DIR/.memory_env" <<EOF
|
||||
# Memory System Environment Variables
|
||||
export WORKSPACE_DIR="$WORKSPACE_DIR"
|
||||
export USER_ID="$USER_ID"
|
||||
export REDIS_HOST="$REDIS_HOST"
|
||||
export REDIS_PORT="$REDIS_PORT"
|
||||
export QDRANT_URL="$QDRANT_URL"
|
||||
export OLLAMA_URL="$OLLAMA_URL"
|
||||
export MEMORY_PYTHON="$PYTHON_BIN"
|
||||
export MEMORY_INITIALIZED="true"
|
||||
EOF
|
||||
echo " ✓ Created $WORKSPACE_DIR/.memory_env"
|
||||
|
||||
# Step 7: Initialize Qdrant collections
|
||||
echo -e "${YELLOW}[7/10] Initializing Qdrant collections...${NC}"
|
||||
if [ "$SKIP_QDRANT_INIT" = "1" ]; then
|
||||
echo " ℹ️ SKIP_QDRANT_INIT=1 set; skipping Qdrant collection init"
|
||||
else
|
||||
"$PYTHON_BIN" <<EOF
|
||||
import sys
|
||||
sys.path.insert(0, "$WORKSPACE_DIR/skills/qdrant-memory/scripts")
|
||||
# init_kimi_memories exposes create_collection() in this blueprint
|
||||
from init_kimi_memories import create_collection, collection_exists
|
||||
if not collection_exists():
|
||||
ok = create_collection()
|
||||
if not ok:
|
||||
raise SystemExit(1)
|
||||
print(" ✓ kimi_memories collection ready")
|
||||
EOF
|
||||
fi
|
||||
|
||||
# Step 8: Set up cron jobs
|
||||
echo -e "${YELLOW}[8/10] Setting up cron jobs...${NC}"
|
||||
if [ "$SKIP_CRON" = "1" ]; then
|
||||
echo " ℹ️ SKIP_CRON=1 set; skipping crontab modifications"
|
||||
else
|
||||
CRON_FILE=$(mktemp)
|
||||
crontab -l 2>/dev/null > "$CRON_FILE" || true
|
||||
|
||||
# Add memory backup cron jobs if not present
|
||||
if ! grep -q "cron_backup.py" "$CRON_FILE" 2>/dev/null; then
|
||||
echo "" >> "$CRON_FILE"
|
||||
echo "# Memory System - Daily backup (3:00 AM)" >> "$CRON_FILE"
|
||||
echo "0 3 * * * cd $WORKSPACE_DIR && $PYTHON_BIN skills/mem-redis/scripts/cron_backup.py >> /var/log/memory-backup.log 2>&1 || true" >> "$CRON_FILE"
|
||||
fi
|
||||
|
||||
if ! grep -q "sliding_backup.sh" "$CRON_FILE" 2>/dev/null; then
|
||||
echo "" >> "$CRON_FILE"
|
||||
echo "# Memory System - File backup (3:30 AM)" >> "$CRON_FILE"
|
||||
echo "30 3 * * * $WORKSPACE_DIR/skills/qdrant-memory/scripts/sliding_backup.sh >> /var/log/memory-backup.log 2>&1 || true" >> "$CRON_FILE"
|
||||
fi
|
||||
|
||||
crontab "$CRON_FILE"
|
||||
rm "$CRON_FILE"
|
||||
echo " ✓ Cron jobs configured"
|
||||
fi
|
||||
|
||||
# Step 9: Create HEARTBEAT.md template
|
||||
echo -e "${YELLOW}[9/10] Creating HEARTBEAT.md...${NC}"
|
||||
if [ "$SKIP_HEARTBEAT" = "1" ]; then
|
||||
echo " ℹ️ SKIP_HEARTBEAT=1 set; skipping HEARTBEAT.md write"
|
||||
else
|
||||
cat > "$WORKSPACE_DIR/HEARTBEAT.md" <<'EOF'
|
||||
# HEARTBEAT.md - Memory System Automation
|
||||
|
||||
## Memory Buffer (Every Heartbeat)
|
||||
|
||||
Saves current session context to Redis buffer:
|
||||
|
||||
```bash
|
||||
python3 ~/.openclaw/workspace/skills/mem-redis/scripts/save_mem.py --user-id YOUR_USER_ID
|
||||
```
|
||||
|
||||
## Daily Backup Schedule
|
||||
|
||||
- **3:00 AM**: Redis buffer → Qdrant flush
|
||||
- **3:30 AM**: File-based sliding backup
|
||||
|
||||
## Manual Commands
|
||||
|
||||
| Command | Action |
|
||||
|---------|--------|
|
||||
| `save mem` | Save all context to Redis |
|
||||
| `save q` | Store immediately to Qdrant |
|
||||
| `q <topic>` | Search memories |
|
||||
|
||||
EOF
|
||||
echo " ✓ HEARTBEAT.md created"
|
||||
fi
|
||||
|
||||
# Create backup manifest
|
||||
echo ""
|
||||
echo -e "${YELLOW}Creating backup manifest...${NC}"
|
||||
MANIFEST_FILE="${BACKUP_PREFIX}_MANIFEST.txt"
|
||||
cat > "$MANIFEST_FILE" <<EOF
|
||||
OpenClaw Jarvis Memory - Installation Backup Manifest
|
||||
======================================================
|
||||
Date: $(date)
|
||||
Timestamp: $TIMESTAMP
|
||||
Backup Directory: $BACKUP_DIR
|
||||
|
||||
Files Backed Up:
|
||||
EOF
|
||||
|
||||
# List backed up files
|
||||
for file in "$BACKUP_DIR"/install_${TIMESTAMP}_*.bak.rush; do
|
||||
if [ -f "$file" ]; then
|
||||
basename "$file" >> "$MANIFEST_FILE"
|
||||
fi
|
||||
done
|
||||
|
||||
cat >> "$MANIFEST_FILE" <<EOF
|
||||
|
||||
To Restore Files Manually:
|
||||
==========================
|
||||
|
||||
1. Restore crontab:
|
||||
crontab "$BACKUP_DIR/install_${TIMESTAMP}_crontab.bak.rush"
|
||||
|
||||
2. Restore HEARTBEAT.md:
|
||||
cp "$BACKUP_DIR/install_${TIMESTAMP}_HEARTBEAT.md.bak.rush" "$WORKSPACE_DIR/HEARTBEAT.md"
|
||||
|
||||
3. Restore .memory_env:
|
||||
cp "$BACKUP_DIR/install_${TIMESTAMP}_memory_env.bak.rush" "$WORKSPACE_DIR/.memory_env"
|
||||
|
||||
All backups are stored in: $BACKUP_DIR
|
||||
EOF
|
||||
|
||||
echo -e "${GREEN} ✓ Backup manifest created: ${BACKUP_PREFIX}_MANIFEST.txt${NC}"
|
||||
|
||||
echo ""
|
||||
echo "═══════════════════════════════════════════════════════════════"
|
||||
echo -e "${GREEN} Installation Complete!${NC}"
|
||||
echo "═══════════════════════════════════════════════════════════════"
|
||||
echo ""
|
||||
if [ $BACKUP_COUNT -gt 0 ]; then
|
||||
echo -e "${BLUE}Backups created:${NC} $BACKUP_COUNT file(s) in $BACKUP_DIR"
|
||||
echo " Timestamp: install_${TIMESTAMP}_*.bak.rush"
|
||||
echo ""
|
||||
fi
|
||||
echo "Next steps:"
|
||||
echo " 1. Source the environment: source $WORKSPACE_DIR/.memory_env"
|
||||
echo " 2. Test the system: $PYTHON_BIN $WORKSPACE_DIR/skills/mem-redis/scripts/save_mem.py --user-id $USER_ID"
|
||||
echo " 3. Add to your HEARTBEAT.md to enable automatic saving"
|
||||
echo ""
|
||||
echo "To undo installation:"
|
||||
echo " ./uninstall.sh"
|
||||
echo ""
|
||||
echo "To restore from backup:"
|
||||
echo " See $BACKUP_DIR/install_${TIMESTAMP}_MANIFEST.txt"
|
||||
echo ""
|
||||
echo "Documentation:"
|
||||
echo " - $WORKSPACE_DIR/docs/MEM_DIAGRAM.md"
|
||||
echo " - $WORKSPACE_DIR/skills/mem-redis/SKILL.md"
|
||||
echo " - $WORKSPACE_DIR/skills/qdrant-memory/SKILL.md"
|
||||
echo ""
|
||||
echo "Happy building! 🚀"
|
||||
Reference in New Issue
Block a user