#!/usr/bin/env python3
"""
API Scraper - REST API client with pagination support
Usage: api_scraper.py https://api.example.com/items --domain "API" --path "Endpoints/Items"
"""

import argparse
import sys
import json
import urllib.request
from pathlib import Path
from datetime import datetime

sys.path.insert(0, str(Path(__file__).parent))
from scrape_to_kb import chunk_text, get_embedding, compute_checksum, store_in_kb

QDRANT_URL = "http://10.0.0.40:6333"
COLLECTION_NAME = "knowledge_base"

class APIScraper:
    def __init__(self, base_url, headers=None, rate_limit=0):
        self.base_url = base_url
        self.headers = headers or {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)',
            'Accept': 'application/json'
        }
        self.rate_limit = rate_limit  # seconds between requests
        
    def fetch(self, url, params=None):
        """Fetch JSON from API"""
        if params:
            import urllib.parse
            query = urllib.parse.urlencode(params)
            url = f"{url}?{query}" if '?' not in url else f"{url}&{query}"
        
        req = urllib.request.Request(url, headers=self.headers)
        
        try:
            with urllib.request.urlopen(req, timeout=30) as response:
                return json.loads(response.read().decode())
        except urllib.error.HTTPError as e:
            print(f"❌ HTTP {e.code}: {e.reason}", file=sys.stderr)
            return None
        except Exception as e:
            print(f"❌ Error: {e}", file=sys.stderr)
            return None
    
    def paginate(self, endpoint, page_param="page", size_param="limit", 
                 size=100, max_pages=None, data_key=None):
        """Fetch paginated results"""
        all_data = []
        page = 1
        
        while True:
            params = {page_param: page, size_param: size}
            url = f"{self.base_url}{endpoint}" if not endpoint.startswith('http') else endpoint
            
            print(f"📄 Fetching page {page}...")
            data = self.fetch(url, params)
            
            if not data:
                break
            
            # Extract items from response
            if data_key:
                items = data.get(data_key, [])
            elif isinstance(data, list):
                items = data
            else:
                # Try common keys
                for key in ['data', 'items', 'results', 'records', 'docs']:
                    if key in data:
                        items = data[key]
                        break
                else:
                    items = [data]  # Single item
            
            if not items:
                break
            
            all_data.extend(items)
            
            # Check for more pages
            if max_pages and page >= max_pages:
                print(f"   Reached max pages ({max_pages})")
                break
            
            # Check if we got less than requested (last page)
            if len(items) < size:
                break
            
            page += 1
            
            if self.rate_limit:
                import time
                time.sleep(self.rate_limit)
        
        return all_data
    
    def format_for_kb(self, items, format_template=None):
        """Format API items as text for knowledge base"""
        if not items:
            return ""
        
        parts = []
        
        for i, item in enumerate(items):
            if format_template:
                # Use custom template
                try:
                    text = format_template.format(**item, index=i+1)
                except KeyError:
                    text = json.dumps(item, indent=2)
            else:
                # Auto-format
                text = self._auto_format(item)
            
            parts.append(text)
        
        return "\n\n---\n\n".join(parts)
    
    def _auto_format(self, item):
        """Auto-format a JSON item as readable text"""
        if isinstance(item, str):
            return item
        
        if not isinstance(item, dict):
            return json.dumps(item, indent=2)
        
        parts = []
        
        # Title/Name first
        for key in ['name', 'title', 'id', 'key']:
            if key in item:
                parts.append(f"# {item[key]}")
                break
        
        # Description/summary
        for key in ['description', 'summary', 'content', 'body', 'text']:
            if key in item:
                parts.append(f"\n{item[key]}")
                break
        
        # Other fields
        skip = ['name', 'title', 'id', 'key', 'description', 'summary', 'content', 'body', 'text']
        for key, value in item.items():
            if key in skip:
                continue
            if value is None:
                continue
            if isinstance(value, (list, dict)):
                value = json.dumps(value, indent=2)
            parts.append(f"\n**{key}:** {value}")
        
        return "\n".join(parts)

def main():
    parser = argparse.ArgumentParser(description="Scrape REST API to knowledge base")
    parser.add_argument("url", help="API endpoint URL")
    parser.add_argument("--domain", required=True, help="Knowledge domain")
    parser.add_argument("--path", required=True, help="Hierarchical path")
    parser.add_argument("--paginate", action="store_true", help="Enable pagination")
    parser.add_argument("--page-param", default="page", help="Page parameter name")
    parser.add_argument("--size-param", default="limit", help="Page size parameter name")
    parser.add_argument("--size", type=int, default=100, help="Items per page")
    parser.add_argument("--max-pages", type=int, help="Max pages to fetch")
    parser.add_argument("--data-key", help="Key containing data array in response")
    parser.add_argument("--header", action='append', nargs=2, metavar=('KEY', 'VALUE'),
                       help="Custom headers (e.g., --header Authorization 'Bearer token')")
    parser.add_argument("--format", help="Python format string for item display")
    parser.add_argument("--category", default="reference")
    parser.add_argument("--content-type", default="api_data")
    parser.add_argument("--subjects", help="Comma-separated subjects")
    parser.add_argument("--title", help="Content title")
    parser.add_argument("--output", "-o", help="Save to JSON file instead of KB")
    parser.add_argument("--rate-limit", type=float, default=0.5, 
                       help="Seconds between requests (default: 0.5)")
    
    args = parser.parse_args()
    
    # Build headers
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)',
        'Accept': 'application/json'
    }
    if args.header:
        for key, value in args.header:
            headers[key] = value
    
    scraper = APIScraper(args.url, headers=headers, rate_limit=args.rate_limit)
    
    print(f"🔌 API: {args.url}")
    print(f"🏷️  Domain: {args.domain}")
    print(f"📂 Path: {args.path}")
    
    # Fetch data
    if args.paginate:
        print("📄 Pagination enabled\n")
        items = scraper.paginate(
            args.url,
            page_param=args.page_param,
            size_param=args.size_param,
            size=args.size,
            max_pages=args.max_pages,
            data_key=args.data_key
        )
    else:
        print("📄 Single request\n")
        data = scraper.fetch(args.url)
        if data_key := args.data_key:
            items = data.get(data_key, []) if data else []
        elif isinstance(data, list):
            items = data
        else:
            items = [data] if data else []
    
    if not items:
        print("❌ No data fetched", file=sys.stderr)
        sys.exit(1)
    
    print(f"✓ Fetched {len(items)} items")
    
    if args.output:
        with open(args.output, 'w') as f:
            json.dump(items, f, indent=2)
        print(f"💾 Saved raw data to {args.output}")
        return
    
    # Format for KB
    text = scraper.format_for_kb(items, args.format)
    
    print(f"📝 Formatted: {len(text)} chars")
    
    if len(text) < 200:
        print("❌ Content too short", file=sys.stderr)
        sys.exit(1)
    
    chunks = chunk_text(text)
    print(f"🧩 Chunks: {len(chunks)}")
    
    subjects = [s.strip() for s in args.subjects.split(",")] if args.subjects else []
    checksum = compute_checksum(text)
    title = args.title or f"API Data from {args.url}"
    
    print("💾 Storing...")
    stored = 0
    for i, chunk in enumerate(chunks):
        chunk_metadata = {
            "domain": args.domain,
            "path": f"{args.path}/chunk-{i+1}",
            "subjects": subjects,
            "category": args.category,
            "content_type": args.content_type,
            "title": f"{title} (part {i+1}/{len(chunks)})",
            "checksum": checksum,
            "source_url": args.url,
            "date_added": datetime.now().strftime("%Y-%m-%d"),
            "chunk_index": i + 1,
            "total_chunks": len(chunks),
            "text_preview": chunk[:200] + "..." if len(chunk) > 200 else chunk,
            "scraper_type": "api_rest",
            "item_count": len(items),
            "api_endpoint": args.url
        }
        
        if store_in_kb(chunk, chunk_metadata):
            stored += 1
            print(f"   ✓ Chunk {i+1}")
    
    print(f"\n🎉 Stored {stored}/{len(chunks)} chunks")
    print(f"   Source: {args.url}")
    print(f"   Items: {len(items)}")

if __name__ == "__main__":
    main()