docker-compose.yml

version: '3.8'

services:
  # Redis for rate limiting and caching (optional but recommended)
  redis:
    image: redis:7-alpine
    container_name: scraper-redis
    ports:
      - "6379:6379"
    volumes:
      - redis-data:/data
    command: redis-server --appendonly yes --maxmemory 256mb --maxmemory-policy allkeys-lru
    healthcheck:
      test: ["CMD", "redis-cli", "ping"]
      interval: 5s
      timeout: 3s
      retries: 3
      start_period: 5s
    networks:
      - scraper-network
    restart: unless-stopped
    deploy:
      restart_policy:
        condition: on-failure
        delay: 1s
        max_attempts: 5
        window: 30s

  # Web Scraping API - Main Service
  api:
    build:
      context: .
      dockerfile: Dockerfile
    container_name: scraper-api
    ports:
      - "8000:8000"
    environment:
      # API Configuration
      - API_HOST=0.0.0.0
      - API_PORT=8000
      - API_WORKERS=4
      - DEBUG=False
      
      # Rate Limiting - Optimized for 50+ req/min
      - MAX_SEARCH_REQUESTS_PER_MINUTE=120
      - MAX_WEBSITE_REQUESTS_PER_MINUTE=60
      - MAX_CONCURRENT_REQUESTS=100
      - CONNECTION_POOL_SIZE=100
      
      # Redis Configuration (optional)
      - USE_REDIS=true
      - REDIS_HOST=redis
      - REDIS_PORT=6379
      - REDIS_DB=0
      
      # Proxy Configuration - Auto-fetch enabled
      - USE_PROXY=true
      - PROXY_ROTATION=true
      - PROXY_TIMEOUT=10
      - AUTO_FETCH_PROXIES=true
      - PROXY_FETCH_INTERVAL=300
      - MIN_WORKING_PROXIES=10
      # Add custom proxies (comma-separated): CUSTOM_PROXIES=http://proxy1:8080,socks5://proxy2:1080
      - CUSTOM_PROXIES=
      
      # Captcha Solving
      - ENABLE_CAPTCHA_SOLVER=true
      - CAPTCHA_TIMEOUT=30
      - CAPTCHA_MAX_RETRIES=3
      
      # Request Configuration - Optimized for speed
      - REQUEST_TIMEOUT=15
      - MAX_RETRIES=2
      - RETRY_DELAY=0.5
      
      # Scraping Configuration
      - JAVASCRIPT_RENDERING=true
      - BROWSER_HEADLESS=true
      - PAGE_LOAD_TIMEOUT=15
      
      # Search Engine Fallback
      - ENABLE_FALLBACK=true
      - FALLBACK_ORDER=google,duckduckgo,bing,yahoo
      
      # Fingerprint Randomization
      - RANDOMIZE_FINGERPRINT=true
      
      # Logging
      - LOG_LEVEL=INFO
      - LOG_FILE=logs/scraper.log
    
    volumes:
      - ./config:/app/config
      - ./logs:/app/logs
    
    depends_on:
      redis:
        condition: service_healthy
    
    networks:
      - scraper-network
    
    restart: unless-stopped
    
    # Healthcheck for faster recovery
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
      interval: 10s
      timeout: 5s
      retries: 3
      start_period: 30s
    
    # Resource limits for production with faster restart
    deploy:
      restart_policy:
        condition: on-failure
        delay: 1s
        max_attempts: 10
        window: 60s
      resources:
        limits:
          cpus: '4'
          memory: 4G
        reservations:
          cpus: '2'
          memory: 2G

  # Development mode (with hot-reload)
  api-dev:
    build:
      context: .
      dockerfile: Dockerfile
    container_name: scraper-api-dev
    ports:
      - "8001:8000"
    environment:
      - API_HOST=0.0.0.0
      - API_PORT=8000
      - DEBUG=True
      - LOG_LEVEL=DEBUG
      - USE_REDIS=false
      - USE_PROXY=true
      - AUTO_FETCH_PROXIES=true
      - ENABLE_CAPTCHA_SOLVER=true
      - ENABLE_FALLBACK=true
    volumes:
      - ./app:/app/app
      - ./config:/app/config
      - ./logs:/app/logs
    networks:
      - scraper-network
    profiles:
      - dev
    # Faster healthcheck for development
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
      interval: 5s
      timeout: 3s
      retries: 3
      start_period: 15s
    deploy:
      restart_policy:
        condition: on-failure
        delay: 1s
        max_attempts: 5
        window: 30s

  # Nginx reverse proxy (optional, for production)
  nginx:
    image: nginx:alpine
    container_name: scraper-nginx
    ports:
      - "80:80"
      - "443:443"
    volumes:
      - ./nginx.conf:/etc/nginx/nginx.conf:ro
      - ./ssl:/etc/nginx/ssl:ro
    depends_on:
      - api
    networks:
      - scraper-network
    restart: unless-stopped
    profiles:
      - production

volumes:
  redis-data:
    driver: local

networks:
  scraper-network:
    driver: bridge