diff --git a/.gitignore b/.gitignore index 07e5595..1b97eb4 100644 --- a/.gitignore +++ b/.gitignore @@ -68,3 +68,13 @@ Thumbs.db tmp/ temp/ .tmp/ + +# Docker +.dockerignore + +# Backups and deployment +backups/ +backup_*/ + +# Health check files +/tmp/health diff --git a/DOCKER_DEPLOYMENT.md b/DOCKER_DEPLOYMENT.md new file mode 100644 index 0000000..f4f8560 --- /dev/null +++ b/DOCKER_DEPLOYMENT.md @@ -0,0 +1,288 @@ +# Docker Deployment Guide + +This guide covers deploying the trading system for 24/7 operation using Docker. + +## šŸš€ Quick Start + +### 1. Setup Environment +```bash +# Copy environment template +cp .env.production .env + +# Edit with your API keys +nano .env +``` + +### 2. Deploy the System +```bash +# Start the trading system +./deploy.sh start + +# View logs +./deploy.sh logs live + +# Check status +./deploy.sh status +``` + +## šŸ“‹ Deployment Options + +### Paper Trading (Recommended) +```bash +# Safe paper trading mode +./deploy.sh start +``` + +### Live Trading (Advanced) +```bash +# Edit .env file first +TRADING_MODE=live + +# Deploy with extreme caution +./deploy.sh start +``` + +## šŸ³ Docker Architecture + +### Services + +1. **trading-bot**: Main trading application + - Runs 24/7 with auto-restart + - Health monitoring every 60 seconds + - Resource limits for safety + +2. **monitoring**: Optional monitoring service + - Displays system status every 5 minutes + - Shows recent performance and trades + - Lightweight Alpine Linux container + +### Volumes + +- `./logs:/app/logs` - Persistent log storage +- `./config:/app/config:ro` - Read-only configuration + +### Security Features + +- Non-root user (trader:1000) +- Resource limits (512MB RAM, 0.5 CPU) +- Read-only configuration mounting +- Environment-based secrets + +## šŸ›”ļø Safety Features + +### Health Monitoring +- Container health checks every 60 seconds +- Application health checks every 5 minutes +- Automatic restart on failure + +### Risk Management +- Paper trading by default +- Resource constraints +- Graceful shutdown handling +- Comprehensive error logging + +### Backup & Recovery +```bash +# Create backup +./deploy.sh backup + +# View backups +ls backups/ +``` + +## šŸ“Š Monitoring & Logs + +### Real-time Monitoring +```bash +# Follow live logs +./deploy.sh logs live + +# System status +./deploy.sh status + +# Container status +docker compose ps +``` + +### Log Files +- `logs/trading.log` - General system logs +- `logs/trades.log` - Trade execution details +- `logs/risk.log` - Risk management events +- `logs/performance.log` - Performance metrics + +### Health Endpoints +- Health file: `/tmp/health` (inside container) +- Status format: `OK:timestamp:price` + +## šŸ”§ Management Commands + +### Deployment Script (`deploy.sh`) +```bash +./deploy.sh start # Start system +./deploy.sh stop # Stop system +./deploy.sh restart # Restart system +./deploy.sh logs # Show recent logs +./deploy.sh logs live # Follow live logs +./deploy.sh status # System status +./deploy.sh backup # Create backup +./deploy.sh update # Update and restart +``` + +### Docker Compose Commands +```bash +# Manual control +docker compose up -d # Start services +docker compose down # Stop services +docker compose logs -f # Follow logs +docker compose ps # Service status + +# Rebuild containers +docker compose build --no-cache +``` + +## šŸ”„ Updates & Maintenance + +### Updating the System +```bash +# Automated update (with backup) +./deploy.sh update + +# Manual update +docker compose down +git pull # if using git +docker compose build --no-cache +docker compose up -d +``` + +### Configuration Changes +```bash +# Edit configuration +nano .env + +# Restart to apply changes +./deploy.sh restart +``` + +### Log Rotation +Logs are automatically managed by Docker. To manually clean: +```bash +# Clean old logs (be careful!) +docker system prune -f + +# Or manually rotate logs +./deploy.sh backup # Backup first +> logs/trading.log +> logs/trades.log +``` + +## šŸšØ Troubleshooting + +### Common Issues + +1. **Container won't start** + ```bash + # Check logs + docker compose logs trading-bot + + # Verify environment + cat .env + + # Test configuration + docker compose config + ``` + +2. **API connection errors** + ```bash + # Verify API keys in .env + grep -E "ALPACA_(API|SECRET)_KEY" .env + + # Test connection + docker exec trading-system python3 -c "from src.data_handler import DataHandler; print(DataHandler().get_latest_price('AAPL'))" + ``` + +3. **Health check failures** + ```bash + # Check health status + docker inspect trading-system | grep -A 5 Health + + # Manual health check + docker exec trading-system python3 -c "from main import health_check; print(health_check())" + ``` + +4. **Performance issues** + ```bash + # Check resource usage + docker stats trading-system + + # Increase limits in docker compose.yml + nano docker compose.yml + ``` + +### Emergency Procedures + +1. **Immediate stop** + ```bash + ./deploy.sh stop + # or + docker kill trading-system + ``` + +2. **Emergency backup** + ```bash + ./deploy.sh backup + cp -r logs backups/emergency_backup_$(date +%Y%m%d_%H%M%S)/ + ``` + +3. **Reset system** + ```bash + ./deploy.sh stop + docker compose down -v # WARNING: Removes volumes + ./deploy.sh start + ``` + +## šŸŒ Production Considerations + +### Server Requirements +- **Minimum**: 1 CPU, 1GB RAM, 10GB storage +- **Recommended**: 2 CPU, 2GB RAM, 50GB storage +- **OS**: Linux (Ubuntu 20.04+ recommended) + +### Network Requirements +- Stable internet connection +- HTTPS access to Alpaca API +- NTP for accurate timestamps + +### Security Best Practices +1. Use paper trading initially +2. Limit server access (SSH keys only) +3. Regular backups to external storage +4. Monitor logs for anomalies +5. Keep system updated + +### Scaling Considerations +- Multiple symbols: Use separate containers +- High frequency: Increase resource limits +- Redundancy: Deploy across multiple servers + +## šŸ“ž Support + +### Getting Help +1. Check logs: `./deploy.sh logs` +2. Verify configuration: `docker compose config` +3. Test health: `./deploy.sh status` +4. Review documentation + +### Important Notes +- Always test in paper trading first +- Monitor performance regularly +- Keep backups of profitable configurations +- Never disable risk management features + +## šŸ”’ Disclaimer + +This deployment setup is for educational purposes. Trading involves risk of loss. Always: +- Use paper trading for testing +- Understand the risks involved +- Never trade more than you can afford to lose +- Monitor the system regularly +- Keep security best practices diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..13f2c02 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,35 @@ +# Trading System Dockerfile +FROM python:3.11-slim + +# Set working directory +WORKDIR /app + +# Install system dependencies +RUN apt-get update && apt-get install -y \ + gcc \ + g++ \ + && rm -rf /var/lib/apt/lists/* + +# Copy requirements first for better caching +COPY requirements.txt . + +# Install Python dependencies +RUN pip install --no-cache-dir -r requirements.txt + +# Copy source code +COPY . . + +# Create logs directory +RUN mkdir -p logs + +# Create non-root user for security +RUN useradd -m -u 1000 trader && \ + chown -R trader:trader /app +USER trader + +# Health check +HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \ + CMD python3 -c "import sys; sys.path.append('src'); from src.data_handler import DataHandler; DataHandler().get_latest_price('AAPL')" || exit 1 + +# Default command +CMD ["python3", "main.py", "--mode", "paper"] diff --git a/deploy.sh b/deploy.sh new file mode 100755 index 0000000..5cf43bc --- /dev/null +++ b/deploy.sh @@ -0,0 +1,249 @@ +#!/bin/bash + +# Trading System Deployment Script +# Usage: ./deploy.sh [start|stop|restart|logs|status|backup] + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$SCRIPT_DIR" + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Helper functions +log_info() { + echo -e "${BLUE}[INFO]${NC} $1" +} + +log_success() { + echo -e "${GREEN}[SUCCESS]${NC} $1" +} + +log_warning() { + echo -e "${YELLOW}[WARNING]${NC} $1" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +# Check if .env file exists +check_env() { + if [ ! -f .env ]; then + log_warning ".env file not found. Creating from template..." + if [ -f .env.production ]; then + cp .env.production .env + log_info "Please edit .env file with your actual API keys" + log_info "nano .env" + exit 1 + else + log_error "No .env template found!" + exit 1 + fi + fi +} + +# Validate environment +validate_env() { + log_info "Validating environment configuration..." + + # Source the .env file + export $(grep -v '^#' .env | xargs) + + # Check required variables + if [ -z "$ALPACA_API_KEY" ] || [ "$ALPACA_API_KEY" = "your_paper_api_key_here" ]; then + log_error "ALPACA_API_KEY not set in .env file" + exit 1 + fi + + if [ -z "$ALPACA_SECRET_KEY" ] || [ "$ALPACA_SECRET_KEY" = "your_paper_secret_key_here" ]; then + log_error "ALPACA_SECRET_KEY not set in .env file" + exit 1 + fi + + # Warn about live trading + if [ "$TRADING_MODE" = "live" ]; then + log_warning "āš ļø LIVE TRADING MODE ENABLED! āš ļø" + log_warning "This will use real money. Are you sure? (y/N)" + read -r response + if [[ ! "$response" =~ ^[Yy]$ ]]; then + log_info "Switching to paper trading mode for safety" + sed -i.bak 's/TRADING_MODE=live/TRADING_MODE=paper/' .env + fi + fi + + log_success "Environment validation passed" +} + +# Start the trading system +start_system() { + log_info "Starting trading system..." + + check_env + validate_env + + # Create logs directory + mkdir -p logs + + # Build and start containers + docker compose build + docker compose up -d + + log_success "Trading system started!" + log_info "View logs with: ./deploy.sh logs" + log_info "Check status with: ./deploy.sh status" +} + +# Stop the trading system +stop_system() { + log_info "Stopping trading system..." + docker compose down + log_success "Trading system stopped" +} + +# Restart the trading system +restart_system() { + log_info "Restarting trading system..." + stop_system + sleep 2 + start_system +} + +# Show logs +show_logs() { + if [ "$2" = "live" ]; then + log_info "Following live logs (Ctrl+C to exit)..." + docker compose logs -f trading-bot + else + log_info "Recent logs:" + docker compose logs --tail=50 trading-bot + fi +} + +# Show system status +show_status() { + log_info "Trading system status:" + echo + + # Container status + docker compose ps + echo + + # Recent performance + log_info "Recent performance (last 5 entries):" + if [ -f logs/performance.log ]; then + tail -5 logs/performance.log + else + log_warning "No performance data yet" + fi + echo + + # Recent trades + log_info "Recent trades (last 3 entries):" + if [ -f logs/trades.log ]; then + tail -3 logs/trades.log + else + log_warning "No trades yet" + fi + echo + + # Risk events + log_info "Recent risk events:" + if [ -f logs/risk.log ]; then + tail -3 logs/risk.log + else + log_success "No risk events" + fi +} + +# Backup logs and data +backup_data() { + log_info "Creating backup..." + + TIMESTAMP=$(date +"%Y%m%d_%H%M%S") + BACKUP_DIR="backups/backup_$TIMESTAMP" + + mkdir -p "$BACKUP_DIR" + + # Backup logs + if [ -d logs ]; then + cp -r logs "$BACKUP_DIR/" + fi + + # Backup configuration + cp .env "$BACKUP_DIR/.env.backup" 2>/dev/null || true + cp docker-compose.yml "$BACKUP_DIR/" + + log_success "Backup created: $BACKUP_DIR" +} + +# Update system +update_system() { + log_info "Updating trading system..." + + # Stop system + stop_system + + # Backup current state + backup_data + + # Rebuild containers + docker compose build --no-cache + + # Start system + start_system + + log_success "System updated successfully" +} + +# Main script logic +case "${1:-help}" in + start) + start_system + ;; + stop) + stop_system + ;; + restart) + restart_system + ;; + logs) + show_logs "$@" + ;; + status) + show_status + ;; + backup) + backup_data + ;; + update) + update_system + ;; + help|*) + echo "Trading System Deployment Script" + echo + echo "Usage: $0 [command]" + echo + echo "Commands:" + echo " start - Start the trading system" + echo " stop - Stop the trading system" + echo " restart - Restart the trading system" + echo " logs - Show recent logs" + echo " logs live - Follow live logs" + echo " status - Show system status and performance" + echo " backup - Backup logs and configuration" + echo " update - Update and restart system" + echo " help - Show this help message" + echo + echo "Examples:" + echo " $0 start # Start trading" + echo " $0 logs live # Watch live logs" + echo " $0 status # Check performance" + echo + ;; +esac diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..6c661d3 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,89 @@ +services: + trading-bot: + build: . + container_name: trading-system + restart: unless-stopped + environment: + # Alpaca API Configuration + - ALPACA_API_KEY=${ALPACA_API_KEY} + - ALPACA_SECRET_KEY=${ALPACA_SECRET_KEY} + - ALPACA_BASE_URL=${ALPACA_BASE_URL:-https://paper-api.alpaca.markets} + + # Trading Configuration + - STRATEGY_TYPE=${STRATEGY_TYPE:-enhanced} + - SYMBOL=${SYMBOL:-AAPL} + - TRADING_MODE=${TRADING_MODE:-paper} + + # Risk Management + - MAX_POSITION_SIZE=${MAX_POSITION_SIZE:-0.95} + - RISK_PER_TRADE=${RISK_PER_TRADE:-0.02} + - MAX_DRAWDOWN_LIMIT=${MAX_DRAWDOWN_LIMIT:-0.15} + + # Logging + - LOG_LEVEL=${LOG_LEVEL:-INFO} + - TZ=UTC + + volumes: + # Persist logs + - ./logs:/app/logs + # Optional: Mount config for easy updates + - ./config:/app/config:ro + + # Resource limits for safety + deploy: + resources: + limits: + memory: 512M + cpus: '0.5' + reservations: + memory: 256M + cpus: '0.25' + + # Health check + healthcheck: + test: ["CMD", "python3", "-c", "import sys; sys.path.append('src'); from src.data_handler import DataHandler; DataHandler().get_latest_price('AAPL')"] + interval: 60s + timeout: 30s + retries: 3 + start_period: 30s + + # Command override based on mode + command: > + sh -c " + if [ '${TRADING_MODE}' = 'live' ]; then + echo 'āš ļø LIVE TRADING MODE - USE WITH EXTREME CAUTION āš ļø'; + sleep 10; + python3 main.py --mode live --strategy ${STRATEGY_TYPE:-enhanced}; + else + echo 'šŸ“„ PAPER TRADING MODE (Safe)'; + python3 main.py --mode paper --strategy ${STRATEGY_TYPE:-enhanced}; + fi + " + + # Optional: Monitoring service + monitoring: + image: alpine:latest + container_name: trading-monitor + restart: unless-stopped + depends_on: + - trading-bot + volumes: + - ./logs:/logs:ro + command: > + sh -c " + while true; do + echo 'šŸ“Š Trading System Status - $(date)'; + echo 'šŸ“ˆ Latest Performance:'; + tail -5 /logs/performance.log 2>/dev/null || echo 'No performance data yet'; + echo 'šŸ”„ Latest Trades:'; + tail -3 /logs/trades.log 2>/dev/null || echo 'No trades yet'; + echo 'āš ļø Latest Warnings:'; + tail -3 /logs/risk.log 2>/dev/null || echo 'No risk events'; + echo '----------------------------------------'; + sleep 300; # Check every 5 minutes + done + " + +networks: + default: + name: trading-network diff --git a/main.py b/main.py index 70075d7..953c0b8 100644 --- a/main.py +++ b/main.py @@ -2,6 +2,7 @@ """ Main Trading Application Entry point for the automated trading system. +Supports Docker deployment with health monitoring. """ import sys @@ -22,6 +23,7 @@ from config.trading_config import trading_config # Global variables for graceful shutdown running = True trading_engine = None +last_health_check = datetime.now() def signal_handler(signum, frame): """Handle shutdown signals gracefully""" @@ -30,6 +32,26 @@ def signal_handler(signum, frame): logger.info(f"Received signal {signum}, shutting down gracefully...") running = False +def health_check(): + """Perform health check for Docker monitoring""" + global last_health_check + try: + from src.data_handler import DataHandler + data_handler = DataHandler() + # Quick health check - just verify we can get market data + price = data_handler.get_latest_price(trading_config.symbol) + last_health_check = datetime.now() + + # Write health status to file for Docker monitoring + with open('/tmp/health', 'w') as f: + f.write(f"OK:{datetime.now().isoformat()}:{price}") + + return True + except Exception as e: + logger = logging.getLogger(__name__) + logger.error(f"Health check failed: {e}") + return False + def run_backtesting_mode(strategy_type=None): """Run backtesting using historical data""" logger = logging.getLogger(__name__) @@ -228,20 +250,28 @@ def display_backtest_results(results, strategy_name): print("="*60) def run_live_trading_mode(): - """Run live trading mode""" + """Run live trading mode with Docker health monitoring""" global trading_engine, running logger = logging.getLogger(__name__) - logger.info("Starting live trading mode") + logger.info("šŸš€ Starting live trading mode for 24/7 operation") try: # Initialize trading engine trading_engine = TradingEngine(paper_trading=True) + # Track start time for uptime monitoring + start_time = datetime.now() + # Main trading loop cycle_count = 0 last_summary_time = datetime.now() + # Initial health check + health_check() + + logger.info("šŸ“Š Entering main trading loop...") + while running: try: # Run trading cycle @@ -250,7 +280,7 @@ def run_live_trading_mode(): # Log cycle results if cycle_results['success']: - logger.info(f"Cycle {cycle_count} completed successfully") + logger.info(f"āœ… Cycle {cycle_count} completed successfully") if cycle_results['actions_taken']: log_trading_action("trading_cycle", { @@ -260,8 +290,9 @@ def run_live_trading_mode(): 'signals': cycle_results['signals'] }) - # Log performance metrics periodically + # Log performance metrics periodically and health check if datetime.now() - last_summary_time > timedelta(hours=1): + # Performance logging summary = trading_engine.get_trading_summary() if summary['current_position'] > 0 and summary['entry_price']: current_profit = (cycle_results['current_price'] / summary['entry_price'] - 1) * 100 @@ -270,16 +301,25 @@ def run_live_trading_mode(): 'entry_price': summary['entry_price'], 'current_price': cycle_results['current_price'] }) + + # Health check for Docker monitoring + health_check() last_summary_time = datetime.now() + + # Docker deployment status + logger.info(f"šŸ³ Docker Status - Cycle: {cycle_count}, Uptime: {datetime.now() - start_time}") else: logger.error(f"Cycle {cycle_count} failed") # Wait before next cycle (3600 seconds = 1 hour for hourly timeframe) sleep_duration = 3600 # 1 hour - for _ in range(sleep_duration): + # Health check every 5 minutes during sleep + for i in range(sleep_duration): if not running: break + if i % 300 == 0: # Every 5 minutes + health_check() time.sleep(1) except KeyboardInterrupt: @@ -300,12 +340,14 @@ def main(): # Setup command line arguments parser = argparse.ArgumentParser(description='Automated Trading System') - parser.add_argument('--mode', choices=['live', 'backtest'], default='backtest', - help='Trading mode: live or backtest (default: backtest)') + parser.add_argument('--mode', choices=['live', 'paper', 'backtest'], default='backtest', + help='Trading mode: live, paper, or backtest (default: backtest)') parser.add_argument('--strategy', choices=['conservative', 'enhanced'], default=None, help='Strategy type (default: from config)') parser.add_argument('--log-level', choices=['DEBUG', 'INFO', 'WARNING', 'ERROR'], default='INFO', help='Logging level (default: INFO)') + parser.add_argument('--days', type=int, default=180, + help='Days of historical data for backtesting (default: 180)') args = parser.parse_args() @@ -330,7 +372,9 @@ def main(): try: if args.mode == 'backtest': run_backtesting_mode(args.strategy) - elif args.mode == 'live': + elif args.mode in ['live', 'paper']: + # Both live and paper use the same function - paper is safer + logger.info(f"šŸ›”ļø Running in {'PAPER' if args.mode == 'paper' else 'LIVE'} trading mode") run_live_trading_mode() except Exception as e: diff --git a/monitor.py b/monitor.py new file mode 100644 index 0000000..0829e11 --- /dev/null +++ b/monitor.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +""" +Trading System Monitor +Real-time monitoring for Docker deployment. +""" + +import time +import os +from datetime import datetime, timedelta + +def monitor_system(): + """Monitor the trading system performance""" + + print("šŸ” Trading System Monitor - Starting...") + print("=" * 60) + + while True: + try: + current_time = datetime.now() + print(f"\nšŸ“Š System Status - {current_time.strftime('%Y-%m-%d %H:%M:%S')}") + print("-" * 40) + + # Check health file + health_status = "āŒ UNKNOWN" + if os.path.exists('/tmp/health'): + try: + with open('/tmp/health', 'r') as f: + health_data = f.read().strip() + if health_data.startswith('OK:'): + parts = health_data.split(':') + health_time = datetime.fromisoformat(parts[1]) + price = parts[2] if len(parts) > 2 else "N/A" + + # Check if health is recent (within 10 minutes) + if current_time - health_time < timedelta(minutes=10): + health_status = f"āœ… HEALTHY (Price: ${price})" + else: + health_status = f"āš ļø STALE ({(current_time - health_time).seconds//60}m ago)" + else: + health_status = "āŒ ERROR" + except Exception as e: + health_status = f"āŒ ERROR: {e}" + + print(f"Health Status: {health_status}") + + # Check log files + log_files = { + 'trading.log': 'General', + 'trades.log': 'Trades', + 'risk.log': 'Risk Events', + 'performance.log': 'Performance' + } + + for log_file, description in log_files.items(): + log_path = f'/app/logs/{log_file}' + if os.path.exists(log_path): + try: + stat = os.stat(log_path) + size_mb = stat.st_size / (1024 * 1024) + mod_time = datetime.fromtimestamp(stat.st_mtime) + age_minutes = (current_time - mod_time).seconds // 60 + + print(f"{description:12}: {size_mb:.1f}MB, {age_minutes}m ago") + except Exception as e: + print(f"{description:12}: Error - {e}") + else: + print(f"{description:12}: Not found") + + # Show recent performance if available + perf_file = '/app/logs/performance.log' + if os.path.exists(perf_file): + try: + with open(perf_file, 'r') as f: + lines = f.readlines() + if lines: + recent_lines = lines[-3:] # Last 3 entries + print("\nšŸ“ˆ Recent Performance:") + for line in recent_lines: + if line.strip(): + print(f" {line.strip()}") + except Exception as e: + print(f"Performance read error: {e}") + + # Show recent trades if available + trades_file = '/app/logs/trades.log' + if os.path.exists(trades_file): + try: + with open(trades_file, 'r') as f: + lines = f.readlines() + if lines: + recent_trades = [line for line in lines[-5:] if 'BUY' in line or 'SELL' in line] + if recent_trades: + print("\nšŸ’° Recent Trades:") + for trade in recent_trades[-2:]: # Last 2 trades + print(f" {trade.strip()}") + except Exception as e: + print(f"Trades read error: {e}") + + print("\n" + "=" * 60) + + # Wait 5 minutes before next check + time.sleep(300) + + except KeyboardInterrupt: + print("\nšŸ‘‹ Monitor stopped by user") + break + except Exception as e: + print(f"āŒ Monitor error: {e}") + time.sleep(60) # Wait 1 minute on error + +if __name__ == "__main__": + monitor_system()