bakery-ia/test_all_services.py

#!/usr/bin/env python3
"""
Comprehensive test to verify all services can start correctly after database refactoring
"""

import subprocess
import time
import sys
from typing import Dict, List, Tuple

# Service configurations
SERVICES = {
    "infrastructure": {
        "services": ["redis", "rabbitmq", "auth-db", "data-db", "tenant-db", 
                   "forecasting-db", "notification-db", "training-db", "prometheus"],
        "wait_time": 60
    },
    "core": {
        "services": ["auth-service", "data-service"],
        "wait_time": 45,
        "health_checks": [
            ("auth-service", "http://localhost:8001/health"),
            ("data-service", "http://localhost:8002/health"),
        ]
    },
    "business": {
        "services": ["tenant-service", "training-service", "forecasting-service", 
                   "notification-service"],
        "wait_time": 45,
        "health_checks": [
            ("tenant-service", "http://localhost:8003/health"),
            ("training-service", "http://localhost:8004/health"),
            ("forecasting-service", "http://localhost:8005/health"),
            ("notification-service", "http://localhost:8006/health"),
        ]
    },
    "ui": {
        "services": ["gateway", "dashboard"],
        "wait_time": 30,
        "health_checks": [
            ("gateway", "http://localhost:8000/health"),
        ]
    }
}

def run_command(command: str, description: str = None) -> Tuple[int, str]:
    """Run a shell command and return exit code and output"""
    if description:
        print(f"Running: {command}")
    
    try:
        result = subprocess.run(
            command, 
            shell=True, 
            capture_output=True, 
            text=True, 
            timeout=120
        )
        return result.returncode, result.stdout + result.stderr
    except subprocess.TimeoutExpired:
        return -1, f"Command timed out: {command}"

def check_container_status() -> Dict[str, str]:
    """Get status of all containers"""
    exit_code, output = run_command("docker compose ps")
    
    if exit_code != 0:
        return {}
    
    status_dict = {}
    lines = output.strip().split('\n')[1:]  # Skip header
    
    for line in lines:
        if line.strip():
            parts = line.split()
            if len(parts) >= 4:
                name = parts[0].replace('bakery-', '')
                status = ' '.join(parts[3:])
                status_dict[name] = status
    
    return status_dict

def check_health(service: str, url: str) -> bool:
    """Check if a service health endpoint is responding"""
    try:
        # Use curl for health checks instead of requests
        exit_code, _ = run_command(f"curl -f {url}", None)
        return exit_code == 0
    except Exception as e:
        print(f"❌ Health check failed for {service}: {str(e)}")
        return False

def wait_for_healthy_containers(services: List[str], max_wait: int = 60) -> bool:
    """Wait for containers to become healthy"""
    print(f"⏳ Waiting up to {max_wait} seconds for services to become healthy...")
    
    for i in range(max_wait):
        status = check_container_status()
        healthy_count = 0
        
        for service in services:
            service_status = status.get(service, "not found")
            if "healthy" in service_status.lower() or "up" in service_status.lower():
                healthy_count += 1
        
        if healthy_count == len(services):
            print(f"✅ All {len(services)} services are healthy after {i+1} seconds")
            return True
        
        time.sleep(1)
    
    print(f"⚠️  Only {healthy_count}/{len(services)} services became healthy")
    return False

def test_service_group(group_name: str, config: Dict) -> bool:
    """Test a group of services"""
    print(f"\n🧪 Testing {group_name} services...")
    print(f"Services: {', '.join(config['services'])}")
    
    # Start services
    services_str = ' '.join(config['services'])
    exit_code, output = run_command(f"docker compose up -d {services_str}")
    
    if exit_code != 0:
        print(f"❌ Failed to start {group_name} services")
        print(output[-1000:])  # Last 1000 chars of output
        return False
    
    print(f"✅ {group_name.title()} services started")
    
    # Wait for services to be healthy
    if not wait_for_healthy_containers(config['services'], config['wait_time']):
        print(f"⚠️  Some {group_name} services didn't become healthy")
        
        # Show container status
        status = check_container_status()
        for service in config['services']:
            service_status = status.get(service, "not found")
            print(f"  {service}: {service_status}")
        
        # Show logs for failed services
        for service in config['services']:
            service_status = status.get(service, "")
            if "unhealthy" in service_status.lower() or "restarting" in service_status.lower():
                print(f"\n📋 Logs for failed service {service}:")
                _, logs = run_command(f"docker compose logs --tail=10 {service}")
                print(logs[-800:])  # Last 800 chars
        
        return False
    
    # Run health checks if defined
    if "health_checks" in config:
        print(f"🔍 Running health checks for {group_name} services...")
        
        for service, url in config['health_checks']:
            if check_health(service, url):
                print(f"✅ {service} health check passed")
            else:
                print(f"❌ {service} health check failed")
                return False
    
    print(f"🎉 {group_name.title()} services test PASSED!")
    return True

def main():
    """Main test function"""
    print("🔧 COMPREHENSIVE SERVICES STARTUP TEST")
    print("=" * 50)
    
    # Clean up any existing containers
    print("🧹 Cleaning up existing containers...")
    run_command("docker compose down")
    
    all_passed = True
    
    # Test each service group in order
    for group_name, config in SERVICES.items():
        if not test_service_group(group_name, config):
            all_passed = False
            break
    
    # Final status check
    print("\n📊 Final container status:")
    status = check_container_status()
    
    healthy_count = 0
    total_count = 0
    
    for group_config in SERVICES.values():
        for service in group_config['services']:
            total_count += 1
            service_status = status.get(service, "not found")
            status_icon = "✅" if ("healthy" in service_status.lower() or "up" in service_status.lower()) else "❌"
            
            if "healthy" in service_status.lower() or "up" in service_status.lower():
                healthy_count += 1
            
            print(f"{status_icon} {service}: {service_status}")
    
    # Clean up
    print("\n🧹 Cleaning up containers...")
    run_command("docker compose down")
    
    # Final result
    print("=" * 50)
    if all_passed and healthy_count == total_count:
        print(f"🎉 ALL SERVICES TEST PASSED!")
        print(f"✅ {healthy_count}/{total_count} services started successfully")
        print("💡 Your docker-compose setup is working correctly")
        print("🚀 You can now run: docker compose up -d")
        return 0
    else:
        print(f"❌ SERVICES TEST FAILED")
        print(f"⚠️  {healthy_count}/{total_count} services started successfully")
        print("💡 Check the logs above for details")
        return 1

if __name__ == "__main__":
    sys.exit(main())