REFACTOR data service

services/external/app/main.py

@@ -0,0 +1,186 @@
+# services/external/app/main.py
+"""
+External Service Main Application
+"""
+
+import structlog
+from contextlib import asynccontextmanager
+from fastapi import FastAPI, Request
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import JSONResponse
+
+from app.core.config import settings
+from app.core.database import init_db, close_db
+from shared.monitoring import setup_logging, HealthChecker
+from shared.monitoring.metrics import setup_metrics_early
+
+# Setup logging first
+setup_logging("external-service", settings.LOG_LEVEL)
+logger = structlog.get_logger()
+
+# Global variables for lifespan access
+metrics_collector = None
+health_checker = None
+
+# Create FastAPI app FIRST
+app = FastAPI(
+    title="Bakery External Data Service",
+    description="External data collection service for weather, traffic, and events data",
+    version="1.0.0"
+)
+
+# Setup metrics BEFORE any middleware and BEFORE lifespan
+metrics_collector = setup_metrics_early(app, "external-service")
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    """Application lifespan events"""
+    global health_checker
+    
+    # Startup
+    logger.info("Starting External Service...")
+    
+    try:
+        # Initialize database
+        await init_db()
+        logger.info("Database initialized")
+        
+        # Register custom metrics
+        metrics_collector.register_counter("weather_api_calls_total", "Total weather API calls")
+        metrics_collector.register_counter("weather_api_success_total", "Successful weather API calls")
+        metrics_collector.register_counter("weather_api_failures_total", "Failed weather API calls")
+        
+        metrics_collector.register_counter("traffic_api_calls_total", "Total traffic API calls")
+        metrics_collector.register_counter("traffic_api_success_total", "Successful traffic API calls")
+        metrics_collector.register_counter("traffic_api_failures_total", "Failed traffic API calls")
+        
+        metrics_collector.register_counter("data_collection_jobs_total", "Data collection jobs")
+        metrics_collector.register_counter("data_records_stored_total", "Data records stored")
+        metrics_collector.register_counter("data_quality_issues_total", "Data quality issues detected")
+        
+        metrics_collector.register_histogram("weather_api_duration_seconds", "Weather API call duration")
+        metrics_collector.register_histogram("traffic_api_duration_seconds", "Traffic API call duration")
+        metrics_collector.register_histogram("data_collection_duration_seconds", "Data collection job duration")
+        metrics_collector.register_histogram("data_processing_duration_seconds", "Data processing duration")
+        
+        # Setup health checker
+        health_checker = HealthChecker("external-service")
+        
+        # Add database health check
+        async def check_database():
+            try:
+                from app.core.database import get_db
+                from sqlalchemy import text
+                async for db in get_db():
+                    await db.execute(text("SELECT 1"))
+                    return True
+            except Exception as e:
+                return f"Database error: {e}"
+        
+        # Add external API health checks
+        async def check_weather_api():
+            try:
+                # Simple connectivity check
+                if settings.AEMET_API_KEY:
+                    return True
+                else:
+                    return "AEMET API key not configured"
+            except Exception as e:
+                return f"Weather API error: {e}"
+        
+        async def check_traffic_api():
+            try:
+                # Simple connectivity check
+                if settings.MADRID_OPENDATA_API_KEY:
+                    return True
+                else:
+                    return "Madrid Open Data API key not configured"
+            except Exception as e:
+                return f"Traffic API error: {e}"
+        
+        health_checker.add_check("database", check_database, timeout=5.0, critical=True)
+        health_checker.add_check("weather_api", check_weather_api, timeout=10.0, critical=False)
+        health_checker.add_check("traffic_api", check_traffic_api, timeout=10.0, critical=False)
+        
+        # Store health checker in app state
+        app.state.health_checker = health_checker
+        
+        logger.info("External Service started successfully")
+        
+    except Exception as e:
+        logger.error(f"Failed to start External Service: {e}")
+        raise
+    
+    yield
+    
+    # Shutdown
+    logger.info("Shutting down External Service...")
+    await close_db()
+
+# Set lifespan AFTER metrics setup
+app.router.lifespan_context = lifespan
+
+# CORS middleware (added after metrics setup)
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=settings.CORS_ORIGINS,
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+# Include routers
+from app.api.weather import router as weather_router
+from app.api.traffic import router as traffic_router
+app.include_router(weather_router, prefix="/api/v1", tags=["weather"])
+app.include_router(traffic_router, prefix="/api/v1", tags=["traffic"])
+
+# Health check endpoint
+@app.get("/health")
+async def health_check():
+    """Comprehensive health check endpoint"""
+    if health_checker:
+        return await health_checker.check_health()
+    else:
+        return {
+            "service": "external-service",
+            "status": "healthy",
+            "version": "1.0.0"
+        }
+
+# Root endpoint
+@app.get("/")
+async def root():
+    """Root endpoint"""
+    return {
+        "service": "External Data Service",
+        "version": "1.0.0",
+        "status": "running",
+        "endpoints": {
+            "health": "/health",
+            "docs": "/docs",
+            "weather": "/api/v1/weather",
+            "traffic": "/api/v1/traffic",
+            "jobs": "/api/v1/jobs"
+        },
+        "data_sources": {
+            "weather": "AEMET (Spanish Weather Service)",
+            "traffic": "Madrid Open Data Portal",
+            "coverage": "Madrid, Spain"
+        }
+    }
+
+# Exception handlers
+@app.exception_handler(Exception)
+async def global_exception_handler(request: Request, exc: Exception):
+    """Global exception handler with metrics"""
+    logger.error(f"Unhandled exception: {exc}", exc_info=True)
+    
+    # Record error metric if available
+    if metrics_collector:
+        metrics_collector.increment_counter("errors_total", labels={"type": "unhandled"})
+    
+    return JSONResponse(
+        status_code=500,
+        content={"detail": "Internal server error"}
+    )