Refactor all main.py
This commit is contained in:
Urtzi Alfaro
320
services/external/app/main.py
vendored
320
services/external/app/main.py
vendored
@@ -3,184 +3,158 @@
|
||||
External Service Main Application
|
||||
"""
|
||||
|
||||
import structlog
|
||||
from contextlib import asynccontextmanager
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from fastapi import FastAPI
|
||||
from app.core.config import settings
|
||||
from app.core.database import init_db, close_db
|
||||
from shared.monitoring import setup_logging, HealthChecker
|
||||
from shared.monitoring.metrics import setup_metrics_early
|
||||
|
||||
# Setup logging first
|
||||
setup_logging("external-service", settings.LOG_LEVEL)
|
||||
logger = structlog.get_logger()
|
||||
|
||||
# Global variables for lifespan access
|
||||
metrics_collector = None
|
||||
health_checker = None
|
||||
|
||||
# Create FastAPI app FIRST
|
||||
app = FastAPI(
|
||||
title="Bakery External Data Service",
|
||||
description="External data collection service for weather, traffic, and events data",
|
||||
version="1.0.0"
|
||||
)
|
||||
|
||||
# Setup metrics BEFORE any middleware and BEFORE lifespan
|
||||
metrics_collector = setup_metrics_early(app, "external-service")
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
"""Application lifespan events"""
|
||||
global health_checker
|
||||
|
||||
# Startup
|
||||
logger.info("Starting External Service...")
|
||||
|
||||
try:
|
||||
# Initialize database
|
||||
await init_db()
|
||||
logger.info("Database initialized")
|
||||
|
||||
# Register custom metrics
|
||||
metrics_collector.register_counter("weather_api_calls_total", "Total weather API calls")
|
||||
metrics_collector.register_counter("weather_api_success_total", "Successful weather API calls")
|
||||
metrics_collector.register_counter("weather_api_failures_total", "Failed weather API calls")
|
||||
|
||||
metrics_collector.register_counter("traffic_api_calls_total", "Total traffic API calls")
|
||||
metrics_collector.register_counter("traffic_api_success_total", "Successful traffic API calls")
|
||||
metrics_collector.register_counter("traffic_api_failures_total", "Failed traffic API calls")
|
||||
|
||||
metrics_collector.register_counter("data_collection_jobs_total", "Data collection jobs")
|
||||
metrics_collector.register_counter("data_records_stored_total", "Data records stored")
|
||||
metrics_collector.register_counter("data_quality_issues_total", "Data quality issues detected")
|
||||
|
||||
metrics_collector.register_histogram("weather_api_duration_seconds", "Weather API call duration")
|
||||
metrics_collector.register_histogram("traffic_api_duration_seconds", "Traffic API call duration")
|
||||
metrics_collector.register_histogram("data_collection_duration_seconds", "Data collection job duration")
|
||||
metrics_collector.register_histogram("data_processing_duration_seconds", "Data processing duration")
|
||||
|
||||
# Setup health checker
|
||||
health_checker = HealthChecker("external-service")
|
||||
|
||||
# Add database health check
|
||||
async def check_database():
|
||||
try:
|
||||
from app.core.database import get_db
|
||||
from sqlalchemy import text
|
||||
async for db in get_db():
|
||||
await db.execute(text("SELECT 1"))
|
||||
return True
|
||||
except Exception as e:
|
||||
return f"Database error: {e}"
|
||||
|
||||
# Add external API health checks
|
||||
async def check_weather_api():
|
||||
try:
|
||||
# Simple connectivity check
|
||||
if settings.AEMET_API_KEY:
|
||||
return True
|
||||
else:
|
||||
return "AEMET API key not configured"
|
||||
except Exception as e:
|
||||
return f"Weather API error: {e}"
|
||||
|
||||
async def check_traffic_api():
|
||||
try:
|
||||
# Simple connectivity check
|
||||
if settings.MADRID_OPENDATA_API_KEY:
|
||||
return True
|
||||
else:
|
||||
return "Madrid Open Data API key not configured"
|
||||
except Exception as e:
|
||||
return f"Traffic API error: {e}"
|
||||
|
||||
health_checker.add_check("database", check_database, timeout=5.0, critical=True)
|
||||
health_checker.add_check("weather_api", check_weather_api, timeout=10.0, critical=False)
|
||||
health_checker.add_check("traffic_api", check_traffic_api, timeout=10.0, critical=False)
|
||||
|
||||
# Store health checker in app state
|
||||
app.state.health_checker = health_checker
|
||||
|
||||
logger.info("External Service started successfully")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to start External Service: {e}")
|
||||
raise
|
||||
|
||||
yield
|
||||
|
||||
# Shutdown
|
||||
logger.info("Shutting down External Service...")
|
||||
await close_db()
|
||||
|
||||
# Set lifespan AFTER metrics setup
|
||||
app.router.lifespan_context = lifespan
|
||||
|
||||
# CORS middleware (added after metrics setup)
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=settings.CORS_ORIGINS,
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
from app.core.database import database_manager
|
||||
from app.services.messaging import setup_messaging, cleanup_messaging
|
||||
from shared.service_base import StandardFastAPIService
|
||||
# Include routers
|
||||
from app.api.weather import router as weather_router
|
||||
from app.api.traffic import router as traffic_router
|
||||
app.include_router(weather_router, prefix="/api/v1", tags=["weather"])
|
||||
app.include_router(traffic_router, prefix="/api/v1", tags=["traffic"])
|
||||
|
||||
# Health check endpoint
|
||||
@app.get("/health")
|
||||
async def health_check():
|
||||
"""Comprehensive health check endpoint"""
|
||||
if health_checker:
|
||||
return await health_checker.check_health()
|
||||
else:
|
||||
return {
|
||||
"service": "external-service",
|
||||
"status": "healthy",
|
||||
"version": "1.0.0"
|
||||
|
||||
class ExternalService(StandardFastAPIService):
|
||||
"""External Data Service with standardized setup"""
|
||||
|
||||
def __init__(self):
|
||||
# Define expected database tables for health checks
|
||||
external_expected_tables = [
|
||||
'weather_data', 'weather_forecasts', 'traffic_data',
|
||||
'traffic_measurement_points', 'traffic_background_jobs'
|
||||
]
|
||||
|
||||
# Define custom API checks
|
||||
async def check_weather_api():
|
||||
"""Check weather API configuration"""
|
||||
try:
|
||||
return bool(settings.AEMET_API_KEY)
|
||||
except Exception as e:
|
||||
self.logger.error("Weather API check failed", error=str(e))
|
||||
return False
|
||||
|
||||
async def check_traffic_api():
|
||||
"""Check traffic API configuration"""
|
||||
try:
|
||||
return bool(settings.MADRID_OPENDATA_API_KEY)
|
||||
except Exception as e:
|
||||
self.logger.error("Traffic API check failed", error=str(e))
|
||||
return False
|
||||
|
||||
# Define custom metrics for external service
|
||||
external_custom_metrics = {
|
||||
"weather_api_calls_total": {
|
||||
"type": "counter",
|
||||
"description": "Total weather API calls"
|
||||
},
|
||||
"weather_api_success_total": {
|
||||
"type": "counter",
|
||||
"description": "Successful weather API calls"
|
||||
},
|
||||
"weather_api_failures_total": {
|
||||
"type": "counter",
|
||||
"description": "Failed weather API calls"
|
||||
},
|
||||
"traffic_api_calls_total": {
|
||||
"type": "counter",
|
||||
"description": "Total traffic API calls"
|
||||
},
|
||||
"traffic_api_success_total": {
|
||||
"type": "counter",
|
||||
"description": "Successful traffic API calls"
|
||||
},
|
||||
"traffic_api_failures_total": {
|
||||
"type": "counter",
|
||||
"description": "Failed traffic API calls"
|
||||
},
|
||||
"data_collection_jobs_total": {
|
||||
"type": "counter",
|
||||
"description": "Data collection jobs"
|
||||
},
|
||||
"data_records_stored_total": {
|
||||
"type": "counter",
|
||||
"description": "Data records stored"
|
||||
},
|
||||
"data_quality_issues_total": {
|
||||
"type": "counter",
|
||||
"description": "Data quality issues detected"
|
||||
},
|
||||
"weather_api_duration_seconds": {
|
||||
"type": "histogram",
|
||||
"description": "Weather API call duration"
|
||||
},
|
||||
"traffic_api_duration_seconds": {
|
||||
"type": "histogram",
|
||||
"description": "Traffic API call duration"
|
||||
},
|
||||
"data_collection_duration_seconds": {
|
||||
"type": "histogram",
|
||||
"description": "Data collection job duration"
|
||||
},
|
||||
"data_processing_duration_seconds": {
|
||||
"type": "histogram",
|
||||
"description": "Data processing duration"
|
||||
}
|
||||
}
|
||||
|
||||
# Root endpoint
|
||||
@app.get("/")
|
||||
async def root():
|
||||
"""Root endpoint"""
|
||||
return {
|
||||
"service": "External Data Service",
|
||||
"version": "1.0.0",
|
||||
"status": "running",
|
||||
"endpoints": {
|
||||
"health": "/health",
|
||||
"docs": "/docs",
|
||||
"weather": "/api/v1/weather",
|
||||
"traffic": "/api/v1/traffic",
|
||||
"jobs": "/api/v1/jobs"
|
||||
},
|
||||
"data_sources": {
|
||||
"weather": "AEMET (Spanish Weather Service)",
|
||||
"traffic": "Madrid Open Data Portal",
|
||||
"coverage": "Madrid, Spain"
|
||||
}
|
||||
}
|
||||
super().__init__(
|
||||
service_name="external-service",
|
||||
app_name="Bakery External Data Service",
|
||||
description="External data collection service for weather, traffic, and events data",
|
||||
version="1.0.0",
|
||||
log_level=settings.LOG_LEVEL,
|
||||
cors_origins=settings.CORS_ORIGINS,
|
||||
api_prefix="/api/v1",
|
||||
database_manager=database_manager,
|
||||
expected_tables=external_expected_tables,
|
||||
custom_health_checks={
|
||||
"weather_api": check_weather_api,
|
||||
"traffic_api": check_traffic_api
|
||||
},
|
||||
custom_metrics=external_custom_metrics,
|
||||
enable_messaging=True
|
||||
)
|
||||
|
||||
# Exception handlers
|
||||
@app.exception_handler(Exception)
|
||||
async def global_exception_handler(request: Request, exc: Exception):
|
||||
"""Global exception handler with metrics"""
|
||||
logger.error(f"Unhandled exception: {exc}", exc_info=True)
|
||||
|
||||
# Record error metric if available
|
||||
if metrics_collector:
|
||||
metrics_collector.increment_counter("errors_total", labels={"type": "unhandled"})
|
||||
|
||||
return JSONResponse(
|
||||
status_code=500,
|
||||
content={"detail": "Internal server error"}
|
||||
)
|
||||
async def _setup_messaging(self):
|
||||
"""Setup messaging for external service"""
|
||||
await setup_messaging()
|
||||
self.logger.info("External service messaging initialized")
|
||||
|
||||
async def _cleanup_messaging(self):
|
||||
"""Cleanup messaging for external service"""
|
||||
await cleanup_messaging()
|
||||
|
||||
async def on_startup(self, app: FastAPI):
|
||||
"""Custom startup logic for external service"""
|
||||
pass
|
||||
|
||||
async def on_shutdown(self, app: FastAPI):
|
||||
"""Custom shutdown logic for external service"""
|
||||
# Database cleanup is handled by the base class
|
||||
pass
|
||||
|
||||
def get_service_features(self):
|
||||
"""Return external-specific features"""
|
||||
return [
|
||||
"weather_data_collection",
|
||||
"traffic_data_collection",
|
||||
"aemet_integration",
|
||||
"madrid_opendata_integration",
|
||||
"data_quality_monitoring",
|
||||
"scheduled_collection_jobs",
|
||||
"external_api_monitoring"
|
||||
]
|
||||
|
||||
|
||||
|
||||
# Create service instance
|
||||
service = ExternalService()
|
||||
|
||||
# Create FastAPI app with standardized setup
|
||||
app = service.create_app()
|
||||
|
||||
# Setup standard endpoints
|
||||
service.setup_standard_endpoints()
|
||||
|
||||
# Include routers
|
||||
service.add_router(weather_router, tags=["weather"])
|
||||
service.add_router(traffic_router, tags=["traffic"])
|
||||
Reference in New Issue
Block a user