282 lines
8.4 KiB
Python
282 lines
8.4 KiB
Python
# services/training/app/main.py
|
|
"""
|
|
Training Service Main Application
|
|
Enhanced with proper error handling, monitoring, and lifecycle management
|
|
"""
|
|
|
|
import structlog
|
|
import asyncio
|
|
from contextlib import asynccontextmanager
|
|
from fastapi import FastAPI, Request
|
|
from fastapi.middleware.cors import CORSMiddleware
|
|
from fastapi.middleware.trustedhost import TrustedHostMiddleware
|
|
from fastapi.responses import JSONResponse
|
|
import uvicorn
|
|
|
|
from app.core.config import settings
|
|
from app.core.database import database_manager, get_db_health
|
|
from app.api import training, models
|
|
from app.services.messaging import setup_messaging, cleanup_messaging
|
|
from shared.monitoring.logging import setup_logging
|
|
from shared.monitoring.metrics import MetricsCollector
|
|
from shared.auth.decorators import require_auth
|
|
|
|
# Setup structured logging
|
|
setup_logging("training-service", settings.LOG_LEVEL)
|
|
logger = structlog.get_logger()
|
|
|
|
# Initialize metrics collector
|
|
metrics_collector = MetricsCollector("training-service")
|
|
|
|
@asynccontextmanager
|
|
async def lifespan(app: FastAPI):
|
|
"""
|
|
Application lifespan manager for startup and shutdown events
|
|
"""
|
|
# Startup
|
|
logger.info("Starting Training Service", version="1.0.0")
|
|
|
|
try:
|
|
# Initialize database
|
|
logger.info("Initializing database connection")
|
|
await database_manager.create_tables()
|
|
logger.info("Database initialized successfully")
|
|
|
|
# Initialize messaging
|
|
logger.info("Setting up messaging")
|
|
await setup_messaging()
|
|
logger.info("Messaging setup completed")
|
|
|
|
# Start metrics server
|
|
logger.info("Starting metrics server")
|
|
metrics_collector.start_metrics_server(8080)
|
|
logger.info("Metrics server started on port 8080")
|
|
|
|
# Mark service as ready
|
|
app.state.ready = True
|
|
logger.info("Training Service startup completed successfully")
|
|
|
|
yield
|
|
|
|
except Exception as e:
|
|
logger.error("Failed to start Training Service", error=str(e))
|
|
app.state.ready = False
|
|
raise
|
|
|
|
# Shutdown
|
|
logger.info("Shutting down Training Service")
|
|
|
|
try:
|
|
# Cleanup messaging
|
|
logger.info("Cleaning up messaging")
|
|
await cleanup_messaging()
|
|
|
|
# Close database connections
|
|
logger.info("Closing database connections")
|
|
await database_manager.close_connections()
|
|
|
|
logger.info("Training Service shutdown completed")
|
|
|
|
except Exception as e:
|
|
logger.error("Error during shutdown", error=str(e))
|
|
|
|
# Create FastAPI app with lifespan
|
|
app = FastAPI(
|
|
title="Training Service",
|
|
description="ML model training service for bakery demand forecasting",
|
|
version="1.0.0",
|
|
docs_url="/docs" if settings.DEBUG else None,
|
|
redoc_url="/redoc" if settings.DEBUG else None,
|
|
lifespan=lifespan
|
|
)
|
|
|
|
# Initialize app state
|
|
app.state.ready = False
|
|
|
|
# Security middleware
|
|
if not settings.DEBUG:
|
|
app.add_middleware(
|
|
TrustedHostMiddleware,
|
|
allowed_hosts=["localhost", "127.0.0.1", "training-service", "*.bakery-forecast.local"]
|
|
)
|
|
|
|
# CORS middleware
|
|
app.add_middleware(
|
|
CORSMiddleware,
|
|
allow_origins=["*"] if settings.DEBUG else [
|
|
"http://localhost:3000",
|
|
"http://localhost:8000",
|
|
"https://dashboard.bakery-forecast.es"
|
|
],
|
|
allow_credentials=True,
|
|
allow_methods=["GET", "POST", "PUT", "DELETE", "OPTIONS"],
|
|
allow_headers=["*"],
|
|
)
|
|
|
|
# Request logging middleware
|
|
@app.middleware("http")
|
|
async def log_requests(request: Request, call_next):
|
|
"""Log all incoming requests with timing"""
|
|
start_time = asyncio.get_event_loop().time()
|
|
|
|
# Log request
|
|
logger.info(
|
|
"Request started",
|
|
method=request.method,
|
|
path=request.url.path,
|
|
client_ip=request.client.host if request.client else "unknown"
|
|
)
|
|
|
|
# Process request
|
|
try:
|
|
response = await call_next(request)
|
|
|
|
# Calculate duration
|
|
duration = asyncio.get_event_loop().time() - start_time
|
|
|
|
# Log response
|
|
logger.info(
|
|
"Request completed",
|
|
method=request.method,
|
|
path=request.url.path,
|
|
status_code=response.status_code,
|
|
duration_ms=round(duration * 1000, 2)
|
|
)
|
|
|
|
# Update metrics
|
|
metrics_collector.record_request(
|
|
method=request.method,
|
|
endpoint=request.url.path,
|
|
status_code=response.status_code,
|
|
duration=duration
|
|
)
|
|
|
|
return response
|
|
|
|
except Exception as e:
|
|
duration = asyncio.get_event_loop().time() - start_time
|
|
|
|
logger.error(
|
|
"Request failed",
|
|
method=request.method,
|
|
path=request.url.path,
|
|
error=str(e),
|
|
duration_ms=round(duration * 1000, 2)
|
|
)
|
|
|
|
metrics_collector.increment_counter("http_requests_failed_total")
|
|
raise
|
|
|
|
# Exception handlers
|
|
@app.exception_handler(Exception)
|
|
async def global_exception_handler(request: Request, exc: Exception):
|
|
"""Global exception handler for unhandled errors"""
|
|
logger.error(
|
|
"Unhandled exception",
|
|
path=request.url.path,
|
|
method=request.method,
|
|
error=str(exc),
|
|
exc_info=True
|
|
)
|
|
|
|
metrics_collector.increment_counter("unhandled_exceptions_total")
|
|
|
|
return JSONResponse(
|
|
status_code=500,
|
|
content={
|
|
"detail": "Internal server error",
|
|
"error_id": structlog.get_logger().new().info("Error logged", error=str(exc))
|
|
}
|
|
)
|
|
|
|
# Include API routers
|
|
app.include_router(
|
|
training.router,
|
|
prefix="/training",
|
|
tags=["training"],
|
|
dependencies=[require_auth] if not settings.DEBUG else []
|
|
)
|
|
|
|
app.include_router(
|
|
models.router,
|
|
prefix="/models",
|
|
tags=["models"],
|
|
dependencies=[require_auth] if not settings.DEBUG else []
|
|
)
|
|
|
|
# Health check endpoints
|
|
@app.get("/health")
|
|
async def health_check():
|
|
"""Basic health check endpoint"""
|
|
return {
|
|
"status": "healthy" if app.state.ready else "starting",
|
|
"service": "training-service",
|
|
"version": "1.0.0",
|
|
"timestamp": structlog.get_logger().new().info("Health check")
|
|
}
|
|
|
|
@app.get("/health/ready")
|
|
async def readiness_check():
|
|
"""Kubernetes readiness probe"""
|
|
if not app.state.ready:
|
|
return JSONResponse(
|
|
status_code=503,
|
|
content={"status": "not_ready", "message": "Service is starting up"}
|
|
)
|
|
|
|
return {"status": "ready", "service": "training-service"}
|
|
|
|
@app.get("/health/live")
|
|
async def liveness_check():
|
|
"""Kubernetes liveness probe"""
|
|
# Check database connectivity
|
|
try:
|
|
db_healthy = await get_db_health()
|
|
if not db_healthy:
|
|
return JSONResponse(
|
|
status_code=503,
|
|
content={"status": "unhealthy", "reason": "database_unavailable"}
|
|
)
|
|
except Exception as e:
|
|
logger.error("Database health check failed", error=str(e))
|
|
return JSONResponse(
|
|
status_code=503,
|
|
content={"status": "unhealthy", "reason": "database_error"}
|
|
)
|
|
|
|
return {"status": "alive", "service": "training-service"}
|
|
|
|
@app.get("/metrics")
|
|
async def get_metrics():
|
|
"""Expose service metrics"""
|
|
return {
|
|
"training_jobs_active": metrics_collector.get_gauge("training_jobs_active", 0),
|
|
"training_jobs_completed": metrics_collector.get_counter("training_jobs_completed", 0),
|
|
"training_jobs_failed": metrics_collector.get_counter("training_jobs_failed", 0),
|
|
"models_trained_total": metrics_collector.get_counter("models_trained_total", 0),
|
|
"uptime_seconds": metrics_collector.get_gauge("uptime_seconds", 0)
|
|
}
|
|
|
|
@app.get("/")
|
|
async def root():
|
|
"""Root endpoint with service information"""
|
|
return {
|
|
"service": "training-service",
|
|
"version": "1.0.0",
|
|
"description": "ML model training service for bakery demand forecasting",
|
|
"docs": "/docs" if settings.DEBUG else "Documentation disabled in production",
|
|
"health": "/health"
|
|
}
|
|
|
|
# Development server configuration
|
|
if __name__ == "__main__":
|
|
uvicorn.run(
|
|
"app.main:app",
|
|
host="0.0.0.0",
|
|
port=8000,
|
|
reload=settings.DEBUG,
|
|
log_level=settings.LOG_LEVEL.lower(),
|
|
access_log=settings.DEBUG,
|
|
server_header=False,
|
|
date_header=False
|
|
) |