Initial microservices setup from artifacts

This commit is contained in:
Urtzi Alfaro
2025-07-17 13:09:24 +02:00
commit 347ff51bd7
200 changed files with 9559 additions and 0 deletions

0
gateway/app/__init__.py Normal file
View File

View File

View File

@@ -0,0 +1,52 @@
"""
Gateway configuration
"""
import os
from typing import List, Dict
from pydantic import BaseSettings
class Settings(BaseSettings):
"""Application settings"""
# Basic settings
APP_NAME: str = "Bakery Forecasting Gateway"
VERSION: str = "1.0.0"
DEBUG: bool = os.getenv("DEBUG", "False").lower() == "true"
LOG_LEVEL: str = os.getenv("LOG_LEVEL", "INFO")
# CORS settings
CORS_ORIGINS: List[str] = os.getenv("CORS_ORIGINS", "http://localhost:3000,http://localhost:3001").split(",")
# Service URLs
AUTH_SERVICE_URL: str = os.getenv("AUTH_SERVICE_URL", "http://auth-service:8000")
TRAINING_SERVICE_URL: str = os.getenv("TRAINING_SERVICE_URL", "http://training-service:8000")
FORECASTING_SERVICE_URL: str = os.getenv("FORECASTING_SERVICE_URL", "http://forecasting-service:8000")
DATA_SERVICE_URL: str = os.getenv("DATA_SERVICE_URL", "http://data-service:8000")
TENANT_SERVICE_URL: str = os.getenv("TENANT_SERVICE_URL", "http://tenant-service:8000")
NOTIFICATION_SERVICE_URL: str = os.getenv("NOTIFICATION_SERVICE_URL", "http://notification-service:8000")
# Redis settings
REDIS_URL: str = os.getenv("REDIS_URL", "redis://redis:6379/6")
# Rate limiting
RATE_LIMIT_REQUESTS: int = int(os.getenv("RATE_LIMIT_REQUESTS", "100"))
RATE_LIMIT_WINDOW: int = int(os.getenv("RATE_LIMIT_WINDOW", "60"))
# JWT settings
JWT_SECRET_KEY: str = os.getenv("JWT_SECRET_KEY", "your-secret-key-change-in-production")
JWT_ALGORITHM: str = os.getenv("JWT_ALGORITHM", "HS256")
@property
def SERVICES(self) -> Dict[str, str]:
"""Service registry"""
return {
"auth": self.AUTH_SERVICE_URL,
"training": self.TRAINING_SERVICE_URL,
"forecasting": self.FORECASTING_SERVICE_URL,
"data": self.DATA_SERVICE_URL,
"tenant": self.TENANT_SERVICE_URL,
"notification": self.NOTIFICATION_SERVICE_URL
}
settings = Settings()

View File

@@ -0,0 +1,122 @@
"""
Service discovery for microservices
"""
import asyncio
import logging
from typing import Dict, List, Optional
import httpx
import redis.asyncio as redis
from datetime import datetime, timedelta
from app.core.config import settings
logger = logging.getLogger(__name__)
class ServiceDiscovery:
"""Service discovery and health checking"""
def __init__(self):
self.redis_client = redis.from_url(settings.REDIS_URL)
self.services = settings.SERVICES
self.health_check_interval = 30 # seconds
self.health_check_task = None
async def initialize(self):
"""Initialize service discovery"""
logger.info("Initializing service discovery")
# Start health check task
self.health_check_task = asyncio.create_task(self._health_check_loop())
# Initial health check
await self._check_all_services()
async def cleanup(self):
"""Cleanup service discovery"""
if self.health_check_task:
self.health_check_task.cancel()
try:
await self.health_check_task
except asyncio.CancelledError:
pass
await self.redis_client.close()
async def get_service_url(self, service_name: str) -> Optional[str]:
"""Get service URL"""
return self.services.get(service_name)
async def get_healthy_services(self) -> List[str]:
"""Get list of healthy services"""
healthy_services = []
for service_name in self.services:
is_healthy = await self._is_service_healthy(service_name)
if is_healthy:
healthy_services.append(service_name)
return healthy_services
async def _health_check_loop(self):
"""Continuous health check loop"""
while True:
try:
await self._check_all_services()
await asyncio.sleep(self.health_check_interval)
except asyncio.CancelledError:
break
except Exception as e:
logger.error(f"Health check error: {e}")
await asyncio.sleep(self.health_check_interval)
async def _check_all_services(self):
"""Check health of all services"""
for service_name, service_url in self.services.items():
try:
is_healthy = await self._check_service_health(service_url)
await self._update_service_health(service_name, is_healthy)
except Exception as e:
logger.error(f"Health check failed for {service_name}: {e}")
await self._update_service_health(service_name, False)
async def _check_service_health(self, service_url: str) -> bool:
"""Check individual service health"""
try:
async with httpx.AsyncClient(timeout=5.0) as client:
response = await client.get(f"{service_url}/health")
return response.status_code == 200
except Exception as e:
logger.warning(f"Service health check failed: {e}")
return False
async def _update_service_health(self, service_name: str, is_healthy: bool):
"""Update service health status in Redis"""
try:
key = f"service_health:{service_name}"
value = {
"healthy": is_healthy,
"last_check": datetime.utcnow().isoformat(),
"url": self.services[service_name]
}
await self.redis_client.hset(key, mapping=value)
await self.redis_client.expire(key, 300) # 5 minutes TTL
except Exception as e:
logger.error(f"Failed to update service health for {service_name}: {e}")
async def _is_service_healthy(self, service_name: str) -> bool:
"""Check if service is healthy from Redis cache"""
try:
key = f"service_health:{service_name}"
health_data = await self.redis_client.hgetall(key)
if not health_data:
return False
return health_data.get(b'healthy', b'false').decode() == 'True'
except Exception as e:
logger.error(f"Failed to check service health for {service_name}: {e}")
return False

131
gateway/app/main.py Normal file
View File

@@ -0,0 +1,131 @@
"""
API Gateway - Central entry point for all microservices
Handles routing, authentication, rate limiting, and cross-cutting concerns
"""
import asyncio
import logging
from fastapi import FastAPI, Request, HTTPException, Depends
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import JSONResponse
import httpx
import time
from typing import Dict, Any
from app.core.config import settings
from app.core.service_discovery import ServiceDiscovery
from app.middleware.auth import auth_middleware
from app.middleware.logging import logging_middleware
from app.middleware.rate_limit import rate_limit_middleware
from app.routes import auth, training, forecasting, data, tenant, notification
from shared.monitoring.logging import setup_logging
from shared.monitoring.metrics import MetricsCollector
# Setup logging
setup_logging("gateway", settings.LOG_LEVEL)
logger = logging.getLogger(__name__)
# Create FastAPI app
app = FastAPI(
title="Bakery Forecasting API Gateway",
description="Central API Gateway for bakery forecasting microservices",
version="1.0.0",
docs_url="/docs",
redoc_url="/redoc"
)
# Initialize metrics collector
metrics_collector = MetricsCollector("gateway")
# Service discovery
service_discovery = ServiceDiscovery()
# CORS middleware
app.add_middleware(
CORSMiddleware,
allow_origins=settings.CORS_ORIGINS,
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
# Custom middleware
app.add_middleware(auth_middleware)
app.add_middleware(logging_middleware)
app.add_middleware(rate_limit_middleware)
# Include routers
app.include_router(auth.router, prefix="/api/v1/auth", tags=["authentication"])
app.include_router(training.router, prefix="/api/v1/training", tags=["training"])
app.include_router(forecasting.router, prefix="/api/v1/forecasting", tags=["forecasting"])
app.include_router(data.router, prefix="/api/v1/data", tags=["data"])
app.include_router(tenant.router, prefix="/api/v1/tenants", tags=["tenants"])
app.include_router(notification.router, prefix="/api/v1/notifications", tags=["notifications"])
@app.on_event("startup")
async def startup_event():
"""Application startup"""
logger.info("Starting API Gateway")
# Start metrics server
metrics_collector.start_metrics_server(8080)
# Initialize service discovery
await service_discovery.initialize()
logger.info("API Gateway started successfully")
@app.on_event("shutdown")
async def shutdown_event():
"""Application shutdown"""
logger.info("Shutting down API Gateway")
# Clean up service discovery
await service_discovery.cleanup()
logger.info("API Gateway shutdown complete")
@app.get("/health")
async def health_check():
"""Health check endpoint"""
healthy_services = await service_discovery.get_healthy_services()
return {
"status": "healthy",
"service": "gateway",
"version": "1.0.0",
"healthy_services": healthy_services,
"total_services": len(settings.SERVICES),
"timestamp": time.time()
}
@app.get("/metrics")
async def get_metrics():
"""Get basic metrics"""
return {
"service": "gateway",
"uptime": time.time() - app.state.start_time if hasattr(app.state, 'start_time') else 0,
"healthy_services": await service_discovery.get_healthy_services()
}
@app.exception_handler(HTTPException)
async def http_exception_handler(request: Request, exc: HTTPException):
"""Handle HTTP exceptions"""
logger.error(f"HTTP {exc.status_code}: {exc.detail}")
return JSONResponse(
status_code=exc.status_code,
content={"detail": exc.detail, "service": "gateway"}
)
@app.exception_handler(Exception)
async def general_exception_handler(request: Request, exc: Exception):
"""Handle general exceptions"""
logger.error(f"Unhandled exception: {exc}", exc_info=True)
return JSONResponse(
status_code=500,
content={"detail": "Internal server error", "service": "gateway"}
)
if __name__ == "__main__":
import uvicorn
uvicorn.run(app, host="0.0.0.0", port=8000)

View File

View File

@@ -0,0 +1,101 @@
"""
Authentication middleware for gateway
"""
import logging
from fastapi import Request, HTTPException
from fastapi.responses import JSONResponse
import httpx
from typing import Optional
from app.core.config import settings
from shared.auth.jwt_handler import JWTHandler
logger = logging.getLogger(__name__)
# JWT handler
jwt_handler = JWTHandler(settings.JWT_SECRET_KEY, settings.JWT_ALGORITHM)
# Routes that don't require authentication
PUBLIC_ROUTES = [
"/health",
"/metrics",
"/docs",
"/redoc",
"/openapi.json",
"/api/v1/auth/login",
"/api/v1/auth/register",
"/api/v1/auth/refresh"
]
async def auth_middleware(request: Request, call_next):
"""Authentication middleware"""
# Check if route requires authentication
if _is_public_route(request.url.path):
return await call_next(request)
# Get token from header
token = _extract_token(request)
if not token:
return JSONResponse(
status_code=401,
content={"detail": "Authentication required"}
)
# Verify token
try:
# First try to verify token locally
payload = jwt_handler.verify_token(token)
if payload:
# Add user info to request state
request.state.user = payload
return await call_next(request)
else:
# Token invalid or expired, verify with auth service
user_info = await _verify_with_auth_service(token)
if user_info:
request.state.user = user_info
return await call_next(request)
else:
return JSONResponse(
status_code=401,
content={"detail": "Invalid or expired token"}
)
except Exception as e:
logger.error(f"Authentication error: {e}")
return JSONResponse(
status_code=401,
content={"detail": "Authentication failed"}
)
def _is_public_route(path: str) -> bool:
"""Check if route is public"""
return any(path.startswith(route) for route in PUBLIC_ROUTES)
def _extract_token(request: Request) -> Optional[str]:
"""Extract JWT token from request"""
auth_header = request.headers.get("Authorization")
if auth_header and auth_header.startswith("Bearer "):
return auth_header.split(" ")[1]
return None
async def _verify_with_auth_service(token: str) -> Optional[dict]:
"""Verify token with auth service"""
try:
async with httpx.AsyncClient(timeout=5.0) as client:
response = await client.post(
f"{settings.AUTH_SERVICE_URL}/verify",
headers={"Authorization": f"Bearer {token}"}
)
if response.status_code == 200:
return response.json()
else:
return None
except Exception as e:
logger.error(f"Auth service verification failed: {e}")
return None

View File

@@ -0,0 +1,48 @@
"""
Logging middleware for gateway
"""
import logging
import time
from fastapi import Request
import json
logger = logging.getLogger(__name__)
async def logging_middleware(request: Request, call_next):
"""Logging middleware"""
start_time = time.time()
# Log request
logger.info(
f"Request: {request.method} {request.url.path}",
extra={
"method": request.method,
"url": request.url.path,
"query_params": str(request.query_params),
"client_host": request.client.host,
"user_agent": request.headers.get("user-agent", ""),
"request_id": getattr(request.state, 'request_id', None)
}
)
# Process request
response = await call_next(request)
# Calculate duration
duration = time.time() - start_time
# Log response
logger.info(
f"Response: {response.status_code} in {duration:.3f}s",
extra={
"status_code": response.status_code,
"duration": duration,
"method": request.method,
"url": request.url.path,
"request_id": getattr(request.state, 'request_id', None)
}
)
return response

View File

@@ -0,0 +1,85 @@
"""
Rate limiting middleware for gateway
"""
import logging
from fastapi import Request, HTTPException
from fastapi.responses import JSONResponse
import redis.asyncio as redis
from datetime import datetime, timedelta
import hashlib
from app.core.config import settings
logger = logging.getLogger(__name__)
# Redis client for rate limiting
redis_client = redis.from_url(settings.REDIS_URL)
async def rate_limit_middleware(request: Request, call_next):
"""Rate limiting middleware"""
# Skip rate limiting for health checks
if request.url.path in ["/health", "/metrics"]:
return await call_next(request)
# Get client identifier (IP address or user ID)
client_id = _get_client_id(request)
# Check rate limit
if await _is_rate_limited(client_id):
return JSONResponse(
status_code=429,
content={
"detail": "Rate limit exceeded",
"retry_after": settings.RATE_LIMIT_WINDOW
}
)
# Process request
response = await call_next(request)
# Update rate limit counter
await _update_rate_limit(client_id)
return response
def _get_client_id(request: Request) -> str:
"""Get client identifier for rate limiting"""
# Use user ID if authenticated, otherwise use IP
if hasattr(request.state, 'user') and request.state.user:
return f"user:{request.state.user.get('user_id', 'unknown')}"
else:
# Hash IP address for privacy
ip = request.client.host
return f"ip:{hashlib.md5(ip.encode()).hexdigest()}"
async def _is_rate_limited(client_id: str) -> bool:
"""Check if client is rate limited"""
try:
key = f"rate_limit:{client_id}"
current_count = await redis_client.get(key)
if current_count is None:
return False
return int(current_count) >= settings.RATE_LIMIT_REQUESTS
except Exception as e:
logger.error(f"Rate limit check failed: {e}")
return False
async def _update_rate_limit(client_id: str):
"""Update rate limit counter"""
try:
key = f"rate_limit:{client_id}"
# Increment counter
current_count = await redis_client.incr(key)
# Set TTL on first request
if current_count == 1:
await redis_client.expire(key, settings.RATE_LIMIT_WINDOW)
except Exception as e:
logger.error(f"Rate limit update failed: {e}")

View File

161
gateway/app/routes/auth.py Normal file
View File

@@ -0,0 +1,161 @@
"""
Authentication routes for gateway
"""
from fastapi import APIRouter, Request, HTTPException
from fastapi.responses import JSONResponse
import httpx
import logging
from app.core.config import settings
from app.core.service_discovery import ServiceDiscovery
logger = logging.getLogger(__name__)
router = APIRouter()
service_discovery = ServiceDiscovery()
@router.post("/login")
async def login(request: Request):
"""Proxy login request to auth service"""
try:
body = await request.body()
async with httpx.AsyncClient(timeout=10.0) as client:
response = await client.post(
f"{settings.AUTH_SERVICE_URL}/login",
content=body,
headers={"Content-Type": "application/json"}
)
if response.status_code == 200:
return response.json()
else:
return JSONResponse(
status_code=response.status_code,
content=response.json()
)
except httpx.RequestError as e:
logger.error(f"Auth service unavailable: {e}")
raise HTTPException(
status_code=503,
detail="Authentication service unavailable"
)
except Exception as e:
logger.error(f"Login error: {e}")
raise HTTPException(status_code=500, detail="Internal server error")
@router.post("/register")
async def register(request: Request):
"""Proxy register request to auth service"""
try:
body = await request.body()
async with httpx.AsyncClient(timeout=10.0) as client:
response = await client.post(
f"{settings.AUTH_SERVICE_URL}/register",
content=body,
headers={"Content-Type": "application/json"}
)
return JSONResponse(
status_code=response.status_code,
content=response.json()
)
except httpx.RequestError as e:
logger.error(f"Auth service unavailable: {e}")
raise HTTPException(
status_code=503,
detail="Authentication service unavailable"
)
except Exception as e:
logger.error(f"Register error: {e}")
raise HTTPException(status_code=500, detail="Internal server error")
@router.post("/refresh")
async def refresh_token(request: Request):
"""Proxy refresh token request to auth service"""
try:
body = await request.body()
async with httpx.AsyncClient(timeout=10.0) as client:
response = await client.post(
f"{settings.AUTH_SERVICE_URL}/refresh",
content=body,
headers={"Content-Type": "application/json"}
)
return JSONResponse(
status_code=response.status_code,
content=response.json()
)
except httpx.RequestError as e:
logger.error(f"Auth service unavailable: {e}")
raise HTTPException(
status_code=503,
detail="Authentication service unavailable"
)
except Exception as e:
logger.error(f"Refresh token error: {e}")
raise HTTPException(status_code=500, detail="Internal server error")
@router.post("/verify")
async def verify_token(request: Request):
"""Proxy token verification to auth service"""
try:
auth_header = request.headers.get("Authorization")
if not auth_header:
raise HTTPException(status_code=401, detail="Authorization header required")
async with httpx.AsyncClient(timeout=5.0) as client:
response = await client.post(
f"{settings.AUTH_SERVICE_URL}/verify",
headers={"Authorization": auth_header}
)
return JSONResponse(
status_code=response.status_code,
content=response.json()
)
except httpx.RequestError as e:
logger.error(f"Auth service unavailable: {e}")
raise HTTPException(
status_code=503,
detail="Authentication service unavailable"
)
except Exception as e:
logger.error(f"Token verification error: {e}")
raise HTTPException(status_code=500, detail="Internal server error")
@router.post("/logout")
async def logout(request: Request):
"""Proxy logout request to auth service"""
try:
auth_header = request.headers.get("Authorization")
if not auth_header:
raise HTTPException(status_code=401, detail="Authorization header required")
async with httpx.AsyncClient(timeout=5.0) as client:
response = await client.post(
f"{settings.AUTH_SERVICE_URL}/logout",
headers={"Authorization": auth_header}
)
return JSONResponse(
status_code=response.status_code,
content=response.json()
)
except httpx.RequestError as e:
logger.error(f"Auth service unavailable: {e}")
raise HTTPException(
status_code=503,
detail="Authentication service unavailable"
)
except Exception as e:
logger.error(f"Logout error: {e}")
raise HTTPException(status_code=500, detail="Internal server error")

View File

@@ -0,0 +1,166 @@
"""
Training routes for gateway
"""
from fastapi import APIRouter, Request, HTTPException, Query
from fastapi.responses import JSONResponse
import httpx
import logging
from typing import Optional
from app.core.config import settings
logger = logging.getLogger(__name__)
router = APIRouter()
@router.post("/train")
async def start_training(request: Request):
"""Proxy training request to training service"""
try:
body = await request.body()
auth_header = request.headers.get("Authorization")
async with httpx.AsyncClient(timeout=30.0) as client:
response = await client.post(
f"{settings.TRAINING_SERVICE_URL}/train",
content=body,
headers={
"Content-Type": "application/json",
"Authorization": auth_header
}
)
return JSONResponse(
status_code=response.status_code,
content=response.json()
)
except httpx.RequestError as e:
logger.error(f"Training service unavailable: {e}")
raise HTTPException(
status_code=503,
detail="Training service unavailable"
)
except Exception as e:
logger.error(f"Training error: {e}")
raise HTTPException(status_code=500, detail="Internal server error")
@router.get("/status/{training_job_id}")
async def get_training_status(training_job_id: str, request: Request):
"""Get training job status"""
try:
auth_header = request.headers.get("Authorization")
async with httpx.AsyncClient(timeout=10.0) as client:
response = await client.get(
f"{settings.TRAINING_SERVICE_URL}/status/{training_job_id}",
headers={"Authorization": auth_header}
)
return JSONResponse(
status_code=response.status_code,
content=response.json()
)
except httpx.RequestError as e:
logger.error(f"Training service unavailable: {e}")
raise HTTPException(
status_code=503,
detail="Training service unavailable"
)
except Exception as e:
logger.error(f"Training status error: {e}")
raise HTTPException(status_code=500, detail="Internal server error")
@router.get("/models")
async def get_trained_models(request: Request):
"""Get trained models"""
try:
auth_header = request.headers.get("Authorization")
async with httpx.AsyncClient(timeout=10.0) as client:
response = await client.get(
f"{settings.TRAINING_SERVICE_URL}/models",
headers={"Authorization": auth_header}
)
return JSONResponse(
status_code=response.status_code,
content=response.json()
)
except httpx.RequestError as e:
logger.error(f"Training service unavailable: {e}")
raise HTTPException(
status_code=503,
detail="Training service unavailable"
)
except Exception as e:
logger.error(f"Get models error: {e}")
raise HTTPException(status_code=500, detail="Internal server error")
@router.get("/jobs")
async def get_training_jobs(
request: Request,
limit: Optional[int] = Query(10, ge=1, le=100),
offset: Optional[int] = Query(0, ge=0)
):
"""Get training jobs"""
try:
auth_header = request.headers.get("Authorization")
async with httpx.AsyncClient(timeout=10.0) as client:
response = await client.get(
f"{settings.TRAINING_SERVICE_URL}/jobs",
params={"limit": limit, "offset": offset},
headers={"Authorization": auth_header}
)
return JSONResponse(
status_code=response.status_code,
content=response.json()
)
except httpx.RequestError as e:
logger.error(f"Training service unavailable: {e}")
raise HTTPException(
status_code=503,
detail="Training service unavailable"
)
except Exception as e:
logger.error(f"Get training jobs error: {e}")
raise HTTPException(status_code=500, detail="Internal server error")
# gateway/Dockerfile
FROM python:3.11-slim
WORKDIR /app
# Install system dependencies
RUN apt-get update && apt-get install -y \
gcc \
curl \
&& rm -rf /var/lib/apt/lists/*
# Copy requirements
COPY requirements.txt .
# Install Python dependencies
RUN pip install --no-cache-dir -r requirements.txt
# Copy application code
COPY . .
# Add shared libraries to Python path
ENV PYTHONPATH="/app:/app/shared:$PYTHONPATH"
# Expose port
EXPOSE 8000
# Health check
HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
CMD curl -f http://localhost:8000/health || exit 1
# Run application
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]