REFACTOR external service and improve websocket training

This commit is contained in:
Urtzi Alfaro
2025-10-09 14:11:02 +02:00
parent 7c72f83c51
commit 3c689b4f98
111 changed files with 13289 additions and 2374 deletions

View File

@@ -0,0 +1,391 @@
# services/external/app/api/city_operations.py
"""
City Operations API - New endpoints for city-based data access
"""
from fastapi import APIRouter, Depends, HTTPException, Query, Path
from typing import List
from datetime import datetime
from uuid import UUID
import structlog
from app.schemas.city_data import CityInfoResponse, DataAvailabilityResponse
from app.schemas.weather import WeatherDataResponse, WeatherForecastResponse, WeatherForecastAPIResponse
from app.schemas.traffic import TrafficDataResponse
from app.registry.city_registry import CityRegistry
from app.registry.geolocation_mapper import GeolocationMapper
from app.repositories.city_data_repository import CityDataRepository
from app.cache.redis_cache import ExternalDataCache
from app.services.weather_service import WeatherService
from app.services.traffic_service import TrafficService
from shared.routing.route_builder import RouteBuilder
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.database import get_db
route_builder = RouteBuilder('external')
router = APIRouter(tags=["city-operations"])
logger = structlog.get_logger()
@router.get(
route_builder.build_base_route("cities"),
response_model=List[CityInfoResponse]
)
async def list_supported_cities():
"""List all enabled cities with data availability"""
registry = CityRegistry()
cities = registry.get_enabled_cities()
return [
CityInfoResponse(
city_id=city.city_id,
name=city.name,
country=city.country.value,
latitude=city.latitude,
longitude=city.longitude,
radius_km=city.radius_km,
weather_provider=city.weather_provider.value,
traffic_provider=city.traffic_provider.value,
enabled=city.enabled
)
for city in cities
]
@router.get(
route_builder.build_operations_route("cities/{city_id}/availability"),
response_model=DataAvailabilityResponse
)
async def get_city_data_availability(
city_id: str = Path(..., description="City ID"),
db: AsyncSession = Depends(get_db)
):
"""Get data availability for a specific city"""
registry = CityRegistry()
city = registry.get_city(city_id)
if not city:
raise HTTPException(status_code=404, detail="City not found")
from sqlalchemy import text
weather_stmt = text(
"SELECT MIN(date), MAX(date), COUNT(*) FROM city_weather_data WHERE city_id = :city_id"
)
weather_result = await db.execute(weather_stmt, {"city_id": city_id})
weather_row = weather_result.fetchone()
weather_min, weather_max, weather_count = weather_row if weather_row else (None, None, 0)
traffic_stmt = text(
"SELECT MIN(date), MAX(date), COUNT(*) FROM city_traffic_data WHERE city_id = :city_id"
)
traffic_result = await db.execute(traffic_stmt, {"city_id": city_id})
traffic_row = traffic_result.fetchone()
traffic_min, traffic_max, traffic_count = traffic_row if traffic_row else (None, None, 0)
return DataAvailabilityResponse(
city_id=city_id,
city_name=city.name,
weather_available=weather_count > 0,
weather_start_date=weather_min.isoformat() if weather_min else None,
weather_end_date=weather_max.isoformat() if weather_max else None,
weather_record_count=weather_count or 0,
traffic_available=traffic_count > 0,
traffic_start_date=traffic_min.isoformat() if traffic_min else None,
traffic_end_date=traffic_max.isoformat() if traffic_max else None,
traffic_record_count=traffic_count or 0
)
@router.get(
route_builder.build_operations_route("historical-weather-optimized"),
response_model=List[WeatherDataResponse]
)
async def get_historical_weather_optimized(
tenant_id: UUID = Path(..., description="Tenant ID"),
latitude: float = Query(..., description="Latitude"),
longitude: float = Query(..., description="Longitude"),
start_date: datetime = Query(..., description="Start date"),
end_date: datetime = Query(..., description="End date"),
db: AsyncSession = Depends(get_db)
):
"""
Get historical weather data using city-based cached data
This is the FAST endpoint for training service
"""
try:
mapper = GeolocationMapper()
mapping = mapper.map_tenant_to_city(latitude, longitude)
if not mapping:
raise HTTPException(
status_code=404,
detail="No supported city found for this location"
)
city, distance = mapping
logger.info(
"Fetching historical weather from cache",
tenant_id=tenant_id,
city=city.name,
distance_km=round(distance, 2)
)
cache = ExternalDataCache()
cached_data = await cache.get_cached_weather(
city.city_id, start_date, end_date
)
if cached_data:
logger.info("Weather cache hit", records=len(cached_data))
return cached_data
repo = CityDataRepository(db)
db_records = await repo.get_weather_by_city_and_range(
city.city_id, start_date, end_date
)
response_data = [
WeatherDataResponse(
id=str(record.id),
location_id=f"{city.city_id}_{record.date.date()}",
date=record.date,
temperature=record.temperature,
precipitation=record.precipitation,
humidity=record.humidity,
wind_speed=record.wind_speed,
pressure=record.pressure,
description=record.description,
source=record.source,
raw_data=None,
created_at=record.created_at,
updated_at=record.updated_at
)
for record in db_records
]
await cache.set_cached_weather(
city.city_id, start_date, end_date, response_data
)
logger.info(
"Historical weather data retrieved",
records=len(response_data),
source="database"
)
return response_data
except HTTPException:
raise
except Exception as e:
logger.error("Error fetching historical weather", error=str(e))
raise HTTPException(status_code=500, detail="Internal server error")
@router.get(
route_builder.build_operations_route("historical-traffic-optimized"),
response_model=List[TrafficDataResponse]
)
async def get_historical_traffic_optimized(
tenant_id: UUID = Path(..., description="Tenant ID"),
latitude: float = Query(..., description="Latitude"),
longitude: float = Query(..., description="Longitude"),
start_date: datetime = Query(..., description="Start date"),
end_date: datetime = Query(..., description="End date"),
db: AsyncSession = Depends(get_db)
):
"""
Get historical traffic data using city-based cached data
This is the FAST endpoint for training service
"""
try:
mapper = GeolocationMapper()
mapping = mapper.map_tenant_to_city(latitude, longitude)
if not mapping:
raise HTTPException(
status_code=404,
detail="No supported city found for this location"
)
city, distance = mapping
logger.info(
"Fetching historical traffic from cache",
tenant_id=tenant_id,
city=city.name,
distance_km=round(distance, 2)
)
cache = ExternalDataCache()
cached_data = await cache.get_cached_traffic(
city.city_id, start_date, end_date
)
if cached_data:
logger.info("Traffic cache hit", records=len(cached_data))
return cached_data
logger.debug("Starting DB query for traffic", city_id=city.city_id)
repo = CityDataRepository(db)
db_records = await repo.get_traffic_by_city_and_range(
city.city_id, start_date, end_date
)
logger.debug("DB query completed", records=len(db_records))
logger.debug("Creating response objects")
response_data = [
TrafficDataResponse(
date=record.date,
traffic_volume=record.traffic_volume,
pedestrian_count=record.pedestrian_count,
congestion_level=record.congestion_level,
average_speed=record.average_speed,
source=record.source
)
for record in db_records
]
logger.debug("Response objects created", count=len(response_data))
logger.debug("Caching traffic data")
await cache.set_cached_traffic(
city.city_id, start_date, end_date, response_data
)
logger.debug("Caching completed")
logger.info(
"Historical traffic data retrieved",
records=len(response_data),
source="database"
)
return response_data
except HTTPException:
raise
except Exception as e:
logger.error("Error fetching historical traffic", error=str(e))
raise HTTPException(status_code=500, detail="Internal server error")
# ================================================================
# REAL-TIME & FORECAST ENDPOINTS
# ================================================================
@router.get(
route_builder.build_operations_route("weather/current"),
response_model=WeatherDataResponse
)
async def get_current_weather(
tenant_id: UUID = Path(..., description="Tenant ID"),
latitude: float = Query(..., description="Latitude"),
longitude: float = Query(..., description="Longitude")
):
"""
Get current weather for a location (real-time data from AEMET)
"""
try:
weather_service = WeatherService()
weather_data = await weather_service.get_current_weather(latitude, longitude)
if not weather_data:
raise HTTPException(
status_code=404,
detail="No weather data available for this location"
)
logger.info(
"Current weather retrieved",
tenant_id=tenant_id,
latitude=latitude,
longitude=longitude
)
return weather_data
except HTTPException:
raise
except Exception as e:
logger.error("Error fetching current weather", error=str(e))
raise HTTPException(status_code=500, detail="Internal server error")
@router.get(
route_builder.build_operations_route("weather/forecast")
)
async def get_weather_forecast(
tenant_id: UUID = Path(..., description="Tenant ID"),
latitude: float = Query(..., description="Latitude"),
longitude: float = Query(..., description="Longitude"),
days: int = Query(7, ge=1, le=14, description="Number of days to forecast")
):
"""
Get weather forecast for a location (from AEMET)
Returns list of forecast objects with: forecast_date, generated_at, temperature, precipitation, humidity, wind_speed, description, source
"""
try:
weather_service = WeatherService()
forecast_data = await weather_service.get_weather_forecast(latitude, longitude, days)
if not forecast_data:
raise HTTPException(
status_code=404,
detail="No forecast data available for this location"
)
logger.info(
"Weather forecast retrieved",
tenant_id=tenant_id,
latitude=latitude,
longitude=longitude,
days=days,
count=len(forecast_data)
)
return forecast_data
except HTTPException:
raise
except Exception as e:
logger.error("Error fetching weather forecast", error=str(e))
raise HTTPException(status_code=500, detail="Internal server error")
@router.get(
route_builder.build_operations_route("traffic/current"),
response_model=TrafficDataResponse
)
async def get_current_traffic(
tenant_id: UUID = Path(..., description="Tenant ID"),
latitude: float = Query(..., description="Latitude"),
longitude: float = Query(..., description="Longitude")
):
"""
Get current traffic conditions for a location (real-time data from Madrid OpenData)
"""
try:
traffic_service = TrafficService()
traffic_data = await traffic_service.get_current_traffic(latitude, longitude)
if not traffic_data:
raise HTTPException(
status_code=404,
detail="No traffic data available for this location"
)
logger.info(
"Current traffic retrieved",
tenant_id=tenant_id,
latitude=latitude,
longitude=longitude
)
return traffic_data
except HTTPException:
raise
except Exception as e:
logger.error("Error fetching current traffic", error=str(e))
raise HTTPException(status_code=500, detail="Internal server error")

View File

@@ -1,407 +0,0 @@
# services/external/app/api/external_operations.py
"""
External Operations API - Business operations for fetching external data
"""
from fastapi import APIRouter, Depends, HTTPException, Query, Path
from typing import List, Dict, Any
from datetime import datetime
from uuid import UUID
import structlog
from app.schemas.weather import (
WeatherDataResponse,
WeatherForecastResponse,
WeatherForecastRequest,
HistoricalWeatherRequest,
HourlyForecastRequest,
HourlyForecastResponse
)
from app.schemas.traffic import (
TrafficDataResponse,
TrafficForecastRequest,
HistoricalTrafficRequest
)
from app.services.weather_service import WeatherService
from app.services.traffic_service import TrafficService
from app.services.messaging import publish_weather_updated, publish_traffic_updated
from shared.auth.decorators import get_current_user_dep
from shared.auth.access_control import require_user_role
from shared.routing.route_builder import RouteBuilder
route_builder = RouteBuilder('external')
router = APIRouter(tags=["external-operations"])
logger = structlog.get_logger()
def get_weather_service():
"""Dependency injection for WeatherService"""
return WeatherService()
def get_traffic_service():
"""Dependency injection for TrafficService"""
return TrafficService()
# Weather Operations
@router.get(
route_builder.build_operations_route("weather/current"),
response_model=WeatherDataResponse
)
@require_user_role(['viewer', 'member', 'admin', 'owner'])
async def get_current_weather(
latitude: float = Query(..., description="Latitude"),
longitude: float = Query(..., description="Longitude"),
tenant_id: UUID = Path(..., description="Tenant ID"),
current_user: Dict[str, Any] = Depends(get_current_user_dep),
weather_service: WeatherService = Depends(get_weather_service)
):
"""Get current weather data for location from external API"""
try:
logger.debug("Getting current weather",
lat=latitude,
lon=longitude,
tenant_id=tenant_id,
user_id=current_user["user_id"])
weather = await weather_service.get_current_weather(latitude, longitude)
if not weather:
raise HTTPException(status_code=503, detail="Weather service temporarily unavailable")
try:
await publish_weather_updated({
"type": "current_weather_requested",
"tenant_id": str(tenant_id),
"latitude": latitude,
"longitude": longitude,
"requested_by": current_user["user_id"],
"timestamp": datetime.utcnow().isoformat()
})
except Exception as e:
logger.warning("Failed to publish weather event", error=str(e))
return weather
except HTTPException:
raise
except Exception as e:
logger.error("Failed to get current weather", error=str(e))
raise HTTPException(status_code=500, detail=f"Internal server error: {str(e)}")
@router.post(
route_builder.build_operations_route("weather/historical"),
response_model=List[WeatherDataResponse]
)
@require_user_role(['viewer', 'member', 'admin', 'owner'])
async def get_historical_weather(
request: HistoricalWeatherRequest,
tenant_id: UUID = Path(..., description="Tenant ID"),
current_user: Dict[str, Any] = Depends(get_current_user_dep),
weather_service: WeatherService = Depends(get_weather_service)
):
"""Get historical weather data with date range"""
try:
if request.end_date <= request.start_date:
raise HTTPException(status_code=400, detail="End date must be after start date")
if (request.end_date - request.start_date).days > 1000:
raise HTTPException(status_code=400, detail="Date range cannot exceed 90 days")
historical_data = await weather_service.get_historical_weather(
request.latitude, request.longitude, request.start_date, request.end_date)
try:
await publish_weather_updated({
"type": "historical_requested",
"latitude": request.latitude,
"longitude": request.longitude,
"start_date": request.start_date.isoformat(),
"end_date": request.end_date.isoformat(),
"records_count": len(historical_data),
"timestamp": datetime.utcnow().isoformat()
})
except Exception as pub_error:
logger.warning("Failed to publish historical weather event", error=str(pub_error))
return historical_data
except HTTPException:
raise
except Exception as e:
logger.error("Unexpected error in historical weather API", error=str(e))
raise HTTPException(status_code=500, detail=f"Internal server error: {str(e)}")
@router.post(
route_builder.build_operations_route("weather/forecast"),
response_model=List[WeatherForecastResponse]
)
@require_user_role(['viewer', 'member', 'admin', 'owner'])
async def get_weather_forecast(
request: WeatherForecastRequest,
tenant_id: UUID = Path(..., description="Tenant ID"),
current_user: Dict[str, Any] = Depends(get_current_user_dep),
weather_service: WeatherService = Depends(get_weather_service)
):
"""Get weather forecast for location"""
try:
logger.debug("Getting weather forecast",
lat=request.latitude,
lon=request.longitude,
days=request.days,
tenant_id=tenant_id)
forecast = await weather_service.get_weather_forecast(request.latitude, request.longitude, request.days)
if not forecast:
logger.info("Weather forecast unavailable - returning empty list")
return []
try:
await publish_weather_updated({
"type": "forecast_requested",
"tenant_id": str(tenant_id),
"latitude": request.latitude,
"longitude": request.longitude,
"days": request.days,
"requested_by": current_user["user_id"],
"timestamp": datetime.utcnow().isoformat()
})
except Exception as e:
logger.warning("Failed to publish forecast event", error=str(e))
return forecast
except HTTPException:
raise
except Exception as e:
logger.error("Failed to get weather forecast", error=str(e))
raise HTTPException(status_code=500, detail=f"Internal server error: {str(e)}")
@router.post(
route_builder.build_operations_route("weather/hourly-forecast"),
response_model=List[HourlyForecastResponse]
)
@require_user_role(['viewer', 'member', 'admin', 'owner'])
async def get_hourly_weather_forecast(
request: HourlyForecastRequest,
tenant_id: UUID = Path(..., description="Tenant ID"),
current_user: Dict[str, Any] = Depends(get_current_user_dep),
weather_service: WeatherService = Depends(get_weather_service)
):
"""Get hourly weather forecast for location"""
try:
logger.debug("Getting hourly weather forecast",
lat=request.latitude,
lon=request.longitude,
hours=request.hours,
tenant_id=tenant_id)
hourly_forecast = await weather_service.get_hourly_forecast(
request.latitude, request.longitude, request.hours
)
if not hourly_forecast:
logger.info("Hourly weather forecast unavailable - returning empty list")
return []
try:
await publish_weather_updated({
"type": "hourly_forecast_requested",
"tenant_id": str(tenant_id),
"latitude": request.latitude,
"longitude": request.longitude,
"hours": request.hours,
"requested_by": current_user["user_id"],
"forecast_count": len(hourly_forecast),
"timestamp": datetime.utcnow().isoformat()
})
except Exception as e:
logger.warning("Failed to publish hourly forecast event", error=str(e))
return hourly_forecast
except HTTPException:
raise
except Exception as e:
logger.error("Failed to get hourly weather forecast", error=str(e))
raise HTTPException(status_code=500, detail=f"Internal server error: {str(e)}")
@router.get(
route_builder.build_operations_route("weather-status"),
response_model=dict
)
async def get_weather_status(
weather_service: WeatherService = Depends(get_weather_service)
):
"""Get weather API status and diagnostics"""
try:
aemet_status = "unknown"
aemet_message = "Not tested"
try:
test_weather = await weather_service.get_current_weather(40.4168, -3.7038)
if test_weather and hasattr(test_weather, 'source') and test_weather.source == "aemet":
aemet_status = "healthy"
aemet_message = "AEMET API responding correctly"
elif test_weather and hasattr(test_weather, 'source') and test_weather.source == "synthetic":
aemet_status = "degraded"
aemet_message = "Using synthetic weather data (AEMET API unavailable)"
else:
aemet_status = "unknown"
aemet_message = "Weather source unknown"
except Exception as test_error:
aemet_status = "unhealthy"
aemet_message = f"AEMET API test failed: {str(test_error)}"
return {
"status": aemet_status,
"message": aemet_message,
"timestamp": datetime.utcnow().isoformat()
}
except Exception as e:
logger.error("Weather status check failed", error=str(e))
raise HTTPException(status_code=500, detail=f"Status check failed: {str(e)}")
# Traffic Operations
@router.get(
route_builder.build_operations_route("traffic/current"),
response_model=TrafficDataResponse
)
@require_user_role(['viewer', 'member', 'admin', 'owner'])
async def get_current_traffic(
latitude: float = Query(..., description="Latitude"),
longitude: float = Query(..., description="Longitude"),
tenant_id: UUID = Path(..., description="Tenant ID"),
current_user: Dict[str, Any] = Depends(get_current_user_dep),
traffic_service: TrafficService = Depends(get_traffic_service)
):
"""Get current traffic data for location from external API"""
try:
logger.debug("Getting current traffic",
lat=latitude,
lon=longitude,
tenant_id=tenant_id,
user_id=current_user["user_id"])
traffic = await traffic_service.get_current_traffic(latitude, longitude)
if not traffic:
raise HTTPException(status_code=503, detail="Traffic service temporarily unavailable")
try:
await publish_traffic_updated({
"type": "current_traffic_requested",
"tenant_id": str(tenant_id),
"latitude": latitude,
"longitude": longitude,
"requested_by": current_user["user_id"],
"timestamp": datetime.utcnow().isoformat()
})
except Exception as e:
logger.warning("Failed to publish traffic event", error=str(e))
return traffic
except HTTPException:
raise
except Exception as e:
logger.error("Failed to get current traffic", error=str(e))
raise HTTPException(status_code=500, detail=f"Internal server error: {str(e)}")
@router.post(
route_builder.build_operations_route("traffic/historical"),
response_model=List[TrafficDataResponse]
)
@require_user_role(['viewer', 'member', 'admin', 'owner'])
async def get_historical_traffic(
request: HistoricalTrafficRequest,
tenant_id: UUID = Path(..., description="Tenant ID"),
current_user: Dict[str, Any] = Depends(get_current_user_dep),
traffic_service: TrafficService = Depends(get_traffic_service)
):
"""Get historical traffic data with date range"""
try:
if request.end_date <= request.start_date:
raise HTTPException(status_code=400, detail="End date must be after start date")
historical_data = await traffic_service.get_historical_traffic(
request.latitude, request.longitude, request.start_date, request.end_date)
try:
await publish_traffic_updated({
"type": "historical_requested",
"latitude": request.latitude,
"longitude": request.longitude,
"start_date": request.start_date.isoformat(),
"end_date": request.end_date.isoformat(),
"records_count": len(historical_data),
"timestamp": datetime.utcnow().isoformat()
})
except Exception as pub_error:
logger.warning("Failed to publish historical traffic event", error=str(pub_error))
return historical_data
except HTTPException:
raise
except Exception as e:
logger.error("Unexpected error in historical traffic API", error=str(e))
raise HTTPException(status_code=500, detail=f"Internal server error: {str(e)}")
@router.post(
route_builder.build_operations_route("traffic/forecast"),
response_model=List[TrafficDataResponse]
)
@require_user_role(['viewer', 'member', 'admin', 'owner'])
async def get_traffic_forecast(
request: TrafficForecastRequest,
tenant_id: UUID = Path(..., description="Tenant ID"),
current_user: Dict[str, Any] = Depends(get_current_user_dep),
traffic_service: TrafficService = Depends(get_traffic_service)
):
"""Get traffic forecast for location"""
try:
logger.debug("Getting traffic forecast",
lat=request.latitude,
lon=request.longitude,
hours=request.hours,
tenant_id=tenant_id)
forecast = await traffic_service.get_traffic_forecast(request.latitude, request.longitude, request.hours)
if not forecast:
logger.info("Traffic forecast unavailable - returning empty list")
return []
try:
await publish_traffic_updated({
"type": "forecast_requested",
"tenant_id": str(tenant_id),
"latitude": request.latitude,
"longitude": request.longitude,
"hours": request.hours,
"requested_by": current_user["user_id"],
"timestamp": datetime.utcnow().isoformat()
})
except Exception as e:
logger.warning("Failed to publish traffic forecast event", error=str(e))
return forecast
except HTTPException:
raise
except Exception as e:
logger.error("Failed to get traffic forecast", error=str(e))
raise HTTPException(status_code=500, detail=f"Internal server error: {str(e)}")