REFACTOR data service
This commit is contained in:
Urtzi Alfaro
1
services/external/app/services/__init__.py
vendored
Normal file
1
services/external/app/services/__init__.py
vendored
Normal file
@@ -0,0 +1 @@
|
||||
# services/external/app/services/__init__.py
|
||||
63
services/external/app/services/messaging.py
vendored
Normal file
63
services/external/app/services/messaging.py
vendored
Normal file
@@ -0,0 +1,63 @@
|
||||
# services/external/app/services/messaging.py
|
||||
"""
|
||||
External Service Messaging - Event Publishing using shared messaging infrastructure
|
||||
"""
|
||||
|
||||
from shared.messaging.rabbitmq import RabbitMQClient
|
||||
from app.core.config import settings
|
||||
import structlog
|
||||
|
||||
logger = structlog.get_logger()
|
||||
|
||||
# Single global instance
|
||||
data_publisher = RabbitMQClient(settings.RABBITMQ_URL, "data-service")
|
||||
|
||||
async def setup_messaging():
|
||||
"""Initialize messaging for data service"""
|
||||
try:
|
||||
success = await data_publisher.connect()
|
||||
if success:
|
||||
logger.info("Data service messaging initialized")
|
||||
else:
|
||||
logger.warning("Data service messaging failed to initialize")
|
||||
return success
|
||||
except Exception as e:
|
||||
logger.warning("Failed to setup messaging", error=str(e))
|
||||
return False
|
||||
|
||||
async def cleanup_messaging():
|
||||
"""Cleanup messaging for data service"""
|
||||
try:
|
||||
await data_publisher.disconnect()
|
||||
logger.info("Data service messaging cleaned up")
|
||||
except Exception as e:
|
||||
logger.warning("Error during messaging cleanup", error=str(e))
|
||||
|
||||
async def publish_weather_updated(data: dict) -> bool:
|
||||
"""Publish weather updated event"""
|
||||
try:
|
||||
return await data_publisher.publish_data_event("weather.updated", data)
|
||||
except Exception as e:
|
||||
logger.warning("Failed to publish weather updated event", error=str(e))
|
||||
return False
|
||||
|
||||
async def publish_traffic_updated(data: dict) -> bool:
|
||||
"""Publish traffic updated event"""
|
||||
try:
|
||||
return await data_publisher.publish_data_event("traffic.updated", data)
|
||||
except Exception as e:
|
||||
logger.warning("Failed to publish traffic updated event", error=str(e))
|
||||
return False
|
||||
|
||||
|
||||
|
||||
# Health check for messaging
|
||||
async def check_messaging_health() -> dict:
|
||||
"""Check messaging system health"""
|
||||
try:
|
||||
if data_publisher.connected:
|
||||
return {"status": "healthy", "service": "rabbitmq", "connected": True}
|
||||
else:
|
||||
return {"status": "unhealthy", "service": "rabbitmq", "connected": False, "error": "Not connected"}
|
||||
except Exception as e:
|
||||
return {"status": "unhealthy", "service": "rabbitmq", "connected": False, "error": str(e)}
|
||||
298
services/external/app/services/traffic_service.py
vendored
Normal file
298
services/external/app/services/traffic_service.py
vendored
Normal file
@@ -0,0 +1,298 @@
|
||||
# ================================================================
|
||||
# services/data/app/services/traffic_service.py
|
||||
# ================================================================
|
||||
"""
|
||||
Abstracted Traffic Service - Universal interface for traffic data across multiple cities
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from datetime import datetime
|
||||
from typing import Dict, List, Any, Optional, Tuple
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
import structlog
|
||||
|
||||
from app.external.apis.traffic import UniversalTrafficClient
|
||||
from app.models.traffic import TrafficData
|
||||
from app.repositories.traffic_repository import TrafficRepository
|
||||
|
||||
logger = structlog.get_logger()
|
||||
from app.core.database import database_manager
|
||||
|
||||
class TrafficService:
|
||||
"""
|
||||
Abstracted traffic service providing unified interface for traffic data
|
||||
Routes requests to appropriate city-specific clients automatically
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.universal_client = UniversalTrafficClient()
|
||||
self.database_manager = database_manager
|
||||
|
||||
async def get_current_traffic(
|
||||
self,
|
||||
latitude: float,
|
||||
longitude: float,
|
||||
tenant_id: Optional[str] = None
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Get current traffic data for any supported location
|
||||
|
||||
Args:
|
||||
latitude: Query location latitude
|
||||
longitude: Query location longitude
|
||||
tenant_id: Optional tenant identifier for logging/analytics
|
||||
|
||||
Returns:
|
||||
Dict with current traffic data or None if not available
|
||||
"""
|
||||
try:
|
||||
logger.info("Getting current traffic data",
|
||||
lat=latitude, lon=longitude, tenant_id=tenant_id)
|
||||
|
||||
# Delegate to universal client
|
||||
traffic_data = await self.universal_client.get_current_traffic(latitude, longitude)
|
||||
|
||||
if traffic_data:
|
||||
# Add service metadata
|
||||
traffic_data['service_metadata'] = {
|
||||
'request_timestamp': datetime.now().isoformat(),
|
||||
'tenant_id': tenant_id,
|
||||
'service_version': '2.0',
|
||||
'query_location': {'latitude': latitude, 'longitude': longitude}
|
||||
}
|
||||
|
||||
logger.info("Successfully retrieved current traffic data",
|
||||
lat=latitude, lon=longitude,
|
||||
source=traffic_data.get('source', 'unknown'))
|
||||
|
||||
return traffic_data
|
||||
else:
|
||||
logger.warning("No current traffic data available",
|
||||
lat=latitude, lon=longitude)
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Error getting current traffic data",
|
||||
lat=latitude, lon=longitude, error=str(e))
|
||||
return None
|
||||
|
||||
async def get_historical_traffic(
|
||||
self,
|
||||
latitude: float,
|
||||
longitude: float,
|
||||
start_date: datetime,
|
||||
end_date: datetime,
|
||||
tenant_id: Optional[str] = None
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Get historical traffic data for any supported location with database storage
|
||||
|
||||
Args:
|
||||
latitude: Query location latitude
|
||||
longitude: Query location longitude
|
||||
start_date: Start date for historical data
|
||||
end_date: End date for historical data
|
||||
tenant_id: Optional tenant identifier
|
||||
|
||||
Returns:
|
||||
List of historical traffic data dictionaries
|
||||
"""
|
||||
try:
|
||||
logger.info("Getting historical traffic data",
|
||||
lat=latitude, lon=longitude,
|
||||
start=start_date, end=end_date, tenant_id=tenant_id)
|
||||
|
||||
# Validate date range
|
||||
if start_date >= end_date:
|
||||
logger.warning("Invalid date range", start=start_date, end=end_date)
|
||||
return []
|
||||
|
||||
location_id = f"{latitude:.4f},{longitude:.4f}"
|
||||
|
||||
async with self.database_manager.get_session() as session:
|
||||
traffic_repo = TrafficRepository(session)
|
||||
# Check database first using the repository
|
||||
db_records = await traffic_repo.get_by_location_and_date_range(
|
||||
latitude, longitude, start_date, end_date, tenant_id
|
||||
)
|
||||
|
||||
if db_records:
|
||||
logger.info("Historical traffic data found in database",
|
||||
count=len(db_records))
|
||||
return [self._convert_db_record_to_dict(record) for record in db_records]
|
||||
|
||||
# Delegate to universal client if not in DB
|
||||
traffic_data = await self.universal_client.get_historical_traffic(
|
||||
latitude, longitude, start_date, end_date
|
||||
)
|
||||
|
||||
if traffic_data:
|
||||
# Add service metadata to each record
|
||||
for record in traffic_data:
|
||||
record['service_metadata'] = {
|
||||
'request_timestamp': datetime.now().isoformat(),
|
||||
'tenant_id': tenant_id,
|
||||
'service_version': '2.0',
|
||||
'query_location': {'latitude': latitude, 'longitude': longitude},
|
||||
'date_range': {
|
||||
'start': start_date.isoformat(),
|
||||
'end': end_date.isoformat()
|
||||
}
|
||||
}
|
||||
|
||||
async with self.database_manager.get_session() as session:
|
||||
traffic_repo = TrafficRepository(session)
|
||||
# Store in database using the repository
|
||||
stored_count = await traffic_repo.store_traffic_data_batch(
|
||||
traffic_data, location_id, tenant_id
|
||||
)
|
||||
logger.info("Traffic data stored for re-training",
|
||||
fetched=len(traffic_data), stored=stored_count,
|
||||
location=location_id)
|
||||
|
||||
logger.info("Successfully retrieved historical traffic data",
|
||||
lat=latitude, lon=longitude, records=len(traffic_data))
|
||||
|
||||
return traffic_data
|
||||
else:
|
||||
logger.info("No historical traffic data available",
|
||||
lat=latitude, lon=longitude)
|
||||
return []
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Error getting historical traffic data",
|
||||
lat=latitude, lon=longitude, error=str(e))
|
||||
return []
|
||||
|
||||
def _convert_db_record_to_dict(self, record: TrafficData) -> Dict[str, Any]:
|
||||
"""Convert database record to dictionary format"""
|
||||
return {
|
||||
'date': record.date,
|
||||
'traffic_volume': record.traffic_volume,
|
||||
'pedestrian_count': record.pedestrian_count,
|
||||
'congestion_level': record.congestion_level,
|
||||
'average_speed': record.average_speed,
|
||||
'source': record.source,
|
||||
'location_id': record.location_id,
|
||||
'raw_data': record.raw_data
|
||||
}
|
||||
|
||||
async def get_traffic_events(
|
||||
self,
|
||||
latitude: float,
|
||||
longitude: float,
|
||||
radius_km: float = 5.0,
|
||||
tenant_id: Optional[str] = None
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Get traffic events and incidents for any supported location
|
||||
|
||||
Args:
|
||||
latitude: Query location latitude
|
||||
longitude: Query location longitude
|
||||
radius_km: Search radius in kilometers
|
||||
tenant_id: Optional tenant identifier
|
||||
|
||||
Returns:
|
||||
List of traffic events
|
||||
"""
|
||||
try:
|
||||
logger.info("Getting traffic events",
|
||||
lat=latitude, lon=longitude, radius=radius_km, tenant_id=tenant_id)
|
||||
|
||||
# Delegate to universal client
|
||||
events = await self.universal_client.get_events(latitude, longitude, radius_km)
|
||||
|
||||
# Add metadata to events
|
||||
for event in events:
|
||||
event['service_metadata'] = {
|
||||
'request_timestamp': datetime.now().isoformat(),
|
||||
'tenant_id': tenant_id,
|
||||
'service_version': '2.0',
|
||||
'query_location': {'latitude': latitude, 'longitude': longitude},
|
||||
'search_radius_km': radius_km
|
||||
}
|
||||
|
||||
logger.info("Retrieved traffic events",
|
||||
lat=latitude, lon=longitude, events=len(events))
|
||||
|
||||
return events
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Error getting traffic events",
|
||||
lat=latitude, lon=longitude, error=str(e))
|
||||
return []
|
||||
|
||||
def get_location_info(self, latitude: float, longitude: float) -> Dict[str, Any]:
|
||||
"""
|
||||
Get information about traffic data availability for location
|
||||
|
||||
Args:
|
||||
latitude: Query location latitude
|
||||
longitude: Query location longitude
|
||||
|
||||
Returns:
|
||||
Dict with location support information
|
||||
"""
|
||||
try:
|
||||
info = self.universal_client.get_location_info(latitude, longitude)
|
||||
|
||||
# Add service layer information
|
||||
info['service_layer'] = {
|
||||
'version': '2.0',
|
||||
'abstraction_level': 'universal',
|
||||
'supported_operations': [
|
||||
'current_traffic',
|
||||
'historical_traffic',
|
||||
'traffic_events',
|
||||
'bulk_requests'
|
||||
]
|
||||
}
|
||||
|
||||
return info
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Error getting location info",
|
||||
lat=latitude, lon=longitude, error=str(e))
|
||||
return {
|
||||
'supported': False,
|
||||
'error': str(e),
|
||||
'service_layer': {'version': '2.0'}
|
||||
}
|
||||
|
||||
async def get_stored_traffic_for_training(self,
|
||||
latitude: float,
|
||||
longitude: float,
|
||||
start_date: datetime,
|
||||
end_date: datetime) -> List[Dict[str, Any]]:
|
||||
"""Retrieve stored traffic data specifically for training purposes"""
|
||||
try:
|
||||
async with self.database_manager.get_session() as session:
|
||||
traffic_repo = TrafficRepository(session)
|
||||
records = await traffic_repo.get_historical_traffic_for_training(
|
||||
latitude, longitude, start_date, end_date
|
||||
)
|
||||
|
||||
# Convert to training format
|
||||
training_data = []
|
||||
for record in records:
|
||||
training_data.append({
|
||||
'date': record.date,
|
||||
'traffic_volume': record.traffic_volume,
|
||||
'pedestrian_count': record.pedestrian_count,
|
||||
'congestion_level': record.congestion_level,
|
||||
'average_speed': record.average_speed,
|
||||
'location_id': record.location_id,
|
||||
'source': record.source,
|
||||
'measurement_point_id': record.raw_data # Contains additional metadata
|
||||
})
|
||||
|
||||
logger.info(f"Retrieved {len(training_data)} traffic records for training",
|
||||
location_id=f"{latitude:.4f},{longitude:.4f}", start=start_date, end=end_date)
|
||||
|
||||
return training_data
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Failed to retrieve traffic data for training",
|
||||
error=str(e), location_id=f"{latitude:.4f},{longitude:.4f}")
|
||||
return []
|
||||
154
services/external/app/services/weather_service.py
vendored
Normal file
154
services/external/app/services/weather_service.py
vendored
Normal file
@@ -0,0 +1,154 @@
|
||||
# services/data/app/services/weather_service.py - REVISED VERSION
|
||||
|
||||
"""Weather data service with repository pattern"""
|
||||
|
||||
from typing import List, Dict, Any, Optional
|
||||
from datetime import datetime, timedelta
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
import structlog
|
||||
|
||||
from app.models.weather import WeatherData, WeatherForecast
|
||||
from app.external.aemet import AEMETClient
|
||||
from app.schemas.weather import WeatherDataResponse, WeatherForecastResponse
|
||||
from app.repositories.weather_repository import WeatherRepository
|
||||
|
||||
logger = structlog.get_logger()
|
||||
from app.core.database import database_manager
|
||||
|
||||
class WeatherService:
|
||||
|
||||
def __init__(self):
|
||||
self.aemet_client = AEMETClient()
|
||||
self.database_manager = database_manager
|
||||
|
||||
async def get_current_weather(self, latitude: float, longitude: float) -> Optional[WeatherDataResponse]:
|
||||
"""Get current weather for location"""
|
||||
try:
|
||||
logger.debug("Getting current weather", lat=latitude, lon=longitude)
|
||||
weather_data = await self.aemet_client.get_current_weather(latitude, longitude)
|
||||
|
||||
if weather_data:
|
||||
logger.debug("Weather data received", source=weather_data.get('source'))
|
||||
return WeatherDataResponse(**weather_data)
|
||||
else:
|
||||
logger.warning("No weather data received from AEMET client")
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Failed to get current weather", error=str(e), lat=latitude, lon=longitude)
|
||||
return None
|
||||
|
||||
async def get_weather_forecast(self, latitude: float, longitude: float, days: int = 7) -> List[WeatherForecastResponse]:
|
||||
"""Get weather forecast for location"""
|
||||
try:
|
||||
logger.debug("Getting weather forecast", lat=latitude, lon=longitude, days=days)
|
||||
forecast_data = await self.aemet_client.get_forecast(latitude, longitude, days)
|
||||
|
||||
if forecast_data:
|
||||
logger.debug("Forecast data received", count=len(forecast_data))
|
||||
# Validate each forecast item before creating response
|
||||
valid_forecasts = []
|
||||
for item in forecast_data:
|
||||
try:
|
||||
if isinstance(item, dict):
|
||||
# Ensure required fields are present
|
||||
forecast_item = {
|
||||
"forecast_date": item.get("forecast_date", datetime.now()),
|
||||
"generated_at": item.get("generated_at", datetime.now()),
|
||||
"temperature": float(item.get("temperature", 15.0)),
|
||||
"precipitation": float(item.get("precipitation", 0.0)),
|
||||
"humidity": float(item.get("humidity", 50.0)),
|
||||
"wind_speed": float(item.get("wind_speed", 10.0)),
|
||||
"description": str(item.get("description", "Variable")),
|
||||
"source": str(item.get("source", "unknown"))
|
||||
}
|
||||
valid_forecasts.append(WeatherForecastResponse(**forecast_item))
|
||||
else:
|
||||
logger.warning("Invalid forecast item type", item_type=type(item))
|
||||
except Exception as item_error:
|
||||
logger.warning("Error processing forecast item", error=str(item_error), item=item)
|
||||
continue
|
||||
|
||||
logger.debug("Valid forecasts processed", count=len(valid_forecasts))
|
||||
return valid_forecasts
|
||||
else:
|
||||
logger.warning("No forecast data received from AEMET client")
|
||||
return []
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Failed to get weather forecast", error=str(e), lat=latitude, lon=longitude)
|
||||
return []
|
||||
|
||||
async def get_historical_weather(self,
|
||||
latitude: float,
|
||||
longitude: float,
|
||||
start_date: datetime,
|
||||
end_date: datetime) -> List[WeatherDataResponse]:
|
||||
"""Get historical weather data"""
|
||||
try:
|
||||
logger.debug("Getting historical weather",
|
||||
lat=latitude, lon=longitude,
|
||||
start=start_date, end=end_date)
|
||||
|
||||
location_id = f"{latitude:.4f},{longitude:.4f}"
|
||||
|
||||
async with self.database_manager.get_session() as session:
|
||||
weather_repository = WeatherRepository(session)
|
||||
# Use the repository to get data from the database
|
||||
db_records = await weather_repository.get_historical_weather(
|
||||
location_id,
|
||||
start_date,
|
||||
end_date
|
||||
)
|
||||
|
||||
if db_records:
|
||||
logger.debug("Historical data found in database", count=len(db_records))
|
||||
return [WeatherDataResponse(
|
||||
date=record.date,
|
||||
temperature=record.temperature,
|
||||
precipitation=record.precipitation,
|
||||
humidity=record.humidity,
|
||||
wind_speed=record.wind_speed,
|
||||
pressure=record.pressure,
|
||||
description=record.description,
|
||||
source=record.source
|
||||
) for record in db_records]
|
||||
|
||||
# If not in database, fetch from API and store
|
||||
logger.debug("Fetching historical data from AEMET API")
|
||||
weather_data = await self.aemet_client.get_historical_weather(
|
||||
latitude, longitude, start_date, end_date
|
||||
)
|
||||
|
||||
if weather_data:
|
||||
# Use the repository to store the new data
|
||||
records_to_store = [{
|
||||
"location_id": location_id,
|
||||
"city": "Madrid", # Default city for AEMET data
|
||||
"date": data.get('date', datetime.now()),
|
||||
"temperature": data.get('temperature'),
|
||||
"precipitation": data.get('precipitation'),
|
||||
"humidity": data.get('humidity'),
|
||||
"wind_speed": data.get('wind_speed'),
|
||||
"pressure": data.get('pressure'),
|
||||
"description": data.get('description'),
|
||||
"source": "aemet",
|
||||
"data_type": "historical",
|
||||
"raw_data": data, # Pass as dict, not string
|
||||
"tenant_id": None
|
||||
} for data in weather_data]
|
||||
|
||||
async with self.database_manager.get_session() as session:
|
||||
weather_repository = WeatherRepository(session)
|
||||
await weather_repository.bulk_create_weather_data(records_to_store)
|
||||
|
||||
logger.debug("Historical data stored in database", count=len(weather_data))
|
||||
|
||||
return [WeatherDataResponse(**item) for item in weather_data]
|
||||
else:
|
||||
logger.warning("No historical weather data received")
|
||||
return []
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Failed to get historical weather", error=str(e))
|
||||
return []
|
||||
Reference in New Issue
Block a user