Add POI feature and imporve the overall backend implementation
This commit is contained in:
Urtzi Alfaro
@@ -98,32 +98,53 @@ class MadridAdapter(CityDataAdapter):
|
||||
async def validate_connection(self) -> bool:
|
||||
"""Validate connection to AEMET and Madrid OpenData
|
||||
|
||||
Note: Validation is lenient - passes if traffic API works.
|
||||
Note: Validation is lenient - allows partial failures for temporary API issues.
|
||||
AEMET rate limits may cause weather validation to fail during initialization.
|
||||
Madrid traffic API outages should not block validation entirely.
|
||||
"""
|
||||
try:
|
||||
test_traffic = await self.traffic_client.get_current_traffic(
|
||||
self.madrid_lat,
|
||||
self.madrid_lon
|
||||
)
|
||||
traffic_validation_passed = False
|
||||
weather_validation_passed = False
|
||||
|
||||
# Traffic API must work (critical for operations)
|
||||
if test_traffic is None:
|
||||
logger.error("Traffic API validation failed - this is critical")
|
||||
# Try traffic API first
|
||||
try:
|
||||
test_traffic = await self.traffic_client.get_current_traffic(
|
||||
self.madrid_lat,
|
||||
self.madrid_lon
|
||||
)
|
||||
|
||||
if test_traffic is not None and len(test_traffic) > 0:
|
||||
traffic_validation_passed = True
|
||||
logger.info("Traffic API validation successful")
|
||||
else:
|
||||
logger.warning("Traffic API validation failed - temporary unavailability (proceeding anyway)")
|
||||
except Exception as traffic_error:
|
||||
logger.warning("Traffic API validation error (temporary unavailability) - proceeding anyway", error=str(traffic_error))
|
||||
|
||||
# Try weather API
|
||||
try:
|
||||
test_weather = await self.aemet_client.get_current_weather(
|
||||
self.madrid_lat,
|
||||
self.madrid_lon
|
||||
)
|
||||
|
||||
if test_weather is not None:
|
||||
weather_validation_passed = True
|
||||
logger.info("Weather API validation successful")
|
||||
else:
|
||||
logger.warning("Weather API validation failed (likely rate limited) - proceeding anyway")
|
||||
except Exception as weather_error:
|
||||
logger.warning("Weather API validation error - proceeding anyway", error=str(weather_error))
|
||||
|
||||
# At least one validation should pass for basic connectivity
|
||||
if not traffic_validation_passed and not weather_validation_passed:
|
||||
logger.error("Both traffic and weather API validations failed - no connectivity")
|
||||
return False
|
||||
|
||||
# Try weather API, but don't fail validation if rate limited
|
||||
test_weather = await self.aemet_client.get_current_weather(
|
||||
self.madrid_lat,
|
||||
self.madrid_lon
|
||||
)
|
||||
|
||||
if test_weather is None:
|
||||
logger.warning("Weather API validation failed (likely rate limited) - proceeding anyway")
|
||||
else:
|
||||
logger.info("Weather API validation successful")
|
||||
|
||||
# Pass validation if traffic works (weather can be fetched later)
|
||||
# Return success if at least one API is accessible
|
||||
logger.info("Adapter connection validation passed",
|
||||
traffic_valid=traffic_validation_passed,
|
||||
weather_valid=weather_validation_passed)
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
|
||||
@@ -58,7 +58,17 @@ class DataIngestionManager:
|
||||
failures=failures
|
||||
)
|
||||
|
||||
return successes == len(results)
|
||||
# Consider success if we have at least some cities initialized (majority success)
|
||||
# This allows the system to continue even if some external APIs are temporarily unavailable
|
||||
if successes > 0:
|
||||
logger.info(
|
||||
"Partial success achieved - continuing with available data",
|
||||
success_ratio=f"{successes}/{len(results)}"
|
||||
)
|
||||
return True
|
||||
else:
|
||||
logger.error("All city initializations failed - system cannot proceed")
|
||||
return False
|
||||
|
||||
async def initialize_city(
|
||||
self,
|
||||
@@ -123,23 +133,55 @@ class DataIngestionManager:
|
||||
logger.error("Adapter validation failed", city=city.name)
|
||||
return False
|
||||
|
||||
weather_data = await adapter.fetch_historical_weather(
|
||||
start_date, end_date
|
||||
)
|
||||
# Fetch data with error handling to allow partial success
|
||||
weather_data = []
|
||||
traffic_data = []
|
||||
|
||||
# Fetch weather data
|
||||
try:
|
||||
weather_data = await adapter.fetch_historical_weather(
|
||||
start_date, end_date
|
||||
)
|
||||
logger.info("Weather data fetched successfully",
|
||||
records=len(weather_data), city=city.name)
|
||||
except Exception as weather_error:
|
||||
logger.error("Failed to fetch weather data",
|
||||
city=city.name, error=str(weather_error))
|
||||
# Don't return False here - continue with whatever data we can get
|
||||
|
||||
traffic_data = await adapter.fetch_historical_traffic(
|
||||
start_date, end_date
|
||||
)
|
||||
# Fetch traffic data
|
||||
try:
|
||||
traffic_data = await adapter.fetch_historical_traffic(
|
||||
start_date, end_date
|
||||
)
|
||||
logger.info("Traffic data fetched successfully",
|
||||
records=len(traffic_data), city=city.name)
|
||||
except Exception as traffic_error:
|
||||
logger.error("Failed to fetch traffic data",
|
||||
city=city.name, error=str(traffic_error))
|
||||
# Don't return False here - continue with weather data only if available
|
||||
|
||||
# Store available data (at least one type should be available for partial success)
|
||||
async with self.database_manager.get_session() as session:
|
||||
repo = CityDataRepository(session)
|
||||
|
||||
weather_stored = await repo.bulk_store_weather(
|
||||
city_id, weather_data
|
||||
)
|
||||
traffic_stored = await repo.bulk_store_traffic(
|
||||
city_id, traffic_data
|
||||
)
|
||||
weather_stored = 0
|
||||
traffic_stored = 0
|
||||
|
||||
if weather_data:
|
||||
weather_stored = await repo.bulk_store_weather(
|
||||
city_id, weather_data
|
||||
)
|
||||
|
||||
if traffic_data:
|
||||
traffic_stored = await repo.bulk_store_traffic(
|
||||
city_id, traffic_data
|
||||
)
|
||||
|
||||
# Only fail if both data types failed to fetch
|
||||
if not weather_data and not traffic_data:
|
||||
logger.error("Both weather and traffic data fetch failed", city=city.name)
|
||||
return False
|
||||
|
||||
logger.info(
|
||||
"City initialization complete",
|
||||
|
||||
Reference in New Issue
Block a user