Initial commit - production deployment

services/external/app/ingestion/adapters/__init__.py

@@ -0,0 +1,20 @@
+# services/external/app/ingestion/adapters/__init__.py
+"""
+Adapter registry - Maps city IDs to adapter implementations
+"""
+
+from typing import Dict, Type
+from ..base_adapter import CityDataAdapter
+from .madrid_adapter import MadridAdapter
+
+ADAPTER_REGISTRY: Dict[str, Type[CityDataAdapter]] = {
+    "madrid": MadridAdapter,
+}
+
+
+def get_adapter(city_id: str, config: Dict) -> CityDataAdapter:
+    """Factory to instantiate appropriate adapter"""
+    adapter_class = ADAPTER_REGISTRY.get(city_id)
+    if not adapter_class:
+        raise ValueError(f"No adapter registered for city: {city_id}")
+    return adapter_class(city_id, config)

services/external/app/ingestion/adapters/madrid_adapter.py

@@ -0,0 +1,152 @@
+# services/external/app/ingestion/adapters/madrid_adapter.py
+"""
+Madrid city data adapter - Uses existing AEMET and Madrid OpenData clients
+"""
+
+from typing import List, Dict, Any
+from datetime import datetime
+import structlog
+
+from ..base_adapter import CityDataAdapter
+from app.external.aemet import AEMETClient
+from app.external.apis.madrid_traffic_client import MadridTrafficClient
+
+logger = structlog.get_logger()
+
+
+class MadridAdapter(CityDataAdapter):
+    """Adapter for Madrid using AEMET + Madrid OpenData"""
+
+    def __init__(self, city_id: str, config: Dict[str, Any]):
+        super().__init__(city_id, config)
+        self.aemet_client = AEMETClient()
+        self.traffic_client = MadridTrafficClient()
+
+        self.madrid_lat = 40.4168
+        self.madrid_lon = -3.7038
+
+    async def fetch_historical_weather(
+        self,
+        start_date: datetime,
+        end_date: datetime
+    ) -> List[Dict[str, Any]]:
+        """Fetch historical weather from AEMET"""
+        try:
+            logger.info(
+                "Fetching Madrid historical weather",
+                start=start_date.isoformat(),
+                end=end_date.isoformat()
+            )
+
+            weather_data = await self.aemet_client.get_historical_weather(
+                self.madrid_lat,
+                self.madrid_lon,
+                start_date,
+                end_date
+            )
+
+            for record in weather_data:
+                record['city_id'] = self.city_id
+                record['city_name'] = 'Madrid'
+
+            logger.info(
+                "Madrid weather data fetched",
+                records=len(weather_data)
+            )
+
+            return weather_data
+
+        except Exception as e:
+            logger.error("Error fetching Madrid weather", error=str(e))
+            return []
+
+    async def fetch_historical_traffic(
+        self,
+        start_date: datetime,
+        end_date: datetime
+    ) -> List[Dict[str, Any]]:
+        """Fetch historical traffic from Madrid OpenData"""
+        try:
+            logger.info(
+                "Fetching Madrid historical traffic",
+                start=start_date.isoformat(),
+                end=end_date.isoformat()
+            )
+
+            traffic_data = await self.traffic_client.get_historical_traffic(
+                self.madrid_lat,
+                self.madrid_lon,
+                start_date,
+                end_date
+            )
+
+            for record in traffic_data:
+                record['city_id'] = self.city_id
+                record['city_name'] = 'Madrid'
+
+            logger.info(
+                "Madrid traffic data fetched",
+                records=len(traffic_data)
+            )
+
+            return traffic_data
+
+        except Exception as e:
+            logger.error("Error fetching Madrid traffic", error=str(e))
+            return []
+
+    async def validate_connection(self) -> bool:
+        """Validate connection to AEMET and Madrid OpenData
+
+        Note: Validation is lenient - allows partial failures for temporary API issues.
+        AEMET rate limits may cause weather validation to fail during initialization.
+        Madrid traffic API outages should not block validation entirely.
+        """
+        try:
+            traffic_validation_passed = False
+            weather_validation_passed = False
+
+            # Try traffic API first
+            try:
+                test_traffic = await self.traffic_client.get_current_traffic(
+                    self.madrid_lat,
+                    self.madrid_lon
+                )
+
+                if test_traffic is not None and len(test_traffic) > 0:
+                    traffic_validation_passed = True
+                    logger.info("Traffic API validation successful")
+                else:
+                    logger.warning("Traffic API validation failed - temporary unavailability (proceeding anyway)")
+            except Exception as traffic_error:
+                logger.warning("Traffic API validation error (temporary unavailability) - proceeding anyway", error=str(traffic_error))
+
+            # Try weather API
+            try:
+                test_weather = await self.aemet_client.get_current_weather(
+                    self.madrid_lat,
+                    self.madrid_lon
+                )
+
+                if test_weather is not None:
+                    weather_validation_passed = True
+                    logger.info("Weather API validation successful")
+                else:
+                    logger.warning("Weather API validation failed (likely rate limited) - proceeding anyway")
+            except Exception as weather_error:
+                logger.warning("Weather API validation error - proceeding anyway", error=str(weather_error))
+
+            # At least one validation should pass for basic connectivity
+            if not traffic_validation_passed and not weather_validation_passed:
+                logger.error("Both traffic and weather API validations failed - no connectivity")
+                return False
+
+            # Return success if at least one API is accessible
+            logger.info("Adapter connection validation passed", 
+                       traffic_valid=traffic_validation_passed,
+                       weather_valid=weather_validation_passed)
+            return True
+
+        except Exception as e:
+            logger.error("Madrid adapter connection validation failed", error=str(e))
+            return False