diff --git a/services/data/app/api/weather.py b/services/data/app/api/weather.py index aa5621b8..5e53567f 100644 --- a/services/data/app/api/weather.py +++ b/services/data/app/api/weather.py @@ -114,8 +114,7 @@ async def get_weather_history( end_date: date = Query(..., description="End date"), latitude: float = Query(..., description="Latitude"), longitude: float = Query(..., description="Longitude"), - tenant_id: str = Depends(get_current_tenant_id_dep), - current_user: Dict[str, Any] = Depends(get_current_user_dep), + tenant_id: str = Depends(get_current_tenant_id_dep) ): """Get historical weather data""" try: diff --git a/services/data/app/external/aemet.py b/services/data/app/external/aemet.py index bf1242eb..b42280ee 100644 --- a/services/data/app/external/aemet.py +++ b/services/data/app/external/aemet.py @@ -1,11 +1,13 @@ # ================================================================ -# services/data/app/external/aemet.py - FIXED VERSION +# services/data/app/external/aemet.py - REFACTORED VERSION # ================================================================ -"""AEMET (Spanish Weather Service) API client - FIXED FORECAST PARSING""" +"""AEMET (Spanish Weather Service) API client with improved modularity""" import math -from typing import List, Dict, Any, Optional +from typing import List, Dict, Any, Optional, Tuple from datetime import datetime, timedelta +from dataclasses import dataclass +from enum import Enum import structlog from app.external.base_client import BaseAPIClient @@ -13,304 +15,79 @@ from app.core.config import settings logger = structlog.get_logger() -class AEMETClient(BaseAPIClient): + +class WeatherSource(Enum): + """Weather data source types""" + AEMET = "aemet" + SYNTHETIC = "synthetic" + DEFAULT = "default" + + +@dataclass +class WeatherStation: + """Weather station data""" + id: str + name: str + latitude: float + longitude: float + + +@dataclass +class GeographicBounds: + """Geographic boundary definition""" + min_lat: float + max_lat: float + min_lon: float + max_lon: float - def __init__(self): - super().__init__( - base_url="https://opendata.aemet.es/opendata/api", - api_key=settings.AEMET_API_KEY - ) + def contains(self, latitude: float, longitude: float) -> bool: + """Check if coordinates are within bounds""" + return (self.min_lat <= latitude <= self.max_lat and + self.min_lon <= longitude <= self.max_lon) + + +class AEMETConstants: + """AEMET API constants and configuration""" - async def get_current_weather(self, latitude: float, longitude: float) -> Optional[Dict[str, Any]]: - """Get current weather for coordinates""" + # API Configuration + MAX_DAYS_PER_REQUEST = 30 + MADRID_MUNICIPALITY_CODE = "28079" + + # Madrid geographic bounds + MADRID_BOUNDS = GeographicBounds( + min_lat=40.3, max_lat=40.6, + min_lon=-3.9, max_lon=-3.5 + ) + + # Weather stations in Madrid area + MADRID_STATIONS = [ + WeatherStation("3195", "Madrid Centro", 40.4117, -3.6780), + WeatherStation("3129", "Madrid Norte", 40.4677, -3.5552), + WeatherStation("3197", "Madrid Sur", 40.2987, -3.7216), + ] + + # Climate simulation parameters + BASE_TEMPERATURE_SEASONAL = 5.0 + TEMPERATURE_SEASONAL_MULTIPLIER = 2.5 + DAILY_TEMPERATURE_AMPLITUDE = 8.0 + EARTH_RADIUS_KM = 6371.0 + + +class WeatherDataParser: + """Handles parsing of different weather data formats""" + + @staticmethod + def safe_float(value: Any, default: Optional[float] = None) -> Optional[float]: + """Safely convert value to float with fallback""" try: - # Find nearest station - station_id = await self._get_nearest_station(latitude, longitude) - if not station_id: - logger.warning("No weather station found", lat=latitude, lon=longitude) - return await self._generate_synthetic_weather() - - # AEMET API STEP 1: Get the datos URL - endpoint = f"/observacion/convencional/datos/estacion/{station_id}" - initial_response = await self._get(endpoint) - - # CRITICAL FIX: Handle AEMET's two-step API response - if not initial_response or not isinstance(initial_response, dict): - logger.info("Invalid initial response from AEMET API", response_type=type(initial_response)) - return await self._generate_synthetic_weather() - - # Check if we got a successful response with datos URL - datos_url = initial_response.get("datos") - if not datos_url or not isinstance(datos_url, str): - logger.info("No datos URL in AEMET response", response=initial_response) - return await self._generate_synthetic_weather() - - # AEMET API STEP 2: Fetch actual data from the datos URL - actual_weather_data = await self._fetch_from_url(datos_url) - - if actual_weather_data and isinstance(actual_weather_data, list) and len(actual_weather_data) > 0: - # Parse the first station's data - weather_data = actual_weather_data[0] - if isinstance(weather_data, dict): - return self._parse_weather_data(weather_data) - - # Fallback to synthetic data - logger.info("Falling back to synthetic weather data", reason="invalid_weather_data") - return await self._generate_synthetic_weather() - - except Exception as e: - logger.error("Failed to get current weather", error=str(e)) - return await self._generate_synthetic_weather() + if value is None: + return default + return float(value) + except (ValueError, TypeError): + return default - async def get_forecast(self, latitude: float, longitude: float, days: int = 7) -> List[Dict[str, Any]]: - """Get weather forecast for coordinates""" - try: - # Get municipality code for location - municipality_code = await self._get_municipality_code(latitude, longitude) - if not municipality_code: - logger.info("No municipality code found, using synthetic data") - return await self._generate_synthetic_forecast(days) - - # AEMET API STEP 1: Get the datos URL - endpoint = f"/prediccion/especifica/municipio/diaria/{municipality_code}" - initial_response = await self._get(endpoint) - - # CRITICAL FIX: Handle AEMET's two-step API response - if not initial_response or not isinstance(initial_response, dict): - logger.info("Invalid initial response from AEMET forecast API", response_type=type(initial_response)) - return await self._generate_synthetic_forecast(days) - - # Check if we got a successful response with datos URL - datos_url = initial_response.get("datos") - if not datos_url or not isinstance(datos_url, str): - logger.info("No datos URL in AEMET forecast response", response=initial_response) - return await self._generate_synthetic_forecast(days) - - # AEMET API STEP 2: Fetch actual data from the datos URL - actual_forecast_data = await self._fetch_from_url(datos_url) - - if actual_forecast_data and isinstance(actual_forecast_data, list): - return self._parse_forecast_data(actual_forecast_data, days) - - # Fallback to synthetic data - logger.info("Falling back to synthetic forecast data", reason="invalid_forecast_data") - return await self._generate_synthetic_forecast(days) - - except Exception as e: - logger.error("Failed to get weather forecast", error=str(e)) - return await self._generate_synthetic_forecast(days) - - async def _fetch_from_url(self, url: str) -> Optional[List[Dict[str, Any]]]: - """Fetch data from AEMET datos URL""" - try: - # Use base client to fetch from the provided URL directly - data = await self._fetch_url_directly(url) - - if data and isinstance(data, list): - return data - else: - logger.warning("Expected list from datos URL", data_type=type(data)) - return None - - except Exception as e: - logger.error("Failed to fetch from datos URL", url=url, error=str(e)) - return None - - async def get_historical_weather(self, - latitude: float, - longitude: float, - start_date: datetime, - end_date: datetime) -> List[Dict[str, Any]]: - """Get historical weather data""" - try: - # For now, generate synthetic historical data - # In production, this would use AEMET historical data API with proper two-step flow - return await self._generate_synthetic_historical(start_date, end_date) - - except Exception as e: - logger.error("Failed to get historical weather", error=str(e)) - return [] - - async def _get_nearest_station(self, latitude: float, longitude: float) -> Optional[str]: - """Find nearest weather station""" - try: - # Madrid area stations (simplified) - madrid_stations = { - "3195": {"lat": 40.4117, "lon": -3.6780, "name": "Madrid Centro"}, - "3129": {"lat": 40.4677, "lon": -3.5552, "name": "Madrid Norte"}, - "3197": {"lat": 40.2987, "lon": -3.7216, "name": "Madrid Sur"} - } - - closest_station = None - min_distance = float('inf') - - for station_id, station_data in madrid_stations.items(): - distance = self._calculate_distance( - latitude, longitude, - station_data["lat"], station_data["lon"] - ) - if distance < min_distance: - min_distance = distance - closest_station = station_id - - return closest_station - - except Exception as e: - logger.error("Failed to find nearest station", error=str(e)) - return None - - async def _get_municipality_code(self, latitude: float, longitude: float) -> Optional[str]: - """Get municipality code for coordinates""" - # Madrid municipality code - if self._is_in_madrid_area(latitude, longitude): - return "28079" # Madrid municipality code - return None - - def _is_in_madrid_area(self, latitude: float, longitude: float) -> bool: - """Check if coordinates are in Madrid area""" - # Madrid approximate bounds - return (40.3 <= latitude <= 40.6) and (-3.9 <= longitude <= -3.5) - - def _calculate_distance(self, lat1: float, lon1: float, lat2: float, lon2: float) -> float: - """Calculate distance between two coordinates in km""" - R = 6371 # Earth's radius in km - - dlat = math.radians(lat2 - lat1) - dlon = math.radians(lon2 - lon1) - - a = (math.sin(dlat/2) * math.sin(dlat/2) + - math.cos(math.radians(lat1)) * math.cos(math.radians(lat2)) * - math.sin(dlon/2) * math.sin(dlon/2)) - - c = 2 * math.atan2(math.sqrt(a), math.sqrt(1-a)) - distance = R * c - - return distance - - def _parse_weather_data(self, data: Dict) -> Dict[str, Any]: - """Parse AEMET weather data format""" - if not isinstance(data, dict): - logger.warning("Weather data is not a dictionary", data_type=type(data)) - return self._get_default_weather_data() - - try: - return { - "date": datetime.now(), - "temperature": self._safe_float(data.get("ta"), 15.0), # Temperature - "precipitation": self._safe_float(data.get("prec"), 0.0), # Precipitation - "humidity": self._safe_float(data.get("hr"), 50.0), # Humidity - "wind_speed": self._safe_float(data.get("vv"), 10.0), # Wind speed - "pressure": self._safe_float(data.get("pres"), 1013.0), # Pressure - "description": str(data.get("descripcion", "Partly cloudy")), - "source": "aemet" - } - except Exception as e: - logger.error("Error parsing weather data", error=str(e), data=data) - return self._get_default_weather_data() - - def _parse_forecast_data(self, data: List, days: int) -> List[Dict[str, Any]]: - """Parse AEMET forecast data - FIXED VERSION""" - forecast = [] - base_date = datetime.now().date() - - if not isinstance(data, list): - logger.warning("Forecast data is not a list", data_type=type(data)) - return [] - - try: - # AEMET forecast structure is complex - parse what we can and fill gaps with synthetic data - logger.debug("Processing AEMET forecast data", data_length=len(data)) - - # If we have actual AEMET data, try to parse it - if len(data) > 0 and isinstance(data[0], dict): - aemet_data = data[0] - logger.debug("AEMET forecast keys", keys=list(aemet_data.keys()) if isinstance(aemet_data, dict) else "not_dict") - - # Try to extract daily forecasts from AEMET structure - dias = aemet_data.get('prediccion', {}).get('dia', []) if isinstance(aemet_data, dict) else [] - - if isinstance(dias, list) and len(dias) > 0: - # Parse AEMET daily forecast format - for i, dia in enumerate(dias[:days]): - if not isinstance(dia, dict): - continue - - forecast_date = base_date + timedelta(days=i) - - # Extract temperature data (AEMET has complex temp structure) - temp_data = dia.get('temperatura', {}) - if isinstance(temp_data, dict): - temp_max = self._extract_temp_value(temp_data.get('maxima')) - temp_min = self._extract_temp_value(temp_data.get('minima')) - avg_temp = (temp_max + temp_min) / 2 if temp_max and temp_min else 15.0 - else: - avg_temp = 15.0 - - # Extract precipitation probability - precip_data = dia.get('probPrecipitacion', []) - precip_prob = 0.0 - if isinstance(precip_data, list) and len(precip_data) > 0: - for precip_item in precip_data: - if isinstance(precip_item, dict) and 'value' in precip_item: - precip_prob = max(precip_prob, self._safe_float(precip_item.get('value'), 0.0)) - - # Extract wind data - viento_data = dia.get('viento', []) - wind_speed = 10.0 - if isinstance(viento_data, list) and len(viento_data) > 0: - for viento_item in viento_data: - if isinstance(viento_item, dict) and 'velocidad' in viento_item: - speed_values = viento_item.get('velocidad', []) - if isinstance(speed_values, list) and len(speed_values) > 0: - wind_speed = self._safe_float(speed_values[0], 10.0) - break - - # Generate description based on precipitation probability - if precip_prob > 70: - description = "Lluvioso" - elif precip_prob > 30: - description = "Parcialmente nublado" - else: - description = "Soleado" - - forecast.append({ - "forecast_date": datetime.combine(forecast_date, datetime.min.time()), - "generated_at": datetime.now(), - "temperature": round(avg_temp, 1), - "precipitation": precip_prob / 10, # Convert percentage to mm estimate - "humidity": 50.0 + (i % 20), # Estimate - "wind_speed": round(wind_speed, 1), - "description": description, - "source": "aemet" - }) - - logger.debug("Parsed forecast day", day=i, temp=avg_temp, precip=precip_prob) - - # If we successfully parsed some days, fill remaining with synthetic - remaining_days = days - len(forecast) - if remaining_days > 0: - synthetic_forecast = self._generate_synthetic_forecast_sync(remaining_days, len(forecast)) - forecast.extend(synthetic_forecast) - - # If no valid AEMET data was parsed, use synthetic - if len(forecast) == 0: - logger.info("No valid AEMET forecast data found, using synthetic") - forecast = self._generate_synthetic_forecast_sync(days, 0) - - except Exception as e: - logger.error("Error parsing AEMET forecast data", error=str(e)) - # Fallback to synthetic forecast - forecast = self._generate_synthetic_forecast_sync(days, 0) - - # Ensure we always return the requested number of days - if len(forecast) < days: - remaining = days - len(forecast) - synthetic_remaining = self._generate_synthetic_forecast_sync(remaining, len(forecast)) - forecast.extend(synthetic_remaining) - - return forecast[:days] # Ensure we don't exceed requested days - - def _extract_temp_value(self, temp_data) -> Optional[float]: + @staticmethod + def extract_temperature_value(temp_data: Any) -> Optional[float]: """Extract temperature value from AEMET complex temperature structure""" if temp_data is None: return None @@ -325,23 +102,237 @@ class AEMETClient(BaseAPIClient): return None if isinstance(temp_data, dict) and 'valor' in temp_data: - return self._safe_float(temp_data['valor'], None) + return WeatherDataParser.safe_float(temp_data['valor']) if isinstance(temp_data, list) and len(temp_data) > 0: first_item = temp_data[0] if isinstance(first_item, dict) and 'valor' in first_item: - return self._safe_float(first_item['valor'], None) + return WeatherDataParser.safe_float(first_item['valor']) return None - def _safe_float(self, value: Any, default: float) -> float: - """Safely convert value to float with fallback""" + @staticmethod + def generate_weather_description(temperature: Optional[float], + precipitation: Optional[float], + humidity: Optional[float]) -> str: + """Generate weather description based on conditions""" + if precipitation and precipitation > 5.0: + return "Lluvioso" + elif precipitation and precipitation > 0.1: + return "Nuboso con lluvia" + elif humidity and humidity > 80: + return "Nuboso" + elif temperature and temperature > 25: + return "Soleado y cálido" + elif temperature and temperature < 5: + return "Frío" + else: + return "Variable" + + def parse_current_weather(self, data: Dict[str, Any]) -> Dict[str, Any]: + """Parse AEMET current weather data format""" + if not isinstance(data, dict): + logger.warning("Weather data is not a dictionary", data_type=type(data)) + return self._get_default_weather_data() + try: - if value is None: - return default - return float(value) - except (ValueError, TypeError): - return default + return { + "date": datetime.now(), + "temperature": self.safe_float(data.get("ta"), 15.0), + "precipitation": self.safe_float(data.get("prec"), 0.0), + "humidity": self.safe_float(data.get("hr"), 50.0), + "wind_speed": self.safe_float(data.get("vv"), 10.0), + "pressure": self.safe_float(data.get("pres"), 1013.0), + "description": str(data.get("descripcion", "Partly cloudy")), + "source": WeatherSource.AEMET.value + } + except Exception as e: + logger.error("Error parsing weather data", error=str(e), data=data) + return self._get_default_weather_data() + + def parse_historical_data(self, data: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """Parse AEMET historical weather data""" + parsed_data = [] + + try: + for record in data: + if not isinstance(record, dict): + continue + + parsed_record = self._parse_single_historical_record(record) + if parsed_record: + parsed_data.append(parsed_record) + + except Exception as e: + logger.error("Error parsing historical weather data", error=str(e)) + + return parsed_data + + def parse_forecast_data(self, data: List[Dict[str, Any]], days: int) -> List[Dict[str, Any]]: + """Parse AEMET forecast data""" + forecast = [] + base_date = datetime.now().date() + + if not isinstance(data, list): + logger.warning("Forecast data is not a list", data_type=type(data)) + return [] + + try: + if len(data) > 0 and isinstance(data[0], dict): + aemet_data = data[0] + dias = aemet_data.get('prediccion', {}).get('dia', []) + + if isinstance(dias, list) and len(dias) > 0: + forecast = self._parse_forecast_days(dias, days, base_date) + + # Fill remaining days with synthetic data if needed + forecast = self._ensure_forecast_completeness(forecast, days) + + except Exception as e: + logger.error("Error parsing AEMET forecast data", error=str(e)) + forecast = [] + + return forecast + + def _parse_single_historical_record(self, record: Dict[str, Any]) -> Optional[Dict[str, Any]]: + """Parse a single historical weather record""" + fecha_str = record.get('fecha') + if not fecha_str: + return None + + try: + record_date = datetime.strptime(fecha_str, '%Y-%m-%d') + except ValueError: + logger.warning("Invalid date format in historical data", fecha=fecha_str) + return None + + # Extract and calculate temperature + temp_max = self.safe_float(record.get('tmax')) + temp_min = self.safe_float(record.get('tmin')) + temperature = self._calculate_average_temperature(temp_max, temp_min) + + # Extract other weather parameters + precipitation = self.safe_float(record.get('prec'), 0.0) + humidity = self.safe_float(record.get('hr')) + wind_speed = self.safe_float(record.get('velmedia')) + pressure = self._extract_pressure(record) + + return { + "date": record_date, + "temperature": temperature, + "precipitation": precipitation, + "humidity": humidity, + "wind_speed": wind_speed, + "pressure": pressure, + "description": self.generate_weather_description(temperature, precipitation, humidity), + "source": WeatherSource.AEMET.value + } + + def _calculate_average_temperature(self, temp_max: Optional[float], temp_min: Optional[float]) -> Optional[float]: + """Calculate average temperature from max and min values""" + if temp_max and temp_min: + return (temp_max + temp_min) / 2 + elif temp_max: + return temp_max - 5 # Estimate average from max + elif temp_min: + return temp_min + 5 # Estimate average from min + return None + + def _extract_pressure(self, record: Dict[str, Any]) -> Optional[float]: + """Extract pressure from historical record""" + pressure = self.safe_float(record.get('presMax')) + if not pressure: + pressure = self.safe_float(record.get('presMin')) + return pressure + + def _parse_forecast_days(self, dias: List[Dict[str, Any]], days: int, base_date: datetime.date) -> List[Dict[str, Any]]: + """Parse forecast days from AEMET data""" + forecast = [] + + for i, dia in enumerate(dias[:days]): + if not isinstance(dia, dict): + continue + + forecast_date = base_date + timedelta(days=i) + forecast_day = self._parse_single_forecast_day(dia, forecast_date, i) + forecast.append(forecast_day) + + return forecast + + def _parse_single_forecast_day(self, dia: Dict[str, Any], forecast_date: datetime.date, day_index: int) -> Dict[str, Any]: + """Parse a single forecast day""" + # Extract temperature + temp_data = dia.get('temperatura', {}) + avg_temp = self._extract_forecast_temperature(temp_data) + + # Extract precipitation probability + precip_prob = self._extract_precipitation_probability(dia.get('probPrecipitacion', [])) + + # Extract wind speed + wind_speed = self._extract_wind_speed(dia.get('viento', [])) + + # Generate description + description = self._generate_forecast_description(precip_prob) + + return { + "forecast_date": datetime.combine(forecast_date, datetime.min.time()), + "generated_at": datetime.now(), + "temperature": round(avg_temp, 1), + "precipitation": precip_prob / 10, # Convert percentage to mm estimate + "humidity": 50.0 + (day_index % 20), # Estimate + "wind_speed": round(wind_speed, 1), + "description": description, + "source": WeatherSource.AEMET.value + } + + def _extract_forecast_temperature(self, temp_data: Dict[str, Any]) -> float: + """Extract temperature from forecast temperature data""" + if isinstance(temp_data, dict): + temp_max = self.extract_temperature_value(temp_data.get('maxima')) + temp_min = self.extract_temperature_value(temp_data.get('minima')) + if temp_max and temp_min: + return (temp_max + temp_min) / 2 + return 15.0 + + def _extract_precipitation_probability(self, precip_data: List[Dict[str, Any]]) -> float: + """Extract precipitation probability from forecast data""" + precip_prob = 0.0 + if isinstance(precip_data, list): + for precip_item in precip_data: + if isinstance(precip_item, dict) and 'value' in precip_item: + precip_prob = max(precip_prob, self.safe_float(precip_item.get('value'), 0.0)) + return precip_prob + + def _extract_wind_speed(self, viento_data: List[Dict[str, Any]]) -> float: + """Extract wind speed from forecast data""" + wind_speed = 10.0 + if isinstance(viento_data, list): + for viento_item in viento_data: + if isinstance(viento_item, dict) and 'velocidad' in viento_item: + speed_values = viento_item.get('velocidad', []) + if isinstance(speed_values, list) and len(speed_values) > 0: + wind_speed = self.safe_float(speed_values[0], 10.0) + break + return wind_speed + + def _generate_forecast_description(self, precip_prob: float) -> str: + """Generate description based on precipitation probability""" + if precip_prob > 70: + return "Lluvioso" + elif precip_prob > 30: + return "Parcialmente nublado" + else: + return "Soleado" + + def _ensure_forecast_completeness(self, forecast: List[Dict[str, Any]], days: int) -> List[Dict[str, Any]]: + """Ensure forecast has the requested number of days""" + if len(forecast) < days: + remaining_days = days - len(forecast) + synthetic_generator = SyntheticWeatherGenerator() + synthetic_forecast = synthetic_generator.generate_forecast_sync(remaining_days, len(forecast)) + forecast.extend(synthetic_forecast) + + return forecast[:days] def _get_default_weather_data(self) -> Dict[str, Any]: """Get default weather data structure""" @@ -353,23 +344,22 @@ class AEMETClient(BaseAPIClient): "wind_speed": 10.0, "pressure": 1013.0, "description": "Data not available", - "source": "default" + "source": WeatherSource.DEFAULT.value } + + +class SyntheticWeatherGenerator: + """Generates realistic synthetic weather data for Madrid""" - async def _generate_synthetic_weather(self) -> Dict[str, Any]: - """Generate realistic synthetic weather for Madrid""" + def generate_current_weather(self) -> Dict[str, Any]: + """Generate realistic synthetic current weather for Madrid""" now = datetime.now() month = now.month hour = now.hour # Madrid climate simulation - base_temp = 5 + (month - 1) * 2.5 # Seasonal variation - temp_variation = math.sin((hour - 6) * math.pi / 12) * 8 # Daily variation - temperature = base_temp + temp_variation - - # Rain probability (higher in winter) - rain_prob = 0.3 if month in [11, 12, 1, 2, 3] else 0.1 - precipitation = 2.5 if hash(now.date()) % 100 < rain_prob * 100 else 0.0 + temperature = self._calculate_current_temperature(month, hour) + precipitation = self._calculate_current_precipitation(now, month) return { "date": now, @@ -379,62 +369,336 @@ class AEMETClient(BaseAPIClient): "wind_speed": 8 + (hour % 12), "pressure": 1013 + math.sin(now.day * 0.2) * 15, "description": "Lluvioso" if precipitation > 0 else "Soleado", - "source": "synthetic" + "source": WeatherSource.SYNTHETIC.value } - def _generate_synthetic_forecast_sync(self, days: int, start_offset: int = 0) -> List[Dict[str, Any]]: + def generate_forecast_sync(self, days: int, start_offset: int = 0) -> List[Dict[str, Any]]: """Generate synthetic forecast data synchronously""" forecast = [] base_date = datetime.now().date() for i in range(days): forecast_date = base_date + timedelta(days=start_offset + i) - - # Seasonal temperature - month = forecast_date.month - base_temp = 5 + (month - 1) * 2.5 - temp_variation = ((start_offset + i) % 7 - 3) * 2 # Weekly variation - - forecast.append({ - "forecast_date": datetime.combine(forecast_date, datetime.min.time()), - "generated_at": datetime.now(), - "temperature": round(base_temp + temp_variation, 1), - "precipitation": 2.0 if (start_offset + i) % 5 == 0 else 0.0, - "humidity": 50 + ((start_offset + i) % 30), - "wind_speed": 10 + ((start_offset + i) % 15), - "description": "Lluvioso" if (start_offset + i) % 5 == 0 else "Soleado", - "source": "synthetic" - }) + forecast_day = self._generate_forecast_day(forecast_date, start_offset + i) + forecast.append(forecast_day) return forecast - async def _generate_synthetic_forecast(self, days: int) -> List[Dict[str, Any]]: + async def generate_forecast(self, days: int) -> List[Dict[str, Any]]: """Generate synthetic forecast data (async version for compatibility)""" - return self._generate_synthetic_forecast_sync(days, 0) + return self.generate_forecast_sync(days, 0) - async def _generate_synthetic_historical(self, start_date: datetime, end_date: datetime) -> List[Dict[str, Any]]: + def generate_historical_data(self, start_date: datetime, end_date: datetime) -> List[Dict[str, Any]]: """Generate synthetic historical weather data""" historical_data = [] current_date = start_date while current_date <= end_date: - month = current_date.month - base_temp = 5 + (month - 1) * 2.5 - - # Add some randomness based on date - temp_variation = math.sin(current_date.day * 0.3) * 5 - - historical_data.append({ - "date": current_date, - "temperature": round(base_temp + temp_variation, 1), - "precipitation": 1.5 if current_date.day % 7 == 0 else 0.0, - "humidity": 45 + (current_date.day % 40), - "wind_speed": 8 + (current_date.day % 20), - "pressure": 1013 + math.sin(current_date.day * 0.2) * 20, - "description": "Variable", - "source": "synthetic" - }) - + historical_day = self._generate_historical_day(current_date) + historical_data.append(historical_day) current_date += timedelta(days=1) - return historical_data \ No newline at end of file + return historical_data + + def _calculate_current_temperature(self, month: int, hour: int) -> float: + """Calculate current temperature based on seasonal and daily patterns""" + base_temp = AEMETConstants.BASE_TEMPERATURE_SEASONAL + (month - 1) * AEMETConstants.TEMPERATURE_SEASONAL_MULTIPLIER + temp_variation = math.sin((hour - 6) * math.pi / 12) * AEMETConstants.DAILY_TEMPERATURE_AMPLITUDE + return base_temp + temp_variation + + def _calculate_current_precipitation(self, now: datetime, month: int) -> float: + """Calculate current precipitation based on seasonal patterns""" + rain_prob = 0.3 if month in [11, 12, 1, 2, 3] else 0.1 + return 2.5 if hash(now.date()) % 100 < rain_prob * 100 else 0.0 + + def _generate_forecast_day(self, forecast_date: datetime.date, day_offset: int) -> Dict[str, Any]: + """Generate a single forecast day""" + month = forecast_date.month + base_temp = AEMETConstants.BASE_TEMPERATURE_SEASONAL + (month - 1) * AEMETConstants.TEMPERATURE_SEASONAL_MULTIPLIER + temp_variation = ((day_offset) % 7 - 3) * 2 # Weekly variation + + return { + "forecast_date": datetime.combine(forecast_date, datetime.min.time()), + "generated_at": datetime.now(), + "temperature": round(base_temp + temp_variation, 1), + "precipitation": 2.0 if day_offset % 5 == 0 else 0.0, + "humidity": 50 + (day_offset % 30), + "wind_speed": 10 + (day_offset % 15), + "description": "Lluvioso" if day_offset % 5 == 0 else "Soleado", + "source": WeatherSource.SYNTHETIC.value + } + + def _generate_historical_day(self, date: datetime) -> Dict[str, Any]: + """Generate a single historical day""" + month = date.month + base_temp = AEMETConstants.BASE_TEMPERATURE_SEASONAL + (month - 1) * AEMETConstants.TEMPERATURE_SEASONAL_MULTIPLIER + temp_variation = math.sin(date.day * 0.3) * 5 + + return { + "date": date, + "temperature": round(base_temp + temp_variation, 1), + "precipitation": 1.5 if date.day % 7 == 0 else 0.0, + "humidity": 45 + (date.day % 40), + "wind_speed": 8 + (date.day % 20), + "pressure": 1013 + math.sin(date.day * 0.2) * 20, + "description": "Variable", + "source": WeatherSource.SYNTHETIC.value + } + + +class LocationService: + """Handles location-related operations""" + + @staticmethod + def find_nearest_station(latitude: float, longitude: float) -> Optional[str]: + """Find nearest weather station to given coordinates""" + try: + # Check if coordinates are reasonable (not extreme values) + if not (-90 <= latitude <= 90 and -180 <= longitude <= 180): + logger.warning("Invalid coordinate range", lat=latitude, lon=longitude) + return None + + # Check if coordinates are too far from Madrid area (more than 1000km away) + madrid_center = (40.4168, -3.7038) + distance_to_madrid = LocationService.calculate_distance( + latitude, longitude, madrid_center[0], madrid_center[1] + ) + + if distance_to_madrid > 1000: # More than 1000km from Madrid + logger.warning("Coordinates too far from Madrid", + lat=latitude, lon=longitude, distance_km=distance_to_madrid) + return None + + closest_station = None + min_distance = float('inf') + + for station in AEMETConstants.MADRID_STATIONS: + distance = LocationService.calculate_distance( + latitude, longitude, station.latitude, station.longitude + ) + if distance < min_distance: + min_distance = distance + closest_station = station.id + + return closest_station + + except Exception as e: + logger.error("Failed to find nearest station", error=str(e)) + return None + + @staticmethod + def get_municipality_code(latitude: float, longitude: float) -> Optional[str]: + """Get municipality code for coordinates""" + if AEMETConstants.MADRID_BOUNDS.contains(latitude, longitude): + return AEMETConstants.MADRID_MUNICIPALITY_CODE + return None + + @staticmethod + def calculate_distance(lat1: float, lon1: float, lat2: float, lon2: float) -> float: + """Calculate distance between two coordinates using Haversine formula""" + dlat = math.radians(lat2 - lat1) + dlon = math.radians(lon2 - lon1) + + a = (math.sin(dlat/2) * math.sin(dlat/2) + + math.cos(math.radians(lat1)) * math.cos(math.radians(lat2)) * + math.sin(dlon/2) * math.sin(dlon/2)) + + c = 2 * math.atan2(math.sqrt(a), math.sqrt(1-a)) + return AEMETConstants.EARTH_RADIUS_KM * c + + +class AEMETClient(BaseAPIClient): + """AEMET (Spanish Weather Service) API client with improved modularity""" + + def __init__(self): + super().__init__( + base_url="https://opendata.aemet.es/opendata/api", + api_key=settings.AEMET_API_KEY + ) + self.parser = WeatherDataParser() + self.synthetic_generator = SyntheticWeatherGenerator() + self.location_service = LocationService() + + async def get_current_weather(self, latitude: float, longitude: float) -> Optional[Dict[str, Any]]: + """Get current weather for coordinates""" + try: + station_id = self.location_service.find_nearest_station(latitude, longitude) + if not station_id: + logger.warning("No weather station found", lat=latitude, lon=longitude) + return await self._get_synthetic_current_weather() + + weather_data = await self._fetch_current_weather_data(station_id) + if weather_data: + return self.parser.parse_current_weather(weather_data) + + logger.info("Falling back to synthetic weather data", reason="invalid_weather_data") + return await self._get_synthetic_current_weather() + + except Exception as e: + logger.error("Failed to get current weather", error=str(e)) + return await self._get_synthetic_current_weather() + + async def get_forecast(self, latitude: float, longitude: float, days: int = 7) -> List[Dict[str, Any]]: + """Get weather forecast for coordinates""" + try: + municipality_code = self.location_service.get_municipality_code(latitude, longitude) + if not municipality_code: + logger.info("No municipality code found, using synthetic data") + return await self.synthetic_generator.generate_forecast(days) + + forecast_data = await self._fetch_forecast_data(municipality_code) + if forecast_data: + parsed_forecast = self.parser.parse_forecast_data(forecast_data, days) + if parsed_forecast: + return parsed_forecast + + logger.info("Falling back to synthetic forecast data", reason="invalid_forecast_data") + return await self.synthetic_generator.generate_forecast(days) + + except Exception as e: + logger.error("Failed to get weather forecast", error=str(e)) + return await self.synthetic_generator.generate_forecast(days) + + async def get_historical_weather(self, + latitude: float, + longitude: float, + start_date: datetime, + end_date: datetime) -> List[Dict[str, Any]]: + """Get historical weather data""" + try: + logger.debug("Getting historical weather from AEMET API", + lat=latitude, lon=longitude, + start=start_date, end=end_date) + + station_id = self.location_service.find_nearest_station(latitude, longitude) + if not station_id: + logger.warning("No weather station found for historical data", + lat=latitude, lon=longitude) + return self.synthetic_generator.generate_historical_data(start_date, end_date) + + historical_data = await self._fetch_historical_data_in_chunks( + station_id, start_date, end_date + ) + + if historical_data: + logger.debug("Successfully fetched historical weather data", + total_count=len(historical_data)) + return historical_data + else: + logger.info("No real historical data available, using synthetic data") + return self.synthetic_generator.generate_historical_data(start_date, end_date) + + except Exception as e: + logger.error("Failed to get historical weather from AEMET API", error=str(e)) + return self.synthetic_generator.generate_historical_data(start_date, end_date) + + async def _fetch_current_weather_data(self, station_id: str) -> Optional[Dict[str, Any]]: + """Fetch current weather data from AEMET API""" + endpoint = f"/observacion/convencional/datos/estacion/{station_id}" + initial_response = await self._get(endpoint) + + if not self._is_valid_initial_response(initial_response): + return None + + datos_url = initial_response.get("datos") + actual_weather_data = await self._fetch_from_url(datos_url) + + if (actual_weather_data and isinstance(actual_weather_data, list) + and len(actual_weather_data) > 0): + return actual_weather_data[0] + + return None + + async def _fetch_forecast_data(self, municipality_code: str) -> Optional[List[Dict[str, Any]]]: + """Fetch forecast data from AEMET API""" + endpoint = f"/prediccion/especifica/municipio/diaria/{municipality_code}" + initial_response = await self._get(endpoint) + + if not self._is_valid_initial_response(initial_response): + return None + + datos_url = initial_response.get("datos") + return await self._fetch_from_url(datos_url) + + async def _fetch_historical_data_in_chunks(self, + station_id: str, + start_date: datetime, + end_date: datetime) -> List[Dict[str, Any]]: + """Fetch historical data in chunks due to AEMET API limitations""" + historical_data = [] + current_date = start_date + + while current_date <= end_date: + chunk_end_date = min( + current_date + timedelta(days=AEMETConstants.MAX_DAYS_PER_REQUEST), + end_date + ) + + chunk_data = await self._fetch_historical_chunk( + station_id, current_date, chunk_end_date + ) + + if chunk_data: + historical_data.extend(chunk_data) + + current_date = chunk_end_date + timedelta(days=1) + + return historical_data + + async def _fetch_historical_chunk(self, + station_id: str, + start_date: datetime, + end_date: datetime) -> List[Dict[str, Any]]: + """Fetch a single chunk of historical data""" + start_str = start_date.strftime("%Y-%m-%dT00:00:00UTC") + end_str = end_date.strftime("%Y-%m-%dT23:59:59UTC") + + endpoint = f"/valores/climatologicos/diarios/datos/fechaini/{start_str}/fechafin/{end_str}/estacion/{station_id}" + initial_response = await self._get(endpoint) + + if not self._is_valid_initial_response(initial_response): + logger.warning("Invalid initial response from AEMET historical API", + start=start_str, end=end_str) + return [] + + datos_url = initial_response.get("datos") + if not datos_url: + logger.warning("No datos URL in AEMET historical response", + start=start_str, end=end_str) + return [] + + actual_historical_data = await self._fetch_from_url(datos_url) + + if actual_historical_data and isinstance(actual_historical_data, list): + chunk_data = self.parser.parse_historical_data(actual_historical_data) + logger.debug("Fetched historical data chunk", + count=len(chunk_data), start=start_str, end=end_str) + return chunk_data + else: + logger.warning("No valid historical data received for chunk", + start=start_str, end=end_str) + return [] + + async def _fetch_from_url(self, url: str) -> Optional[List[Dict[str, Any]]]: + """Fetch data from AEMET datos URL""" + try: + data = await self._fetch_url_directly(url) + + if data and isinstance(data, list): + return data + else: + logger.warning("Expected list from datos URL", data_type=type(data)) + return None + + except Exception as e: + logger.error("Failed to fetch from datos URL", url=url, error=str(e)) + return None + + def _is_valid_initial_response(self, response: Any) -> bool: + """Check if initial AEMET API response is valid""" + return (response and isinstance(response, dict) and + response.get("datos") and isinstance(response.get("datos"), str)) + + async def _get_synthetic_current_weather(self) -> Dict[str, Any]: + """Get synthetic current weather data""" + return self.synthetic_generator.generate_current_weather() \ No newline at end of file diff --git a/services/data/requirements.txt b/services/data/requirements.txt index 5bef0f3a..51c8f8ca 100644 --- a/services/data/requirements.txt +++ b/services/data/requirements.txt @@ -22,7 +22,7 @@ aio-pika==9.3.1 # HTTP client httpx==0.25.2 -# Data processing (UPDATED - Added openpyxl and xlrd) +# Data processing pandas==2.1.3 numpy==1.25.2 openpyxl==3.1.2 # For Excel (.xlsx) files @@ -43,5 +43,10 @@ bcrypt==4.1.2 pytest==7.4.3 pytest-asyncio==0.21.1 pytest-cov==4.1.0 +pytest-mock==3.12.0 +pytest-xdist==3.5.0 +pytest-timeout==2.2.0 +psutil==5.9.8 +# Cartographic projections and coordinate transformations library pyproj==3.4.0 \ No newline at end of file diff --git a/services/data/tests/conftest.py b/services/data/tests/conftest.py index f506bf4e..93a6b057 100644 --- a/services/data/tests/conftest.py +++ b/services/data/tests/conftest.py @@ -1,18 +1,653 @@ # ================================================================ -# services/data/tests/conftest.py +# services/data/tests/conftest.py - AEMET Test Configuration # ================================================================ -"""Test configuration for data service""" +""" +Test configuration and fixtures for AEMET weather API client tests +Provides shared fixtures, mock data, and test utilities +""" import pytest -import pytest_asyncio -from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession, async_sessionmaker -from sqlalchemy.pool import StaticPool -from fastapi.testclient import TestClient -import uuid -from datetime import datetime +import asyncio +from datetime import datetime, timedelta +from unittest.mock import Mock, AsyncMock, patch +from typing import Dict, List, Any, Generator +import os -from app.main import app -from app.core.database import Base, get_db -from app.models.sales import SalesData -from app.models.weather import WeatherData, WeatherForecast -from app.models.traffic import TrafficData +# Import the classes we're testing +from app.external.aemet import ( + AEMETClient, + WeatherDataParser, + SyntheticWeatherGenerator, + LocationService, + WeatherSource +) + + +# ================================================================ +# PYTEST CONFIGURATION +# ================================================================ + +@pytest.fixture(scope="session") +def event_loop(): + """Create an instance of the default event loop for the test session.""" + loop = asyncio.get_event_loop_policy().new_event_loop() + yield loop + loop.close() + + +# ================================================================ +# CLIENT AND SERVICE FIXTURES +# ================================================================ + +@pytest.fixture +def aemet_client(): + """Create AEMET client instance for testing""" + return AEMETClient() + + +@pytest.fixture +def weather_parser(): + """Create WeatherDataParser instance for testing""" + return WeatherDataParser() + + +@pytest.fixture +def synthetic_generator(): + """Create SyntheticWeatherGenerator instance for testing""" + return SyntheticWeatherGenerator() + + +@pytest.fixture +def location_service(): + """Create LocationService instance for testing""" + return LocationService() + + +# ================================================================ +# COORDINATE AND LOCATION FIXTURES +# ================================================================ + +@pytest.fixture +def madrid_coords(): + """Standard Madrid coordinates for testing""" + return (40.4168, -3.7038) # Madrid city center + + +@pytest.fixture +def madrid_coords_variants(): + """Various Madrid area coordinates for testing""" + return { + "center": (40.4168, -3.7038), # Madrid center + "north": (40.4677, -3.5552), # Madrid north (near station) + "south": (40.2987, -3.7216), # Madrid south (near station) + "east": (40.4200, -3.6500), # Madrid east + "west": (40.4100, -3.7500), # Madrid west + } + + +@pytest.fixture +def invalid_coords(): + """Invalid coordinates for error testing""" + return [ + (200, 200), # Out of range + (-200, -200), # Out of range + (0, 0), # Not in Madrid area + (50, 10), # Europe but not Madrid + (None, None), # None values + ] + + +# ================================================================ +# DATE AND TIME FIXTURES +# ================================================================ + +@pytest.fixture +def test_dates(): + """Standard date ranges for testing""" + now = datetime.now() + return { + "now": now, + "yesterday": now - timedelta(days=1), + "last_week": now - timedelta(days=7), + "last_month": now - timedelta(days=30), + "last_quarter": now - timedelta(days=90), + "one_year_ago": now - timedelta(days=365), + } + + +@pytest.fixture +def historical_date_ranges(): + """Historical date ranges for testing""" + end_date = datetime.now() + return { + "one_day": { + "start": end_date - timedelta(days=1), + "end": end_date, + "expected_days": 1 + }, + "one_week": { + "start": end_date - timedelta(days=7), + "end": end_date, + "expected_days": 7 + }, + "one_month": { + "start": end_date - timedelta(days=30), + "end": end_date, + "expected_days": 30 + }, + "large_range": { + "start": end_date - timedelta(days=65), + "end": end_date, + "expected_days": 65 + } + } + + +# ================================================================ +# MOCK API RESPONSE FIXTURES +# ================================================================ + +@pytest.fixture +def mock_aemet_api_response(): + """Mock AEMET API initial response structure""" + return { + "datos": "https://opendata.aemet.es/opendata/sh/12345abcdef", + "metadatos": "https://opendata.aemet.es/opendata/sh/metadata123" + } + + +@pytest.fixture +def mock_aemet_error_response(): + """Mock AEMET API error response""" + return { + "descripcion": "Error en la petición", + "estado": 404 + } + + +# ================================================================ +# WEATHER DATA FIXTURES +# ================================================================ + +@pytest.fixture +def mock_current_weather_data(): + """Mock current weather data from AEMET API""" + return { + "idema": "3195", # Station ID + "ubi": "MADRID", # Location + "fint": "2025-07-24T14:00:00", # Observation time + "ta": 18.5, # Temperature (°C) + "tamin": 12.3, # Min temperature + "tamax": 25.7, # Max temperature + "hr": 65.0, # Humidity (%) + "prec": 0.0, # Precipitation (mm) + "vv": 12.0, # Wind speed (km/h) + "dv": 180, # Wind direction (degrees) + "pres": 1015.2, # Pressure (hPa) + "presMax": 1018.5, # Max pressure + "presMin": 1012.1, # Min pressure + "descripcion": "Despejado" # Description + } + + +@pytest.fixture +def mock_forecast_data(): + """Mock forecast data from AEMET API""" + return [{ + "origen": { + "productor": "Agencia Estatal de Meteorología - AEMET" + }, + "elaborado": "2025-07-24T12:00:00UTC", + "nombre": "Madrid", + "provincia": "Madrid", + "prediccion": { + "dia": [ + { + "fecha": "2025-07-25T00:00:00", + "temperatura": { + "maxima": 28, + "minima": 15, + "dato": [ + {"value": 15, "hora": 6}, + {"value": 28, "hora": 15} + ] + }, + "sensTermica": { + "maxima": 30, + "minima": 16 + }, + "humedadRelativa": { + "maxima": 85, + "minima": 45, + "dato": [ + {"value": 85, "hora": 6}, + {"value": 45, "hora": 15} + ] + }, + "probPrecipitacion": [ + {"value": 10, "periodo": "00-24"} + ], + "viento": [ + { + "direccion": ["N"], + "velocidad": [15], + "periodo": "00-24" + } + ], + "estadoCielo": [ + { + "value": "11", + "descripcion": "Despejado", + "periodo": "00-24" + } + ] + }, + { + "fecha": "2025-07-26T00:00:00", + "temperatura": { + "maxima": 30, + "minima": 17 + }, + "probPrecipitacion": [ + {"value": 5, "periodo": "00-24"} + ], + "viento": [ + { + "direccion": ["NE"], + "velocidad": [10], + "periodo": "00-24" + } + ] + } + ] + } + }] + + +@pytest.fixture +def mock_historical_data(): + """Mock historical weather data from AEMET API""" + return [ + { + "indicativo": "3195", + "nombre": "MADRID", + "fecha": "2025-07-20", + "tmax": 25.2, + "horatmax": "1530", + "tmin": 14.8, + "horatmin": "0630", + "tmed": 20.0, + "prec": 0.0, + "racha": 25.0, + "horaracha": "1445", + "sol": 8.5, + "presMax": 1018.5, + "horaPresMax": "1000", + "presMin": 1012.3, + "horaPresMin": "1700", + "hr": 58, + "velmedia": 8.5, + "dir": "180" + }, + { + "indicativo": "3195", + "nombre": "MADRID", + "fecha": "2025-07-21", + "tmax": 27.1, + "horatmax": "1615", + "tmin": 16.2, + "horatmin": "0700", + "tmed": 21.6, + "prec": 2.5, + "racha": 30.0, + "horaracha": "1330", + "sol": 6.2, + "presMax": 1015.8, + "horaPresMax": "0930", + "presMin": 1010.1, + "horaPresMin": "1800", + "hr": 72, + "velmedia": 12.0, + "dir": "225" + }, + { + "indicativo": "3195", + "nombre": "MADRID", + "fecha": "2025-07-22", + "tmax": 23.8, + "horatmax": "1500", + "tmin": 13.5, + "horatmin": "0615", + "tmed": 18.7, + "prec": 0.2, + "racha": 22.0, + "horaracha": "1200", + "sol": 7.8, + "presMax": 1020.2, + "horaPresMax": "1100", + "presMin": 1014.7, + "horaPresMin": "1900", + "hr": 63, + "velmedia": 9.2, + "dir": "270" + } + ] + + +# ================================================================ +# EXPECTED RESULT FIXTURES +# ================================================================ + +@pytest.fixture +def expected_current_weather_structure(): + """Expected structure for current weather results""" + return { + "required_fields": [ + "date", "temperature", "precipitation", "humidity", + "wind_speed", "pressure", "description", "source" + ], + "field_types": { + "date": datetime, + "temperature": (int, float), + "precipitation": (int, float), + "humidity": (int, float), + "wind_speed": (int, float), + "pressure": (int, float), + "description": str, + "source": str + }, + "valid_ranges": { + "temperature": (-30, 50), + "precipitation": (0, 200), + "humidity": (0, 100), + "wind_speed": (0, 200), + "pressure": (900, 1100) + } + } + + +@pytest.fixture +def expected_forecast_structure(): + """Expected structure for forecast results""" + return { + "required_fields": [ + "forecast_date", "generated_at", "temperature", "precipitation", + "humidity", "wind_speed", "description", "source" + ], + "field_types": { + "forecast_date": datetime, + "generated_at": datetime, + "temperature": (int, float), + "precipitation": (int, float), + "humidity": (int, float), + "wind_speed": (int, float), + "description": str, + "source": str + } + } + + +@pytest.fixture +def expected_historical_structure(): + """Expected structure for historical weather results""" + return { + "required_fields": [ + "date", "temperature", "precipitation", "humidity", + "wind_speed", "pressure", "description", "source" + ], + "field_types": { + "date": datetime, + "temperature": (int, float, type(None)), + "precipitation": (int, float), + "humidity": (int, float, type(None)), + "wind_speed": (int, float, type(None)), + "pressure": (int, float, type(None)), + "description": str, + "source": str + } + } + + +# ================================================================ +# MOCK AND PATCH FIXTURES +# ================================================================ + +@pytest.fixture +def mock_successful_api_calls(): + """Mock successful AEMET API calls""" + def _mock_api_calls(client, response_data, fetch_data): + with patch.object(client, '_get', new_callable=AsyncMock) as mock_get, \ + patch.object(client, '_fetch_from_url', new_callable=AsyncMock) as mock_fetch: + + mock_get.return_value = response_data + mock_fetch.return_value = fetch_data + + return mock_get, mock_fetch + + return _mock_api_calls + + +@pytest.fixture +def mock_failed_api_calls(): + """Mock failed AEMET API calls""" + def _mock_failed_calls(client, error_type="network"): + if error_type == "network": + return patch.object(client, '_get', side_effect=Exception("Network error")) + elif error_type == "timeout": + return patch.object(client, '_get', side_effect=asyncio.TimeoutError("Request timeout")) + elif error_type == "invalid_response": + return patch.object(client, '_get', new_callable=AsyncMock, return_value=None) + else: + return patch.object(client, '_get', new_callable=AsyncMock, return_value={"error": "API error"}) + + return _mock_failed_calls + + +# ================================================================ +# VALIDATION HELPER FIXTURES +# ================================================================ + +@pytest.fixture +def weather_data_validator(): + """Weather data validation helper functions""" + + def validate_weather_record(record: Dict[str, Any], expected_structure: Dict[str, Any]) -> None: + """Validate a weather record against expected structure""" + # Check required fields + for field in expected_structure["required_fields"]: + assert field in record, f"Missing required field: {field}" + + # Check field types + for field, expected_type in expected_structure["field_types"].items(): + if field in record and record[field] is not None: + assert isinstance(record[field], expected_type), f"Field {field} has wrong type: {type(record[field])}" + + # Check valid ranges where applicable + if "valid_ranges" in expected_structure: + for field, (min_val, max_val) in expected_structure["valid_ranges"].items(): + if field in record and record[field] is not None: + value = record[field] + assert min_val <= value <= max_val, f"Field {field} value {value} outside valid range [{min_val}, {max_val}]" + + def validate_weather_list(records: List[Dict[str, Any]], expected_structure: Dict[str, Any]) -> None: + """Validate a list of weather records""" + assert isinstance(records, list), "Records should be a list" + + for i, record in enumerate(records): + try: + validate_weather_record(record, expected_structure) + except AssertionError as e: + raise AssertionError(f"Record {i} validation failed: {e}") + + def validate_date_sequence(records: List[Dict[str, Any]], date_field: str = "date") -> None: + """Validate that dates in records are in chronological order""" + dates = [r[date_field] for r in records if date_field in r and r[date_field] is not None] + + if len(dates) > 1: + assert dates == sorted(dates), "Dates should be in chronological order" + + return { + "validate_record": validate_weather_record, + "validate_list": validate_weather_list, + "validate_dates": validate_date_sequence + } + + +# ================================================================ +# PERFORMANCE TESTING FIXTURES +# ================================================================ + +@pytest.fixture +def performance_tracker(): + """Performance tracking utilities for tests""" + + class PerformanceTracker: + def __init__(self): + self.start_time = None + self.measurements = {} + + def start(self, operation_name: str = "default"): + self.start_time = datetime.now() + self.operation_name = operation_name + + def stop(self) -> float: + if self.start_time: + duration = (datetime.now() - self.start_time).total_seconds() * 1000 + self.measurements[self.operation_name] = duration + return duration + return 0.0 + + def assert_performance(self, max_duration_ms: float, operation_name: str = "default"): + duration = self.measurements.get(operation_name, float('inf')) + assert duration <= max_duration_ms, f"Operation {operation_name} took {duration:.0f}ms, expected <= {max_duration_ms}ms" + + return PerformanceTracker() + + +# ================================================================ +# INTEGRATION TEST FIXTURES +# ================================================================ + +@pytest.fixture +def integration_test_config(): + """Configuration for integration tests""" + return { + "api_timeout_ms": 5000, + "max_retries": 3, + "test_api_key": os.getenv("AEMET_API_KEY_TEST", ""), + "skip_real_api_tests": os.getenv("SKIP_REAL_API_TESTS", "false").lower() == "true", + "madrid_test_coords": (40.4168, -3.7038), + "performance_thresholds": { + "current_weather_ms": 5000, + "forecast_ms": 5000, + "historical_ms": 10000 + } + } + + +# ================================================================ +# TEST REPORTING FIXTURES +# ================================================================ + +@pytest.fixture +def test_reporter(): + """Test reporting utilities""" + + class TestReporter: + def __init__(self): + self.results = [] + + def log_success(self, test_name: str, details: str = ""): + message = f"✅ {test_name}" + if details: + message += f" - {details}" + print(message) + self.results.append({"test": test_name, "status": "PASS", "details": details}) + + def log_failure(self, test_name: str, error: str = ""): + message = f"❌ {test_name}" + if error: + message += f" - {error}" + print(message) + self.results.append({"test": test_name, "status": "FAIL", "error": error}) + + def log_info(self, test_name: str, info: str = ""): + message = f"ℹ️ {test_name}" + if info: + message += f" - {info}" + print(message) + self.results.append({"test": test_name, "status": "INFO", "info": info}) + + def summary(self): + passed = len([r for r in self.results if r["status"] == "PASS"]) + failed = len([r for r in self.results if r["status"] == "FAIL"]) + print(f"\n📊 Test Summary: {passed} passed, {failed} failed") + return passed, failed + + return TestReporter() + + +# ================================================================ +# CLEANUP FIXTURES +# ================================================================ + +@pytest.fixture(autouse=True) +def cleanup_after_test(): + """Automatic cleanup after each test""" + yield + # Add any cleanup logic here + # For example, clearing caches, resetting global state, etc. + pass + + +# ================================================================ +# HELPER FUNCTIONS +# ================================================================ + +def assert_weather_data_structure(data: Dict[str, Any], data_type: str = "current"): + """Assert that weather data has the correct structure""" + if data_type == "current": + required_fields = ["date", "temperature", "precipitation", "humidity", "wind_speed", "pressure", "description", "source"] + elif data_type == "forecast": + required_fields = ["forecast_date", "generated_at", "temperature", "precipitation", "humidity", "wind_speed", "description", "source"] + elif data_type == "historical": + required_fields = ["date", "temperature", "precipitation", "humidity", "wind_speed", "pressure", "description", "source"] + else: + raise ValueError(f"Unknown data type: {data_type}") + + for field in required_fields: + assert field in data, f"Missing required field: {field}" + + # Validate source + valid_sources = [WeatherSource.AEMET.value, WeatherSource.SYNTHETIC.value, WeatherSource.DEFAULT.value] + assert data["source"] in valid_sources, f"Invalid source: {data['source']}" + + +def assert_forecast_list_structure(forecast_list: List[Dict[str, Any]], expected_days: int): + """Assert that forecast list has correct structure""" + assert isinstance(forecast_list, list), "Forecast should be a list" + assert len(forecast_list) == expected_days, f"Expected {expected_days} forecast days, got {len(forecast_list)}" + + for i, day in enumerate(forecast_list): + assert_weather_data_structure(day, "forecast") + + # Check date progression + if len(forecast_list) > 1: + for i in range(1, len(forecast_list)): + prev_date = forecast_list[i-1]["forecast_date"] + curr_date = forecast_list[i]["forecast_date"] + date_diff = (curr_date - prev_date).days + assert date_diff == 1, f"Forecast dates should be consecutive, got {date_diff} day difference" + + +def assert_historical_list_structure(historical_list: List[Dict[str, Any]]): + """Assert that historical list has correct structure""" + assert isinstance(historical_list, list), "Historical data should be a list" + + for i, record in enumerate(historical_list): + assert_weather_data_structure(record, "historical") + + # Check date ordering + dates = [r["date"] for r in historical_list if "date" in r] + if len(dates) > 1: + assert dates == sorted(dates), "Historical dates should be in chronological order" \ No newline at end of file diff --git a/services/data/tests/pytest.ini b/services/data/tests/pytest.ini index 4bd8c845..c98b2fb8 100644 --- a/services/data/tests/pytest.ini +++ b/services/data/tests/pytest.ini @@ -1,16 +1,44 @@ [tool:pytest] -# pytest.ini - Configuration for async testing -asyncio_mode = auto -addopts = -v --tb=short --capture=no +# pytest.ini - Configuration file for AEMET tests + +# Minimum version requirements +minversion = 6.0 + +# Add options +addopts = + -ra + --strict-markers + --strict-config + --disable-warnings + --tb=short + -v + +# Test discovery testpaths = tests python_files = test_*.py python_classes = Test* python_functions = test_* + +# Async support +asyncio_mode = auto + +# Markers markers = - asyncio: mark test as async - slow: mark test as slow - integration: mark test as integration test + unit: Unit tests + integration: Integration tests + api: API tests + performance: Performance tests + slow: Slow tests + asyncio: Async tests + +# Logging +log_cli = true +log_cli_level = INFO +log_cli_format = %(asctime)s [%(levelname)8s] %(name)s: %(message)s +log_cli_date_format = %Y-%m-%d %H:%M:%S + +# Filtering filterwarnings = ignore::DeprecationWarning ignore::PendingDeprecationWarning - ignore::PydanticDeprecatedSince20 \ No newline at end of file + ignore::PytestUnhandledCoroutineWarning \ No newline at end of file diff --git a/services/data/tests/test_aemet.py b/services/data/tests/test_aemet.py new file mode 100644 index 00000000..6d5b0aa6 --- /dev/null +++ b/services/data/tests/test_aemet.py @@ -0,0 +1,677 @@ +# ================================================================ +# services/data/tests/test_aemet.py +# ================================================================ +""" +Comprehensive test suite for AEMET weather API client +Following the same patterns as test_madrid_opendata.py +""" + +import pytest +import asyncio +from datetime import datetime, timedelta +from unittest.mock import Mock, patch, AsyncMock +import math +from typing import Dict, List, Any + +from app.external.aemet import ( + AEMETClient, + WeatherDataParser, + SyntheticWeatherGenerator, + LocationService, + AEMETConstants, + WeatherSource, + WeatherStation, + GeographicBounds +) + +# Configure pytest-asyncio +pytestmark = pytest.mark.asyncio + + +class TestAEMETClient: + """Main test class for AEMET API client functionality""" + + @pytest.fixture + def client(self): + """Create AEMET client instance for testing""" + return AEMETClient() + + @pytest.fixture + def madrid_coords(self): + """Standard Madrid coordinates for testing""" + return (40.4168, -3.7038) # Madrid city center + + @pytest.fixture + def mock_aemet_response(self): + """Mock AEMET API response structure""" + return { + "datos": "https://opendata.aemet.es/opendata/sh/12345", + "metadatos": "https://opendata.aemet.es/opendata/sh/metadata" + } + + @pytest.fixture + def mock_weather_data(self): + """Mock current weather data from AEMET""" + return { + "ta": 18.5, # Temperature + "prec": 0.0, # Precipitation + "hr": 65.0, # Humidity + "vv": 12.0, # Wind speed + "pres": 1015.2, # Pressure + "descripcion": "Despejado" + } + + @pytest.fixture + def mock_forecast_data(self): + """Mock forecast data from AEMET""" + return [{ + "prediccion": { + "dia": [ + { + "fecha": "2025-07-25T00:00:00", + "temperatura": { + "maxima": 28, + "minima": 15 + }, + "probPrecipitacion": [ + {"value": 10, "periodo": "00-24"} + ], + "viento": [ + {"velocidad": [15], "direccion": ["N"]} + ] + }, + { + "fecha": "2025-07-26T00:00:00", + "temperatura": { + "maxima": 30, + "minima": 17 + }, + "probPrecipitacion": [ + {"value": 5, "periodo": "00-24"} + ], + "viento": [ + {"velocidad": [10], "direccion": ["NE"]} + ] + } + ] + } + }] + + @pytest.fixture + def mock_historical_data(self): + """Mock historical weather data from AEMET""" + return [ + { + "fecha": "2025-07-20", + "tmax": 25.2, + "tmin": 14.8, + "prec": 0.0, + "hr": 58, + "velmedia": 8.5, + "presMax": 1018.5, + "presMin": 1012.3 + }, + { + "fecha": "2025-07-21", + "tmax": 27.1, + "tmin": 16.2, + "prec": 2.5, + "hr": 72, + "velmedia": 12.0, + "presMax": 1015.8, + "presMin": 1010.1 + } + ] + + # ================================================================ + # CURRENT WEATHER TESTS + # ================================================================ + + async def test_get_current_weather_success(self, client, madrid_coords, mock_aemet_response, mock_weather_data): + """Test successful current weather retrieval""" + lat, lon = madrid_coords + + with patch.object(client, '_get', new_callable=AsyncMock) as mock_get, \ + patch.object(client, '_fetch_from_url', new_callable=AsyncMock) as mock_fetch: + + mock_get.return_value = mock_aemet_response + mock_fetch.return_value = [mock_weather_data] + + result = await client.get_current_weather(lat, lon) + + # Validate result structure + assert result is not None, "Should return weather data" + assert isinstance(result, dict), "Result should be a dictionary" + + # Check required fields + required_fields = ['date', 'temperature', 'precipitation', 'humidity', 'wind_speed', 'pressure', 'description', 'source'] + for field in required_fields: + assert field in result, f"Missing required field: {field}" + + # Validate data types and ranges + assert isinstance(result['temperature'], float), "Temperature should be float" + assert -20 <= result['temperature'] <= 50, "Temperature should be reasonable" + assert isinstance(result['precipitation'], float), "Precipitation should be float" + assert result['precipitation'] >= 0, "Precipitation should be non-negative" + assert 0 <= result['humidity'] <= 100, "Humidity should be percentage" + assert result['wind_speed'] >= 0, "Wind speed should be non-negative" + assert result['pressure'] > 900, "Pressure should be reasonable" + assert result['source'] == WeatherSource.AEMET.value, "Source should be AEMET" + + print(f"✅ Current weather test passed - Temp: {result['temperature']}°C, Source: {result['source']}") + + async def test_get_current_weather_fallback_to_synthetic(self, client, madrid_coords): + """Test fallback to synthetic data when AEMET API fails""" + lat, lon = madrid_coords + + with patch.object(client, '_get', new_callable=AsyncMock) as mock_get: + mock_get.return_value = None # Simulate API failure + + result = await client.get_current_weather(lat, lon) + + assert result is not None, "Should return synthetic data" + assert result['source'] == WeatherSource.SYNTHETIC.value, "Should use synthetic source" + assert isinstance(result['temperature'], float), "Temperature should be float" + + print(f"✅ Synthetic fallback test passed - Source: {result['source']}") + + async def test_get_current_weather_invalid_coordinates(self, client): + """Test current weather with invalid coordinates""" + invalid_coords = [ + (200, 200), # Out of range + (-200, -200), # Out of range + (0, 0), # Not in Madrid area + ] + + for lat, lon in invalid_coords: + result = await client.get_current_weather(lat, lon) + + # Should still return data (synthetic) + assert result is not None, f"Should handle invalid coords ({lat}, {lon})" + assert result['source'] == WeatherSource.SYNTHETIC.value, "Should use synthetic for invalid coords" + + print(f"✅ Invalid coordinates test passed") + + # ================================================================ + # FORECAST TESTS + # ================================================================ + + async def test_get_forecast_success(self, client, madrid_coords, mock_aemet_response, mock_forecast_data): + """Test successful weather forecast retrieval""" + lat, lon = madrid_coords + days = 7 + + with patch.object(client, '_get', new_callable=AsyncMock) as mock_get, \ + patch.object(client, '_fetch_from_url', new_callable=AsyncMock) as mock_fetch: + + mock_get.return_value = mock_aemet_response + mock_fetch.return_value = mock_forecast_data + + result = await client.get_forecast(lat, lon, days) + + # Validate result structure + assert isinstance(result, list), "Result should be a list" + assert len(result) == days, f"Should return {days} forecast days" + + # Check first forecast day + if result: + forecast_day = result[0] + + required_fields = ['forecast_date', 'generated_at', 'temperature', 'precipitation', 'humidity', 'wind_speed', 'description', 'source'] + for field in required_fields: + assert field in forecast_day, f"Missing required field: {field}" + + # Validate data types + assert isinstance(forecast_day['forecast_date'], datetime), "Forecast date should be datetime" + assert isinstance(forecast_day['temperature'], (int, float)), "Temperature should be numeric" + assert isinstance(forecast_day['precipitation'], (int, float)), "Precipitation should be numeric" + assert forecast_day['source'] in [WeatherSource.AEMET.value, WeatherSource.SYNTHETIC.value], "Valid source" + + print(f"✅ Forecast test passed - {len(result)} days, Source: {forecast_day['source']}") + + async def test_get_forecast_different_durations(self, client, madrid_coords): + """Test forecast for different time durations""" + lat, lon = madrid_coords + test_durations = [1, 3, 7, 14] + + for days in test_durations: + result = await client.get_forecast(lat, lon, days) + + assert isinstance(result, list), f"Result should be list for {days} days" + assert len(result) == days, f"Should return exactly {days} forecast days" + + # Check date progression + if len(result) > 1: + for i in range(1, len(result)): + date_diff = result[i]['forecast_date'] - result[i-1]['forecast_date'] + assert date_diff.days == 1, "Forecast dates should be consecutive days" + + print(f"✅ Multiple duration forecast test passed") + + async def test_get_forecast_fallback_to_synthetic(self, client, madrid_coords): + """Test forecast fallback to synthetic data""" + lat, lon = madrid_coords + + with patch.object(client.location_service, 'get_municipality_code') as mock_municipality: + mock_municipality.return_value = None # No municipality found + + result = await client.get_forecast(lat, lon, 7) + + assert isinstance(result, list), "Should return synthetic forecast" + assert len(result) == 7, "Should return 7 days" + assert all(day['source'] == WeatherSource.SYNTHETIC.value for day in result), "All should be synthetic" + + print(f"✅ Forecast synthetic fallback test passed") + + # ================================================================ + # HISTORICAL WEATHER TESTS + # ================================================================ + + async def test_get_historical_weather_success(self, client, madrid_coords, mock_aemet_response, mock_historical_data): + """Test successful historical weather retrieval""" + lat, lon = madrid_coords + end_date = datetime.now() + start_date = end_date - timedelta(days=7) + + with patch.object(client, '_get', new_callable=AsyncMock) as mock_get, \ + patch.object(client, '_fetch_from_url', new_callable=AsyncMock) as mock_fetch: + + mock_get.return_value = mock_aemet_response + mock_fetch.return_value = mock_historical_data + + result = await client.get_historical_weather(lat, lon, start_date, end_date) + + # Validate result structure + assert isinstance(result, list), "Result should be a list" + assert len(result) > 0, "Should return historical data" + + # Check first historical record + if result: + record = result[0] + + required_fields = ['date', 'temperature', 'precipitation', 'humidity', 'wind_speed', 'pressure', 'description', 'source'] + for field in required_fields: + assert field in record, f"Missing required field: {field}" + + # Validate data types and ranges + assert isinstance(record['date'], datetime), "Date should be datetime" + assert isinstance(record['temperature'], (int, float, type(None))), "Temperature should be numeric or None" + if record['temperature']: + assert -30 <= record['temperature'] <= 50, "Temperature should be reasonable" + assert record['precipitation'] >= 0, "Precipitation should be non-negative" + assert record['source'] == WeatherSource.AEMET.value, "Source should be AEMET" + + print(f"✅ Historical weather test passed - {len(result)} records, Source: {record['source']}") + + async def test_get_historical_weather_date_ranges(self, client, madrid_coords): + """Test historical weather with different date ranges""" + lat, lon = madrid_coords + end_date = datetime.now() + + test_ranges = [ + 1, # 1 day + 7, # 1 week + 30, # 1 month + 90, # 3 months + ] + + for days in test_ranges: + start_date = end_date - timedelta(days=days) + + result = await client.get_historical_weather(lat, lon, start_date, end_date) + + assert isinstance(result, list), f"Result should be list for {days} days" + # Note: Actual count may vary due to chunking and data availability + assert len(result) >= 0, f"Should return non-negative count for {days} days" + + if result: + # Check date ordering + dates = [r['date'] for r in result if 'date' in r] + if len(dates) > 1: + assert dates == sorted(dates), "Historical dates should be in chronological order" + + print(f"✅ Historical date ranges test passed") + + async def test_get_historical_weather_chunking(self, client, madrid_coords): + """Test historical weather data chunking for large date ranges""" + lat, lon = madrid_coords + end_date = datetime.now() + start_date = end_date - timedelta(days=65) # More than 30 days to trigger chunking + + with patch.object(client, '_fetch_historical_chunk', new_callable=AsyncMock) as mock_chunk: + mock_chunk.return_value = [] # Empty chunks + + result = await client.get_historical_weather(lat, lon, start_date, end_date) + + # Should have called chunking at least twice (65 days > 30 day limit) + assert mock_chunk.call_count >= 2, "Should chunk large date ranges" + + print(f"✅ Historical chunking test passed - {mock_chunk.call_count} chunks") + + # ================================================================ + # COMPONENT TESTS + # ================================================================ + @pytest.mark.skip_asyncio + def test_weather_data_parser(self): + """Test WeatherDataParser functionality""" + parser = WeatherDataParser() + + # Test safe_float + assert parser.safe_float("15.5", 0.0) == 15.5 + assert parser.safe_float(None, 10.0) == 10.0 + assert parser.safe_float("invalid", 5.0) == 5.0 + assert parser.safe_float(20) == 20.0 + + # Test extract_temperature_value + assert parser.extract_temperature_value(25.5) == 25.5 + assert parser.extract_temperature_value("20.0") == 20.0 + assert parser.extract_temperature_value({"valor": 18.5}) == 18.5 + assert parser.extract_temperature_value([{"valor": 22.0}]) == 22.0 + assert parser.extract_temperature_value(None) is None + + # Test generate_weather_description + assert "Lluvioso" in parser.generate_weather_description(20, 6.0, 60) + assert "Nuboso con lluvia" in parser.generate_weather_description(20, 1.0, 60) + assert "Nuboso" in parser.generate_weather_description(20, 0, 85) + assert "Soleado y cálido" in parser.generate_weather_description(30, 0, 60) + assert "Frío" in parser.generate_weather_description(2, 0, 60) + + print(f"✅ WeatherDataParser tests passed") + + @pytest.mark.skip_asyncio + def test_synthetic_weather_generator(self): + """Test SyntheticWeatherGenerator functionality""" + generator = SyntheticWeatherGenerator() + + # Test current weather generation + current = generator.generate_current_weather() + + assert isinstance(current, dict), "Should return dictionary" + assert 'temperature' in current, "Should have temperature" + assert 'precipitation' in current, "Should have precipitation" + assert current['source'] == WeatherSource.SYNTHETIC.value, "Should be synthetic source" + assert isinstance(current['date'], datetime), "Should have datetime" + + # Test forecast generation + forecast = generator.generate_forecast_sync(5) + + assert isinstance(forecast, list), "Should return list" + assert len(forecast) == 5, "Should return requested days" + assert all('forecast_date' in day for day in forecast), "All days should have forecast_date" + assert all(day['source'] == WeatherSource.SYNTHETIC.value for day in forecast), "All should be synthetic" + + # Test historical generation + end_date = datetime.now() + start_date = end_date - timedelta(days=7) + historical = generator.generate_historical_data(start_date, end_date) + + assert isinstance(historical, list), "Should return list" + assert len(historical) == 8, "Should return 8 days (inclusive)" + assert all('date' in day for day in historical), "All days should have date" + assert all(day['source'] == WeatherSource.SYNTHETIC.value for day in historical), "All should be synthetic" + + print(f"✅ SyntheticWeatherGenerator tests passed") + + @pytest.mark.skip_asyncio + def test_location_service(self): + """Test LocationService functionality""" + # Test distance calculation + madrid_center = (40.4168, -3.7038) + madrid_north = (40.4677, -3.5552) + + distance = LocationService.calculate_distance( + madrid_center[0], madrid_center[1], + madrid_north[0], madrid_north[1] + ) + + assert isinstance(distance, float), "Distance should be float" + assert 0 < distance < 50, "Distance should be reasonable for Madrid area" + + # Test nearest station finding + station_id = LocationService.find_nearest_station(madrid_center[0], madrid_center[1]) + + assert station_id is not None, "Should find a station" + assert station_id in [station.id for station in AEMETConstants.MADRID_STATIONS], "Should be valid station" + + # Test municipality code + municipality = LocationService.get_municipality_code(madrid_center[0], madrid_center[1]) + assert municipality == AEMETConstants.MADRID_MUNICIPALITY_CODE, "Should return Madrid code" + + # Test outside Madrid + outside_madrid = LocationService.get_municipality_code(41.0, -4.0) # Outside bounds + assert outside_madrid is None, "Should return None for outside Madrid" + + print(f"✅ LocationService tests passed") + + @pytest.mark.skip_asyncio + def test_constants_and_enums(self): + """Test constants and enum definitions""" + # Test WeatherSource enum + assert WeatherSource.AEMET.value == "aemet" + assert WeatherSource.SYNTHETIC.value == "synthetic" + assert WeatherSource.DEFAULT.value == "default" + + # Test GeographicBounds + bounds = AEMETConstants.MADRID_BOUNDS + assert bounds.contains(40.4168, -3.7038), "Should contain Madrid center" + assert not bounds.contains(41.0, -4.0), "Should not contain coordinates outside Madrid" + + # Test WeatherStation + station = AEMETConstants.MADRID_STATIONS[0] + assert isinstance(station, WeatherStation), "Should be WeatherStation instance" + assert station.id is not None, "Station should have ID" + assert station.name is not None, "Station should have name" + + print(f"✅ Constants and enums tests passed") + + # ================================================================ + # ERROR HANDLING TESTS + # ================================================================ + + async def test_api_error_handling(self, client, madrid_coords): + """Test handling of various API errors""" + lat, lon = madrid_coords + + # Test network error + with patch.object(client, '_get', side_effect=Exception("Network error")): + result = await client.get_current_weather(lat, lon) + assert result['source'] == WeatherSource.SYNTHETIC.value, "Should fallback on network error" + + # Test invalid API response + with patch.object(client, '_get', new_callable=AsyncMock) as mock_get: + mock_get.return_value = {"error": "Invalid API key"} + result = await client.get_current_weather(lat, lon) + assert result['source'] == WeatherSource.SYNTHETIC.value, "Should fallback on API error" + + # Test malformed data + with patch.object(client, '_get', new_callable=AsyncMock) as mock_get, \ + patch.object(client, '_fetch_from_url', new_callable=AsyncMock) as mock_fetch: + + mock_get.return_value = {"datos": "http://example.com"} + mock_fetch.return_value = [{"invalid": "data"}] # Missing expected fields + + result = await client.get_current_weather(lat, lon) + assert result is not None, "Should handle malformed data gracefully" + + print(f"✅ API error handling tests passed") + + async def test_timeout_handling(self, client, madrid_coords): + """Test timeout handling""" + lat, lon = madrid_coords + + with patch.object(client, '_get', side_effect=asyncio.TimeoutError("Request timeout")): + result = await client.get_current_weather(lat, lon) + assert result['source'] == WeatherSource.SYNTHETIC.value, "Should fallback on timeout" + + print(f"✅ Timeout handling test passed") + + # ================================================================ + # PERFORMANCE TESTS + # ================================================================ + + async def test_performance_current_weather(self, client, madrid_coords): + """Test current weather performance""" + lat, lon = madrid_coords + + start_time = datetime.now() + result = await client.get_current_weather(lat, lon) + execution_time = (datetime.now() - start_time).total_seconds() * 1000 + + assert result is not None, "Should return weather data" + assert execution_time < 5000, "Should execute within 5 seconds" + + print(f"✅ Current weather performance test passed - {execution_time:.0f}ms") + + async def test_performance_forecast(self, client, madrid_coords): + """Test forecast performance""" + lat, lon = madrid_coords + + start_time = datetime.now() + result = await client.get_forecast(lat, lon, 7) + execution_time = (datetime.now() - start_time).total_seconds() * 1000 + + assert isinstance(result, list), "Should return forecast list" + assert len(result) == 7, "Should return 7 days" + assert execution_time < 5000, "Should execute within 5 seconds" + + print(f"✅ Forecast performance test passed - {execution_time:.0f}ms") + + async def test_performance_historical(self, client, madrid_coords): + """Test historical weather performance""" + lat, lon = madrid_coords + end_date = datetime.now() + start_date = end_date - timedelta(days=7) + + start_time = datetime.now() + result = await client.get_historical_weather(lat, lon, start_date, end_date) + execution_time = (datetime.now() - start_time).total_seconds() * 1000 + + assert isinstance(result, list), "Should return historical list" + assert execution_time < 10000, "Should execute within 10 seconds (allowing for API calls)" + + print(f"✅ Historical performance test passed - {execution_time:.0f}ms") + + # ================================================================ + # INTEGRATION TESTS + # ================================================================ + + async def test_real_aemet_api_access(self, client, madrid_coords): + """Test actual AEMET API access (if API key is available)""" + lat, lon = madrid_coords + + try: + # Test current weather + current_result = await client.get_current_weather(lat, lon) + assert current_result is not None, "Should get current weather" + + if current_result['source'] == WeatherSource.AEMET.value: + print(f"🎉 SUCCESS: Got real AEMET current weather data!") + print(f" Temperature: {current_result['temperature']}°C") + print(f" Description: {current_result['description']}") + else: + print(f"ℹ️ Got synthetic current weather (API key may not be configured)") + + # Test forecast + forecast_result = await client.get_forecast(lat, lon, 3) + assert len(forecast_result) == 3, "Should get 3-day forecast" + + if forecast_result[0]['source'] == WeatherSource.AEMET.value: + print(f"🎉 SUCCESS: Got real AEMET forecast data!") + print(f" Tomorrow: {forecast_result[1]['temperature']}°C - {forecast_result[1]['description']}") + else: + print(f"ℹ️ Got synthetic forecast (API key may not be configured)") + + # Test historical (last week) + end_date = datetime.now() + start_date = end_date - timedelta(days=7) + historical_result = await client.get_historical_weather(lat, lon, start_date, end_date) + + assert isinstance(historical_result, list), "Should get historical data" + + real_historical = [r for r in historical_result if r['source'] == WeatherSource.AEMET.value] + if real_historical: + print(f"🎉 SUCCESS: Got real AEMET historical data!") + print(f" Records: {len(real_historical)} real + {len(historical_result) - len(real_historical)} synthetic") + else: + print(f"ℹ️ Got synthetic historical data (API limitations or key issues)") + + print(f"✅ Real AEMET API integration test completed") + + except Exception as e: + print(f"⚠️ AEMET API integration test failed: {e}") + # This is acceptable if API key is not configured + + async def test_data_consistency(self, client, madrid_coords): + """Test data consistency across different methods""" + lat, lon = madrid_coords + + # Get current weather + current = await client.get_current_weather(lat, lon) + + # Get today's forecast + forecast = await client.get_forecast(lat, lon, 1) + today_forecast = forecast[0] if forecast else None + + if current and today_forecast: + # Temperature should be somewhat consistent + temp_diff = abs(current['temperature'] - today_forecast['temperature']) + assert temp_diff < 15, "Current and forecast temperature should be reasonably consistent" + + # Both should use same source type preference + if current['source'] == WeatherSource.AEMET.value: + assert today_forecast['source'] == WeatherSource.AEMET.value, "Should use consistent data sources" + + print(f"✅ Data consistency test passed") + + +# ================================================================ +# STANDALONE TEST FUNCTIONS +# ================================================================ + +async def run_manual_test(): + """Manual test function that can be run directly""" + print("="*60) + print("AEMET WEATHER CLIENT TEST - JULY 2025") + print("="*60) + + client = AEMETClient() + madrid_lat, madrid_lon = 40.4168, -3.7038 # Madrid center + + print(f"\n=== Testing Madrid Weather ({madrid_lat}, {madrid_lon}) ===") + + # Test current weather + print(f"\n1. Testing Current Weather...") + current = await client.get_current_weather(madrid_lat, madrid_lon) + if current: + print(f" Temperature: {current['temperature']}°C") + print(f" Description: {current['description']}") + print(f" Humidity: {current['humidity']}%") + print(f" Wind: {current['wind_speed']} km/h") + print(f" Source: {current['source']}") + + # Test forecast + print(f"\n2. Testing 7-Day Forecast...") + forecast = await client.get_forecast(madrid_lat, madrid_lon, 7) + if forecast: + print(f" Forecast days: {len(forecast)}") + print(f" Tomorrow: {forecast[1]['temperature']}°C - {forecast[1]['description']}") + print(f" Source: {forecast[0]['source']}") + + # Test historical + print(f"\n3. Testing Historical Weather (last 7 days)...") + end_date = datetime.now() + start_date = end_date - timedelta(days=7) + historical = await client.get_historical_weather(madrid_lat, madrid_lon, start_date, end_date) + if historical: + print(f" Historical records: {len(historical)}") + if historical: + real_count = len([r for r in historical if r['source'] == WeatherSource.AEMET.value]) + synthetic_count = len(historical) - real_count + print(f" Real data: {real_count}, Synthetic: {synthetic_count}") + + print(f"\n✅ Manual test completed!") + + +if __name__ == "__main__": + # If run directly, execute manual test + asyncio.run(run_manual_test()) \ No newline at end of file diff --git a/services/data/tests/test_aemet_edge_cases.py b/services/data/tests/test_aemet_edge_cases.py new file mode 100644 index 00000000..2dd21b69 --- /dev/null +++ b/services/data/tests/test_aemet_edge_cases.py @@ -0,0 +1,594 @@ +# ================================================================ +# services/data/tests/test_aemet_edge_cases.py +# ================================================================ +""" +Edge cases and integration tests for AEMET weather API client +Covers boundary conditions, error scenarios, and complex integrations +""" + +import pytest +import asyncio +from datetime import datetime, timedelta +from unittest.mock import Mock, patch, AsyncMock +import json +from typing import Dict, List, Any + +from app.external.aemet import ( + AEMETClient, + WeatherDataParser, + SyntheticWeatherGenerator, + LocationService, + AEMETConstants, + WeatherSource +) + +# Configure pytest-asyncio +pytestmark = pytest.mark.asyncio + + +class TestAEMETEdgeCases: + """Test edge cases and boundary conditions""" + + async def test_extreme_coordinates(self, aemet_client): + """Test handling of extreme coordinate values""" + extreme_coords = [ + (90, 180), # North pole, antimeridian + (-90, -180), # South pole, antimeridian + (0, 0), # Null island + (40.5, -180), # Valid latitude, extreme longitude + (90, -3.7), # Extreme latitude, Madrid longitude + ] + + for lat, lon in extreme_coords: + result = await aemet_client.get_current_weather(lat, lon) + + assert result is not None, f"Should handle extreme coords ({lat}, {lon})" + assert result['source'] == WeatherSource.SYNTHETIC.value, "Should fallback to synthetic for extreme coords" + assert isinstance(result['temperature'], (int, float)), "Should have valid temperature" + + async def test_boundary_date_ranges(self, aemet_client, madrid_coords): + """Test boundary conditions for date ranges""" + lat, lon = madrid_coords + now = datetime.now() + + # Test same start and end date + result = await aemet_client.get_historical_weather(lat, lon, now, now) + assert isinstance(result, list), "Should return list for same-day request" + + # Test reverse date range (end before start) + start_date = now + end_date = now - timedelta(days=1) + result = await aemet_client.get_historical_weather(lat, lon, start_date, end_date) + assert isinstance(result, list), "Should handle reverse date range gracefully" + + # Test extremely large date range + start_date = now - timedelta(days=1000) + end_date = now + result = await aemet_client.get_historical_weather(lat, lon, start_date, end_date) + assert isinstance(result, list), "Should handle very large date ranges" + + async def test_forecast_edge_durations(self, aemet_client, madrid_coords): + """Test forecast with edge case durations""" + lat, lon = madrid_coords + + edge_durations = [0, 1, 30, 365, -1, 1000] + + for days in edge_durations: + try: + result = await aemet_client.get_forecast(lat, lon, days) + + if days <= 0: + assert len(result) == 0 or result is None, f"Should handle non-positive days ({days})" + elif days > 100: + # Should handle gracefully, possibly with synthetic data + assert isinstance(result, list), f"Should handle large day count ({days})" + else: + assert len(result) == days, f"Should return {days} forecast days" + + except Exception as e: + # Some edge cases might raise exceptions, which is acceptable + print(f"ℹ️ Days={days} raised exception: {e}") + + def test_parser_edge_cases(self, weather_parser): + """Test weather data parser with edge case inputs""" + # Test with None values + result = weather_parser.safe_float(None, 10.0) + assert result == 10.0, "Should return default for None" + + # Test with empty strings + result = weather_parser.safe_float("", 5.0) + assert result == 5.0, "Should return default for empty string" + + # Test with extreme values + result = weather_parser.safe_float("999999.99", 0.0) + assert result == 999999.99, "Should handle large numbers" + + result = weather_parser.safe_float("-999.99", 0.0) + assert result == -999.99, "Should handle negative numbers" + + # Test temperature extraction edge cases + assert weather_parser.extract_temperature_value([]) is None, "Should handle empty list" + assert weather_parser.extract_temperature_value({}) is None, "Should handle empty dict" + assert weather_parser.extract_temperature_value("invalid") is None, "Should handle invalid string" + + def test_synthetic_generator_edge_cases(self, synthetic_generator): + """Test synthetic weather generator edge cases""" + # Test with extreme date ranges + end_date = datetime.now() + start_date = end_date - timedelta(days=1000) + + result = synthetic_generator.generate_historical_data(start_date, end_date) + assert isinstance(result, list), "Should handle large date ranges" + assert len(result) == 1001, "Should generate correct number of days" + + # Test forecast with zero days + result = synthetic_generator.generate_forecast_sync(0) + assert result == [], "Should return empty list for zero days" + + # Test forecast with large number of days + result = synthetic_generator.generate_forecast_sync(1000) + assert len(result) == 1000, "Should handle large forecast ranges" + + def test_location_service_edge_cases(self): + """Test location service edge cases""" + # Test distance calculation with same points + distance = LocationService.calculate_distance(40.4, -3.7, 40.4, -3.7) + assert distance == 0.0, "Distance between same points should be zero" + + # Test distance calculation with antipodal points + distance = LocationService.calculate_distance(40.4, -3.7, -40.4, 176.3) + assert distance > 15000, "Antipodal points should be far apart" + + # Test station finding with no stations (if list were empty) + with patch.object(AEMETConstants, 'MADRID_STATIONS', []): + station = LocationService.find_nearest_station(40.4, -3.7) + assert station is None, "Should return None when no stations available" + + +class TestAEMETDataIntegrity: + """Test data integrity and consistency""" + + async def test_data_type_consistency(self, aemet_client, madrid_coords): + """Test that data types are consistent across calls""" + lat, lon = madrid_coords + + # Get current weather multiple times + results = [] + for _ in range(3): + result = await aemet_client.get_current_weather(lat, lon) + results.append(result) + + # Check that field types are consistent + if all(r is not None for r in results): + for field in ['temperature', 'precipitation', 'humidity', 'wind_speed', 'pressure']: + types = [type(r[field]) for r in results if field in r] + if types: + first_type = types[0] + assert all(t == first_type for t in types), f"Inconsistent types for {field}: {types}" + + async def test_temperature_consistency(self, aemet_client, madrid_coords): + """Test temperature consistency between different data sources""" + lat, lon = madrid_coords + + # Get current weather and today's forecast + current = await aemet_client.get_current_weather(lat, lon) + forecast = await aemet_client.get_forecast(lat, lon, 1) + + if current and forecast and len(forecast) > 0: + current_temp = current['temperature'] + forecast_temp = forecast[0]['temperature'] + + # Temperatures should be reasonably close (within 15°C) + temp_diff = abs(current_temp - forecast_temp) + assert temp_diff < 15, f"Temperature difference too large: current={current_temp}°C, forecast={forecast_temp}°C" + + async def test_source_consistency(self, aemet_client, madrid_coords): + """Test that data source is consistent within same time period""" + lat, lon = madrid_coords + + # Get multiple current weather readings + current1 = await aemet_client.get_current_weather(lat, lon) + current2 = await aemet_client.get_current_weather(lat, lon) + + if current1 and current2: + # Should use same source type (both real or both synthetic) + assert current1['source'] == current2['source'], "Should use consistent data source" + + def test_historical_data_ordering(self, weather_parser, mock_historical_data): + """Test that historical data is properly ordered""" + parsed_data = weather_parser.parse_historical_data(mock_historical_data) + + if len(parsed_data) > 1: + dates = [record['date'] for record in parsed_data] + assert dates == sorted(dates), "Historical data should be chronologically ordered" + + def test_forecast_date_progression(self, weather_parser, mock_forecast_data): + """Test that forecast dates progress correctly""" + parsed_forecast = weather_parser.parse_forecast_data(mock_forecast_data, 7) + + if len(parsed_forecast) > 1: + for i in range(1, len(parsed_forecast)): + prev_date = parsed_forecast[i-1]['forecast_date'] + curr_date = parsed_forecast[i]['forecast_date'] + diff = (curr_date - prev_date).days + assert diff == 1, f"Forecast dates should be consecutive days, got {diff} day difference" + + +class TestAEMETErrorRecovery: + """Test error recovery and resilience""" + + async def test_network_interruption_recovery(self, aemet_client, madrid_coords): + """Test recovery from network interruptions""" + lat, lon = madrid_coords + + # Mock intermittent network failures + call_count = 0 + + async def mock_get_with_failures(*args, **kwargs): + nonlocal call_count + call_count += 1 + if call_count <= 2: # Fail first two calls + raise Exception("Network timeout") + else: + return {"datos": "http://example.com/data"} + + with patch.object(aemet_client, '_get', side_effect=mock_get_with_failures): + result = await aemet_client.get_current_weather(lat, lon) + + # Should eventually succeed or fallback to synthetic + assert result is not None, "Should recover from network failures" + assert result['source'] in [WeatherSource.AEMET.value, WeatherSource.SYNTHETIC.value] + + async def test_partial_data_recovery(self, aemet_client, madrid_coords, weather_parser): + """Test recovery from partial/corrupted data""" + lat, lon = madrid_coords + + # Mock corrupted historical data (some records missing fields) + corrupted_data = [ + {"fecha": "2025-07-20", "tmax": 25.2}, # Missing tmin and other fields + {"fecha": "2025-07-21"}, # Only has date + {"tmax": 27.0, "tmin": 15.0}, # Missing date + {"fecha": "2025-07-22", "tmax": 23.0, "tmin": 14.0, "prec": 0.0} # Complete record + ] + + parsed_data = weather_parser.parse_historical_data(corrupted_data) + + # Should only return valid records and handle corrupted ones gracefully + assert isinstance(parsed_data, list), "Should return list even with corrupted data" + valid_records = [r for r in parsed_data if 'date' in r and r['date'] is not None] + assert len(valid_records) >= 1, "Should salvage at least some valid records" + + async def test_malformed_json_recovery(self, aemet_client, madrid_coords): + """Test recovery from malformed JSON responses""" + lat, lon = madrid_coords + + # Mock malformed responses + malformed_responses = [ + None, + "", + "invalid json", + {"incomplete": "response"}, + {"datos": None}, + {"datos": ""}, + ] + + for response in malformed_responses: + with patch.object(aemet_client, '_get', new_callable=AsyncMock, return_value=response): + result = await aemet_client.get_current_weather(lat, lon) + + assert result is not None, f"Should handle malformed response: {response}" + assert result['source'] == WeatherSource.SYNTHETIC.value, "Should fallback to synthetic" + + async def test_api_rate_limiting_recovery(self, aemet_client, madrid_coords): + """Test recovery from API rate limiting""" + lat, lon = madrid_coords + + # Mock rate limiting responses + rate_limit_response = { + "descripcion": "Demasiadas peticiones", + "estado": 429 + } + + with patch.object(aemet_client, '_get', new_callable=AsyncMock, return_value=rate_limit_response): + result = await aemet_client.get_current_weather(lat, lon) + + assert result is not None, "Should handle rate limiting" + assert result['source'] == WeatherSource.SYNTHETIC.value, "Should fallback to synthetic on rate limit" + + +class TestAEMETPerformanceAndScaling: + """Test performance characteristics and scaling behavior""" + + async def test_concurrent_requests_performance(self, aemet_client, madrid_coords): + """Test performance with concurrent requests""" + lat, lon = madrid_coords + + # Create multiple concurrent requests + tasks = [] + for i in range(10): + task = aemet_client.get_current_weather(lat, lon) + tasks.append(task) + + start_time = datetime.now() + results = await asyncio.gather(*tasks, return_exceptions=True) + execution_time = (datetime.now() - start_time).total_seconds() * 1000 + + # Check that most requests succeeded + successful_results = [r for r in results if isinstance(r, dict) and 'temperature' in r] + assert len(successful_results) >= 8, "Most concurrent requests should succeed" + + # Should complete in reasonable time (allowing for potential API rate limiting) + assert execution_time < 15000, f"Concurrent requests took too long: {execution_time:.0f}ms" + + print(f"✅ Concurrent requests test - {len(successful_results)}/10 succeeded in {execution_time:.0f}ms") + + async def test_memory_usage_with_large_datasets(self, aemet_client, madrid_coords): + """Test memory usage with large historical datasets""" + lat, lon = madrid_coords + + # Request large historical dataset + end_date = datetime.now() + start_date = end_date - timedelta(days=90) # 3 months + + import psutil + import os + + # Get initial memory usage + process = psutil.Process(os.getpid()) + initial_memory = process.memory_info().rss / 1024 / 1024 # MB + + result = await aemet_client.get_historical_weather(lat, lon, start_date, end_date) + + # Get final memory usage + final_memory = process.memory_info().rss / 1024 / 1024 # MB + memory_increase = final_memory - initial_memory + + assert isinstance(result, list), "Should return historical data" + + # Memory increase should be reasonable (less than 100MB for 90 days) + assert memory_increase < 100, f"Memory usage increased too much: {memory_increase:.1f}MB" + + print(f"✅ Memory usage test - {len(result)} records, +{memory_increase:.1f}MB") + + async def test_caching_behavior(self, aemet_client, madrid_coords): + """Test caching behavior and performance improvement""" + lat, lon = madrid_coords + + # First request (cold) + start_time = datetime.now() + result1 = await aemet_client.get_current_weather(lat, lon) + first_call_time = (datetime.now() - start_time).total_seconds() * 1000 + + # Second request (potentially cached) + start_time = datetime.now() + result2 = await aemet_client.get_current_weather(lat, lon) + second_call_time = (datetime.now() - start_time).total_seconds() * 1000 + + assert result1 is not None, "First call should succeed" + assert result2 is not None, "Second call should succeed" + + # Both should return valid data + assert 'temperature' in result1, "First result should have temperature" + assert 'temperature' in result2, "Second result should have temperature" + + print(f"✅ Caching test - First call: {first_call_time:.0f}ms, Second call: {second_call_time:.0f}ms") + + +class TestAEMETIntegrationScenarios: + """Test realistic integration scenarios""" + + async def test_daily_weather_workflow(self, aemet_client, madrid_coords): + """Test a complete daily weather workflow""" + lat, lon = madrid_coords + + # Simulate a daily weather check workflow + workflow_results = {} + + # Step 1: Get current conditions + current = await aemet_client.get_current_weather(lat, lon) + workflow_results['current'] = current + assert current is not None, "Should get current weather" + + # Step 2: Get today's forecast + forecast = await aemet_client.get_forecast(lat, lon, 1) + workflow_results['forecast'] = forecast + assert len(forecast) == 1, "Should get today's forecast" + + # Step 3: Get week ahead forecast + week_forecast = await aemet_client.get_forecast(lat, lon, 7) + workflow_results['week_forecast'] = week_forecast + assert len(week_forecast) == 7, "Should get 7-day forecast" + + # Step 4: Get last week's actual weather for comparison + end_date = datetime.now() - timedelta(days=1) + start_date = end_date - timedelta(days=7) + historical = await aemet_client.get_historical_weather(lat, lon, start_date, end_date) + workflow_results['historical'] = historical + assert isinstance(historical, list), "Should get historical data" + + # Validate workflow consistency + all_sources = set() + if current: all_sources.add(current['source']) + if forecast: all_sources.add(forecast[0]['source']) + if week_forecast: all_sources.add(week_forecast[0]['source']) + if historical: all_sources.update([h['source'] for h in historical]) + + print(f"✅ Daily workflow test - Sources used: {', '.join(all_sources)}") + + return workflow_results + + async def test_weather_alerting_scenario(self, aemet_client, madrid_coords): + """Test weather alerting scenario""" + lat, lon = madrid_coords + + # Get forecast for potential alerts + forecast = await aemet_client.get_forecast(lat, lon, 3) + + alerts = [] + for day in forecast: + # Check for extreme temperatures + if day['temperature'] > 35: + alerts.append(f"High temperature alert: {day['temperature']}°C on {day['forecast_date'].date()}") + elif day['temperature'] < -5: + alerts.append(f"Low temperature alert: {day['temperature']}°C on {day['forecast_date'].date()}") + + # Check for high precipitation + if day['precipitation'] > 20: + alerts.append(f"Heavy rain alert: {day['precipitation']}mm on {day['forecast_date'].date()}") + + # Alerts should be properly formatted + for alert in alerts: + assert isinstance(alert, str), "Alert should be string" + assert "alert" in alert.lower(), "Alert should contain 'alert'" + + print(f"✅ Weather alerting test - {len(alerts)} alerts generated") + + return alerts + + async def test_historical_analysis_scenario(self, aemet_client, madrid_coords): + """Test historical weather analysis scenario""" + lat, lon = madrid_coords + + # Get historical data for analysis + end_date = datetime.now() + start_date = end_date - timedelta(days=30) + + historical = await aemet_client.get_historical_weather(lat, lon, start_date, end_date) + + if historical: + # Calculate statistics + temperatures = [h['temperature'] for h in historical if h['temperature'] is not None] + precipitations = [h['precipitation'] for h in historical if h['precipitation'] is not None] + + if temperatures: + avg_temp = sum(temperatures) / len(temperatures) + max_temp = max(temperatures) + min_temp = min(temperatures) + + # Validate statistics + assert min_temp <= avg_temp <= max_temp, "Temperature statistics should be logical" + assert -20 <= min_temp <= 50, "Min temperature should be reasonable" + assert -20 <= max_temp <= 50, "Max temperature should be reasonable" + + if precipitations: + total_precip = sum(precipitations) + rainy_days = len([p for p in precipitations if p > 0.1]) + + # Validate precipitation statistics + assert total_precip >= 0, "Total precipitation should be non-negative" + assert 0 <= rainy_days <= len(precipitations), "Rainy days should be reasonable" + + print(f"✅ Historical analysis test - {len(historical)} records analyzed") + + return { + 'record_count': len(historical), + 'avg_temp': avg_temp if temperatures else None, + 'temp_range': (min_temp, max_temp) if temperatures else None, + 'total_precip': total_precip if precipitations else None, + 'rainy_days': rainy_days if precipitations else None + } + + return {} + + +class TestAEMETRegressionTests: + """Regression tests for previously fixed issues""" + + async def test_timezone_handling_regression(self, aemet_client, madrid_coords): + """Regression test for timezone handling issues""" + lat, lon = madrid_coords + + # Get current weather and forecast + current = await aemet_client.get_current_weather(lat, lon) + forecast = await aemet_client.get_forecast(lat, lon, 2) + + if current: + # Current weather date should be recent (within last hour) + now = datetime.now() + time_diff = abs((now - current['date']).total_seconds()) + assert time_diff < 3600, "Current weather timestamp should be recent" + + if forecast: + # Forecast dates should be in the future + now = datetime.now().date() + for day in forecast: + forecast_date = day['forecast_date'].date() + assert forecast_date >= now, f"Forecast date {forecast_date} should be today or future" + + async def test_data_type_conversion_regression(self, weather_parser): + """Regression test for data type conversion issues""" + # Test cases that previously caused issues + test_cases = [ + ("25.5", 25.5), # String to float + (25, 25.0), # Int to float + ("", None), # Empty string + ("invalid", None), # Invalid string + (None, None), # None input + ] + + for input_val, expected in test_cases: + result = weather_parser.safe_float(input_val, None) + if expected is None: + assert result is None, f"Expected None for input {input_val}, got {result}" + else: + assert result == expected, f"Expected {expected} for input {input_val}, got {result}" + + def test_empty_data_handling_regression(self, weather_parser): + """Regression test for empty data handling""" + # Empty lists and dictionaries should be handled gracefully + empty_data_cases = [ + [], + [{}], + [{"invalid": "data"}], + None, + ] + + for empty_data in empty_data_cases: + result = weather_parser.parse_historical_data(empty_data if empty_data is not None else []) + assert isinstance(result, list), f"Should return list for empty data: {empty_data}" + # May be empty or have some synthetic data, but should not crash + + +# ================================================================ +# STANDALONE TEST RUNNER FOR EDGE CASES +# ================================================================ + +async def run_edge_case_tests(): + """Run edge case tests manually""" + print("="*60) + print("AEMET EDGE CASE TESTS") + print("="*60) + + client = AEMETClient() + parser = WeatherDataParser() + generator = SyntheticWeatherGenerator() + + madrid_coords = (40.4168, -3.7038) + + print(f"\n1. Testing extreme coordinates...") + extreme_result = await client.get_current_weather(90, 180) + print(f" Extreme coords result: {extreme_result['source']} source") + + print(f"\n2. Testing parser edge cases...") + parser_tests = [ + parser.safe_float(None, 10.0), + parser.safe_float("invalid", 5.0), + parser.extract_temperature_value([]), + ] + print(f" Parser edge cases passed: {len(parser_tests)}") + + print(f"\n3. Testing synthetic generator extremes...") + large_forecast = generator.generate_forecast_sync(100) + print(f" Generated {len(large_forecast)} forecast days") + + print(f"\n4. Testing concurrent requests...") + tasks = [client.get_current_weather(*madrid_coords) for _ in range(5)] + concurrent_results = await asyncio.gather(*tasks, return_exceptions=True) + successful = len([r for r in concurrent_results if isinstance(r, dict)]) + print(f" Concurrent requests: {successful}/5 successful") + + print(f"\n✅ Edge case tests completed!") + + +if __name__ == "__main__": + asyncio.run(run_edge_case_tests()) \ No newline at end of file diff --git a/services/training/app/services/training_service.py b/services/training/app/services/training_service.py index 905ad886..35ef834c 100644 --- a/services/training/app/services/training_service.py +++ b/services/training/app/services/training_service.py @@ -487,7 +487,7 @@ class TrainingService: params["end_date"] = request.end_date.isoformat() response = await client.get( - f"{settings.DATA_SERVICE_URL}/api/weather", + f"{settings.DATA_SERVICE_URL}/weather/history", params=params, timeout=30.0 ) @@ -515,7 +515,7 @@ class TrainingService: params["end_date"] = request.end_date.isoformat() response = await client.get( - f"{settings.DATA_SERVICE_URL}/api/traffic", + f"{settings.DATA_SERVICE_URL}/traffic/historical", params=params, timeout=30.0 )