diff --git a/services/external/app/external/aemet.py b/services/external/app/external/aemet.py index 8840131e..3c406204 100644 --- a/services/external/app/external/aemet.py +++ b/services/external/app/external/aemet.py @@ -136,16 +136,36 @@ class WeatherDataParser: return self._get_default_weather_data() try: - return { + # Log the raw data to understand the structure + logger.debug("Parsing current weather data", data_keys=list(data.keys()) if isinstance(data, dict) else "not_dict") + + # Enhanced parsing for AEMET station data + result = { "date": datetime.now(), - "temperature": self.safe_float(data.get("ta"), 15.0), - "precipitation": self.safe_float(data.get("prec"), 0.0), - "humidity": self.safe_float(data.get("hr"), 50.0), - "wind_speed": self.safe_float(data.get("vv"), 10.0), - "pressure": self.safe_float(data.get("pres"), 1013.0), - "description": str(data.get("descripcion", "Partly cloudy")), + "temperature": self.safe_float(data.get("ta"), self.safe_float(data.get("temperature"))), + "precipitation": self.safe_float(data.get("prec"), self.safe_float(data.get("precipitation"), 0.0)), + "humidity": self.safe_float(data.get("hr"), self.safe_float(data.get("humidity"))), + "wind_speed": self.safe_float(data.get("vv"), self.safe_float(data.get("wind_speed"))), + "pressure": self.safe_float(data.get("pres"), self.safe_float(data.get("pressure"))), + "description": str(data.get("descripcion", data.get("description", "Partly cloudy"))), "source": WeatherSource.AEMET.value } + + # Set fallback values for required fields that are None + if result["temperature"] is None: + result["temperature"] = 15.0 + if result["humidity"] is None: + result["humidity"] = 50.0 + if result["wind_speed"] is None: + result["wind_speed"] = 10.0 + if result["pressure"] is None: + result["pressure"] = 1013.0 + + logger.debug("Parsed current weather successfully", + temperature=result["temperature"], + source=result["source"]) + + return result except Exception as e: logger.error("Error parsing weather data", error=str(e), data=data) return self._get_default_weather_data() @@ -204,22 +224,29 @@ class WeatherDataParser: return [] try: + # AEMET hourly forecast has different structure than daily + logger.debug("Processing AEMET hourly forecast data", data_length=len(data)) + if len(data) > 0 and isinstance(data[0], dict): aemet_data = data[0] prediccion = aemet_data.get('prediccion', {}) + + # AEMET hourly structure: prediccion -> dia[] -> each day has hourly arrays dias = prediccion.get('dia', []) if isinstance(dias, list): - hourly_forecast = self._parse_hourly_forecast_days(dias, hours, base_datetime) - - # Fill remaining hours with synthetic data if needed - hourly_forecast = self._ensure_hourly_forecast_completeness(hourly_forecast, hours) + hourly_forecast = self._parse_real_hourly_forecast_days(dias, hours, base_datetime) + logger.info("Successfully parsed AEMET hourly forecast", count=len(hourly_forecast)) + + if len(hourly_forecast) == 0: + logger.warning("No hourly data parsed from AEMET response") except Exception as e: logger.error("Error parsing AEMET hourly forecast data", error=str(e)) hourly_forecast = [] - return hourly_forecast + return hourly_forecast[:hours] + def _parse_single_historical_record(self, record: Dict[str, Any]) -> Optional[Dict[str, Any]]: """Parse a single historical weather record""" @@ -351,8 +378,121 @@ class WeatherDataParser: else: return "Soleado" + def _parse_real_hourly_forecast_days(self, dias: List[Dict[str, Any]], hours: int, base_datetime: datetime) -> List[Dict[str, Any]]: + """Parse real AEMET hourly forecast days""" + hourly_forecast = [] + current_hour = 0 + + for day_index, day_data in enumerate(dias): + if current_hour >= hours: + break + + if not isinstance(day_data, dict): + continue + + # Parse hourly data from this day using real AEMET structure + day_hourly = self._parse_real_aemet_hourly_day(day_data, base_datetime, day_index, current_hour, hours) + hourly_forecast.extend(day_hourly) + current_hour += len(day_hourly) + + return hourly_forecast[:hours] + + def _parse_real_aemet_hourly_day(self, day_data: Dict[str, Any], base_datetime: datetime, day_index: int, start_hour: int, max_hours: int) -> List[Dict[str, Any]]: + """Parse hourly data for a single day from real AEMET hourly API response""" + hourly_data = [] + + # Extract fecha (date) for this day + fecha_str = day_data.get('fecha', '') + + try: + if fecha_str: + day_date = datetime.strptime(fecha_str, '%Y-%m-%d') + else: + day_date = base_datetime.replace(hour=0, minute=0, second=0, microsecond=0) + timedelta(days=day_index) + except ValueError: + day_date = base_datetime.replace(hour=0, minute=0, second=0, microsecond=0) + timedelta(days=day_index) + + # Extract hourly arrays from AEMET structure + temperatura = day_data.get('temperatura', []) + precipitacion = day_data.get('precipitacion', []) + humedadRelativa = day_data.get('humedadRelativa', []) + viento = day_data.get('viento', []) + + # Process available hourly data + max_available_hours = min( + len(temperatura) if isinstance(temperatura, list) else 0, + len(precipitacion) if isinstance(precipitacion, list) else 0, + 24, # Max 24 hours per day + max_hours - start_hour + ) + + if max_available_hours == 0: + logger.warning("No hourly data arrays found in day data", fecha=fecha_str) + return [] + + for hour_idx in range(max_available_hours): + forecast_datetime = day_date + timedelta(hours=hour_idx) + + # Extract hourly values safely + temp = self._extract_aemet_hourly_value(temperatura, hour_idx) + precip = self._extract_aemet_hourly_value(precipitacion, hour_idx, default=0.0) + humidity = self._extract_aemet_hourly_value(humedadRelativa, hour_idx) + wind_data = viento[hour_idx] if isinstance(viento, list) and hour_idx < len(viento) else {} + wind_speed = self._extract_wind_from_aemet_data(wind_data) + + hourly_data.append({ + "forecast_datetime": forecast_datetime, + "generated_at": datetime.now(), + "temperature": temp if temp is not None else 15.0, + "precipitation": precip if precip is not None else 0.0, + "humidity": humidity if humidity is not None else 50.0, + "wind_speed": wind_speed if wind_speed is not None else 10.0, + "description": self._generate_hourly_description(temp, precip), + "source": WeatherSource.AEMET.value, + "hour": forecast_datetime.hour + }) + + logger.debug("Parsed AEMET hourly day", fecha=fecha_str, hours_parsed=len(hourly_data)) + return hourly_data + + def _extract_aemet_hourly_value(self, data_array: Any, hour_idx: int, default: Optional[float] = None) -> Optional[float]: + """Extract hourly value from AEMET hourly data array""" + if not isinstance(data_array, list) or hour_idx >= len(data_array): + return default + + value_data = data_array[hour_idx] + + # Handle different AEMET data formats + if isinstance(value_data, (int, float)): + return float(value_data) + elif isinstance(value_data, dict): + # Try common AEMET field names + for field in ['value', 'valor', 'v']: + if field in value_data: + return self.safe_float(value_data[field], default) + elif isinstance(value_data, str): + return self.safe_float(value_data, default) + + return default + + def _extract_wind_from_aemet_data(self, wind_data: Any) -> Optional[float]: + """Extract wind speed from AEMET wind data structure""" + if isinstance(wind_data, dict): + # Try common wind speed fields + for field in ['velocidad', 'speed', 'v']: + if field in wind_data: + velocidad = wind_data[field] + if isinstance(velocidad, list) and len(velocidad) > 0: + return self.safe_float(velocidad[0]) + else: + return self.safe_float(velocidad) + elif isinstance(wind_data, (int, float)): + return float(wind_data) + + return None + def _parse_hourly_forecast_days(self, dias: List[Dict[str, Any]], hours: int, base_datetime: datetime) -> List[Dict[str, Any]]: - """Parse hourly forecast days from AEMET data""" + """Parse hourly forecast days from AEMET data (legacy method)""" hourly_forecast = [] current_hour = 0 @@ -453,6 +593,7 @@ class WeatherDataParser: """Return forecast as is - no synthetic data filling""" return forecast[:days] + def _ensure_hourly_forecast_completeness(self, forecast: List[Dict[str, Any]], hours: int) -> List[Dict[str, Any]]: """Return hourly forecast as is - no synthetic data filling""" return forecast[:hours] @@ -630,19 +771,20 @@ class AEMETClient(BaseAPIClient): logger.info("✅ Found municipality code", municipality_code=municipality_code) + # Get real hourly data from AEMET API hourly_data = await self._fetch_hourly_forecast_data(municipality_code) if hourly_data: logger.info("🎉 SUCCESS: Real AEMET hourly data retrieved!", municipality_code=municipality_code) parsed_data = self.parser.parse_hourly_forecast_data(hourly_data, hours) - if parsed_data: + if parsed_data and len(parsed_data) > 0: return parsed_data - logger.error("⚠️ AEMET hourly API connectivity issues", - municipality_code=municipality_code, reason="aemet_hourly_api_unreachable") + logger.error("⚠️ AEMET hourly API failed or returned no data", + municipality_code=municipality_code) return [] except Exception as e: - logger.error("❌ AEMET hourly API failed", + logger.error("❌ AEMET hourly forecast failed", error=str(e), error_type=type(e).__name__) return [] @@ -709,13 +851,20 @@ class AEMETClient(BaseAPIClient): async def _fetch_hourly_forecast_data(self, municipality_code: str) -> Optional[List[Dict[str, Any]]]: """Fetch hourly forecast data from AEMET API""" + # Note: AEMET hourly forecast API endpoint endpoint = f"/prediccion/especifica/municipio/horaria/{municipality_code}" + logger.info("Requesting AEMET hourly forecast", endpoint=endpoint, municipality=municipality_code) + initial_response = await self._get(endpoint) if not self._is_valid_initial_response(initial_response): + logger.warning("Invalid initial response from AEMET hourly API", + response=initial_response, municipality=municipality_code) return None datos_url = initial_response.get("datos") + logger.info("Fetching hourly data from AEMET datos URL", url=datos_url) + return await self._fetch_from_url(datos_url) async def _fetch_historical_data_in_chunks(self,