Improve the dahboard 5
This commit is contained in:
Urtzi Alfaro
181
services/external/app/external/aemet.py
vendored
181
services/external/app/external/aemet.py
vendored
@@ -136,16 +136,36 @@ class WeatherDataParser:
|
||||
return self._get_default_weather_data()
|
||||
|
||||
try:
|
||||
return {
|
||||
# Log the raw data to understand the structure
|
||||
logger.debug("Parsing current weather data", data_keys=list(data.keys()) if isinstance(data, dict) else "not_dict")
|
||||
|
||||
# Enhanced parsing for AEMET station data
|
||||
result = {
|
||||
"date": datetime.now(),
|
||||
"temperature": self.safe_float(data.get("ta"), 15.0),
|
||||
"precipitation": self.safe_float(data.get("prec"), 0.0),
|
||||
"humidity": self.safe_float(data.get("hr"), 50.0),
|
||||
"wind_speed": self.safe_float(data.get("vv"), 10.0),
|
||||
"pressure": self.safe_float(data.get("pres"), 1013.0),
|
||||
"description": str(data.get("descripcion", "Partly cloudy")),
|
||||
"temperature": self.safe_float(data.get("ta"), self.safe_float(data.get("temperature"))),
|
||||
"precipitation": self.safe_float(data.get("prec"), self.safe_float(data.get("precipitation"), 0.0)),
|
||||
"humidity": self.safe_float(data.get("hr"), self.safe_float(data.get("humidity"))),
|
||||
"wind_speed": self.safe_float(data.get("vv"), self.safe_float(data.get("wind_speed"))),
|
||||
"pressure": self.safe_float(data.get("pres"), self.safe_float(data.get("pressure"))),
|
||||
"description": str(data.get("descripcion", data.get("description", "Partly cloudy"))),
|
||||
"source": WeatherSource.AEMET.value
|
||||
}
|
||||
|
||||
# Set fallback values for required fields that are None
|
||||
if result["temperature"] is None:
|
||||
result["temperature"] = 15.0
|
||||
if result["humidity"] is None:
|
||||
result["humidity"] = 50.0
|
||||
if result["wind_speed"] is None:
|
||||
result["wind_speed"] = 10.0
|
||||
if result["pressure"] is None:
|
||||
result["pressure"] = 1013.0
|
||||
|
||||
logger.debug("Parsed current weather successfully",
|
||||
temperature=result["temperature"],
|
||||
source=result["source"])
|
||||
|
||||
return result
|
||||
except Exception as e:
|
||||
logger.error("Error parsing weather data", error=str(e), data=data)
|
||||
return self._get_default_weather_data()
|
||||
@@ -204,22 +224,29 @@ class WeatherDataParser:
|
||||
return []
|
||||
|
||||
try:
|
||||
# AEMET hourly forecast has different structure than daily
|
||||
logger.debug("Processing AEMET hourly forecast data", data_length=len(data))
|
||||
|
||||
if len(data) > 0 and isinstance(data[0], dict):
|
||||
aemet_data = data[0]
|
||||
prediccion = aemet_data.get('prediccion', {})
|
||||
|
||||
# AEMET hourly structure: prediccion -> dia[] -> each day has hourly arrays
|
||||
dias = prediccion.get('dia', [])
|
||||
|
||||
if isinstance(dias, list):
|
||||
hourly_forecast = self._parse_hourly_forecast_days(dias, hours, base_datetime)
|
||||
hourly_forecast = self._parse_real_hourly_forecast_days(dias, hours, base_datetime)
|
||||
logger.info("Successfully parsed AEMET hourly forecast", count=len(hourly_forecast))
|
||||
|
||||
# Fill remaining hours with synthetic data if needed
|
||||
hourly_forecast = self._ensure_hourly_forecast_completeness(hourly_forecast, hours)
|
||||
if len(hourly_forecast) == 0:
|
||||
logger.warning("No hourly data parsed from AEMET response")
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Error parsing AEMET hourly forecast data", error=str(e))
|
||||
hourly_forecast = []
|
||||
|
||||
return hourly_forecast
|
||||
return hourly_forecast[:hours]
|
||||
|
||||
|
||||
def _parse_single_historical_record(self, record: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
||||
"""Parse a single historical weather record"""
|
||||
@@ -351,8 +378,121 @@ class WeatherDataParser:
|
||||
else:
|
||||
return "Soleado"
|
||||
|
||||
def _parse_real_hourly_forecast_days(self, dias: List[Dict[str, Any]], hours: int, base_datetime: datetime) -> List[Dict[str, Any]]:
|
||||
"""Parse real AEMET hourly forecast days"""
|
||||
hourly_forecast = []
|
||||
current_hour = 0
|
||||
|
||||
for day_index, day_data in enumerate(dias):
|
||||
if current_hour >= hours:
|
||||
break
|
||||
|
||||
if not isinstance(day_data, dict):
|
||||
continue
|
||||
|
||||
# Parse hourly data from this day using real AEMET structure
|
||||
day_hourly = self._parse_real_aemet_hourly_day(day_data, base_datetime, day_index, current_hour, hours)
|
||||
hourly_forecast.extend(day_hourly)
|
||||
current_hour += len(day_hourly)
|
||||
|
||||
return hourly_forecast[:hours]
|
||||
|
||||
def _parse_real_aemet_hourly_day(self, day_data: Dict[str, Any], base_datetime: datetime, day_index: int, start_hour: int, max_hours: int) -> List[Dict[str, Any]]:
|
||||
"""Parse hourly data for a single day from real AEMET hourly API response"""
|
||||
hourly_data = []
|
||||
|
||||
# Extract fecha (date) for this day
|
||||
fecha_str = day_data.get('fecha', '')
|
||||
|
||||
try:
|
||||
if fecha_str:
|
||||
day_date = datetime.strptime(fecha_str, '%Y-%m-%d')
|
||||
else:
|
||||
day_date = base_datetime.replace(hour=0, minute=0, second=0, microsecond=0) + timedelta(days=day_index)
|
||||
except ValueError:
|
||||
day_date = base_datetime.replace(hour=0, minute=0, second=0, microsecond=0) + timedelta(days=day_index)
|
||||
|
||||
# Extract hourly arrays from AEMET structure
|
||||
temperatura = day_data.get('temperatura', [])
|
||||
precipitacion = day_data.get('precipitacion', [])
|
||||
humedadRelativa = day_data.get('humedadRelativa', [])
|
||||
viento = day_data.get('viento', [])
|
||||
|
||||
# Process available hourly data
|
||||
max_available_hours = min(
|
||||
len(temperatura) if isinstance(temperatura, list) else 0,
|
||||
len(precipitacion) if isinstance(precipitacion, list) else 0,
|
||||
24, # Max 24 hours per day
|
||||
max_hours - start_hour
|
||||
)
|
||||
|
||||
if max_available_hours == 0:
|
||||
logger.warning("No hourly data arrays found in day data", fecha=fecha_str)
|
||||
return []
|
||||
|
||||
for hour_idx in range(max_available_hours):
|
||||
forecast_datetime = day_date + timedelta(hours=hour_idx)
|
||||
|
||||
# Extract hourly values safely
|
||||
temp = self._extract_aemet_hourly_value(temperatura, hour_idx)
|
||||
precip = self._extract_aemet_hourly_value(precipitacion, hour_idx, default=0.0)
|
||||
humidity = self._extract_aemet_hourly_value(humedadRelativa, hour_idx)
|
||||
wind_data = viento[hour_idx] if isinstance(viento, list) and hour_idx < len(viento) else {}
|
||||
wind_speed = self._extract_wind_from_aemet_data(wind_data)
|
||||
|
||||
hourly_data.append({
|
||||
"forecast_datetime": forecast_datetime,
|
||||
"generated_at": datetime.now(),
|
||||
"temperature": temp if temp is not None else 15.0,
|
||||
"precipitation": precip if precip is not None else 0.0,
|
||||
"humidity": humidity if humidity is not None else 50.0,
|
||||
"wind_speed": wind_speed if wind_speed is not None else 10.0,
|
||||
"description": self._generate_hourly_description(temp, precip),
|
||||
"source": WeatherSource.AEMET.value,
|
||||
"hour": forecast_datetime.hour
|
||||
})
|
||||
|
||||
logger.debug("Parsed AEMET hourly day", fecha=fecha_str, hours_parsed=len(hourly_data))
|
||||
return hourly_data
|
||||
|
||||
def _extract_aemet_hourly_value(self, data_array: Any, hour_idx: int, default: Optional[float] = None) -> Optional[float]:
|
||||
"""Extract hourly value from AEMET hourly data array"""
|
||||
if not isinstance(data_array, list) or hour_idx >= len(data_array):
|
||||
return default
|
||||
|
||||
value_data = data_array[hour_idx]
|
||||
|
||||
# Handle different AEMET data formats
|
||||
if isinstance(value_data, (int, float)):
|
||||
return float(value_data)
|
||||
elif isinstance(value_data, dict):
|
||||
# Try common AEMET field names
|
||||
for field in ['value', 'valor', 'v']:
|
||||
if field in value_data:
|
||||
return self.safe_float(value_data[field], default)
|
||||
elif isinstance(value_data, str):
|
||||
return self.safe_float(value_data, default)
|
||||
|
||||
return default
|
||||
|
||||
def _extract_wind_from_aemet_data(self, wind_data: Any) -> Optional[float]:
|
||||
"""Extract wind speed from AEMET wind data structure"""
|
||||
if isinstance(wind_data, dict):
|
||||
# Try common wind speed fields
|
||||
for field in ['velocidad', 'speed', 'v']:
|
||||
if field in wind_data:
|
||||
velocidad = wind_data[field]
|
||||
if isinstance(velocidad, list) and len(velocidad) > 0:
|
||||
return self.safe_float(velocidad[0])
|
||||
else:
|
||||
return self.safe_float(velocidad)
|
||||
elif isinstance(wind_data, (int, float)):
|
||||
return float(wind_data)
|
||||
|
||||
return None
|
||||
|
||||
def _parse_hourly_forecast_days(self, dias: List[Dict[str, Any]], hours: int, base_datetime: datetime) -> List[Dict[str, Any]]:
|
||||
"""Parse hourly forecast days from AEMET data"""
|
||||
"""Parse hourly forecast days from AEMET data (legacy method)"""
|
||||
hourly_forecast = []
|
||||
current_hour = 0
|
||||
|
||||
@@ -453,6 +593,7 @@ class WeatherDataParser:
|
||||
"""Return forecast as is - no synthetic data filling"""
|
||||
return forecast[:days]
|
||||
|
||||
|
||||
def _ensure_hourly_forecast_completeness(self, forecast: List[Dict[str, Any]], hours: int) -> List[Dict[str, Any]]:
|
||||
"""Return hourly forecast as is - no synthetic data filling"""
|
||||
return forecast[:hours]
|
||||
@@ -630,19 +771,20 @@ class AEMETClient(BaseAPIClient):
|
||||
|
||||
logger.info("✅ Found municipality code", municipality_code=municipality_code)
|
||||
|
||||
# Get real hourly data from AEMET API
|
||||
hourly_data = await self._fetch_hourly_forecast_data(municipality_code)
|
||||
if hourly_data:
|
||||
logger.info("🎉 SUCCESS: Real AEMET hourly data retrieved!", municipality_code=municipality_code)
|
||||
parsed_data = self.parser.parse_hourly_forecast_data(hourly_data, hours)
|
||||
if parsed_data:
|
||||
if parsed_data and len(parsed_data) > 0:
|
||||
return parsed_data
|
||||
|
||||
logger.error("⚠️ AEMET hourly API connectivity issues",
|
||||
municipality_code=municipality_code, reason="aemet_hourly_api_unreachable")
|
||||
logger.error("⚠️ AEMET hourly API failed or returned no data",
|
||||
municipality_code=municipality_code)
|
||||
return []
|
||||
|
||||
except Exception as e:
|
||||
logger.error("❌ AEMET hourly API failed",
|
||||
logger.error("❌ AEMET hourly forecast failed",
|
||||
error=str(e), error_type=type(e).__name__)
|
||||
return []
|
||||
|
||||
@@ -709,13 +851,20 @@ class AEMETClient(BaseAPIClient):
|
||||
|
||||
async def _fetch_hourly_forecast_data(self, municipality_code: str) -> Optional[List[Dict[str, Any]]]:
|
||||
"""Fetch hourly forecast data from AEMET API"""
|
||||
# Note: AEMET hourly forecast API endpoint
|
||||
endpoint = f"/prediccion/especifica/municipio/horaria/{municipality_code}"
|
||||
logger.info("Requesting AEMET hourly forecast", endpoint=endpoint, municipality=municipality_code)
|
||||
|
||||
initial_response = await self._get(endpoint)
|
||||
|
||||
if not self._is_valid_initial_response(initial_response):
|
||||
logger.warning("Invalid initial response from AEMET hourly API",
|
||||
response=initial_response, municipality=municipality_code)
|
||||
return None
|
||||
|
||||
datos_url = initial_response.get("datos")
|
||||
logger.info("Fetching hourly data from AEMET datos URL", url=datos_url)
|
||||
|
||||
return await self._fetch_from_url(datos_url)
|
||||
|
||||
async def _fetch_historical_data_in_chunks(self,
|
||||
|
||||
Reference in New Issue
Block a user