bakery-admin/bakery-ia
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix data fetch 7

Browse Source
This commit is contained in:
Urtzi Alfaro
2025-07-27 22:58:18 +02:00
parent 0201b428e5
commit 946015b80c
8 changed files with 138 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
services/data/app/external/madrid_opendata.py vendored
View File

@@ -15,7 +15,7 @@ Features:
import math
import xml.etree.ElementTree as ET
from typing import List, Dict, Any, Optional, Tuple
from datetime import datetime, timedelta
from datetime import datetime, timedelta, timezone
import structlog
import re
from dataclasses import dataclass
@@ -480,7 +480,19 @@ class MadridOpenDataClient(BaseAPIClient):
# Parse date
record_date = self._parse_madrid_date(row.get('fecha', '').strip().strip('"'))
if not record_date or not (start_date <= record_date <= end_date):
if not record_date:
return None
# ✅ CRITICAL FIX: Ensure both dates are timezone-aware for comparison
if start_date.tzinfo is None:
start_date = start_date.replace(tzinfo=timezone.utc)
if end_date.tzinfo is None:
end_date = end_date.replace(tzinfo=timezone.utc)
if record_date.tzinfo is None:
record_date = record_date.replace(tzinfo=timezone.utc)
# Now we can safely compare timezone-aware datetimes
if not (start_date <= record_date <= end_date):
return None
# Parse traffic data
@@ -749,15 +761,21 @@ class MadridOpenDataClient(BaseAPIClient):
return int(base * multiplier)
def _parse_madrid_date(self, fecha_str: str) -> Optional[datetime]:
"""Parse Madrid date format"""
"""Parse Madrid date format with timezone awareness"""
if not fecha_str:
return None
try:
return datetime.strptime(fecha_str, '%Y-%m-%d %H:%M:%S')
# Parse the date as timezone-naive first
dt = datetime.strptime(fecha_str, '%Y-%m-%d %H:%M:%S')
# Convert to timezone-aware (assume Madrid/UTC timezone)
return dt.replace(tzinfo=timezone.utc)
except ValueError:
try:
return datetime.strptime(fecha_str, '%d/%m/%Y %H:%M:%S')
# Try alternative format
dt = datetime.strptime(fecha_str, '%d/%m/%Y %H:%M:%S')
# Convert to timezone-aware (assume Madrid/UTC timezone)
return dt.replace(tzinfo=timezone.utc)
except ValueError:
return None

26
services/data/app/services/traffic_service.py
View File

@@ -13,6 +13,8 @@ from app.models.traffic import TrafficData
from app.external.madrid_opendata import MadridOpenDataClient
from app.schemas.external import TrafficDataResponse
import uuid
logger = structlog.get_logger()
class TrafficService:
@@ -102,26 +104,26 @@ class TrafficService:
try:
for data in traffic_data:
traffic_record = TrafficData(
id = id,
location_id = location_id,
date = data.get('date', datetime.now()),
traffic_volume = data.get('traffic_volume'),
pedestrian_count = data.get('pedestrian_count'),
congestion_level = data.get('congestion_level'),
average_speed = data.get('average_speed'),
source = "Madrid Open Data",
raw_data = str(data),
created_at = data.get('created_at'),
location_id=location_id,
date=data.get('date', datetime.now()),
traffic_volume=data.get('traffic_volume'),
pedestrian_count=data.get('pedestrian_count'),
congestion_level=data.get('congestion_level'),
average_speed=data.get('average_speed'),
source="madrid_opendata",
raw_data=str(data),
created_at=datetime.now()
)
db.add(traffic_record)
await db.commit()
logger.debug("Historical data stored in database", count=len(traffic_record))
logger.debug("Historical data stored in database", count=len(traffic_data))
except Exception as db_error:
logger.warning("Failed to store historical data in database", error=str(db_error))
await db.rollback()
return [TrafficDataResponse(**item) for item in traffic_data]
return [TrafficDataResponse(**item) for item in traffic_record]
else:
logger.warning("No historical traffic data received")
return []

49
services/data/migrations/versions/add_timezone_to_datetime_columns.py Normal file
View File

@@ -0,0 +1,49 @@
# ================================================================
# services/data/migrations/versions/20250727_add_timezone_to_datetime_columns.py
# ================================================================
"""Add timezone support to datetime columns
Revision ID: 20250727_193000
Revises:
Create Date: 2025-07-27 19:30:00.000000
"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic.
revision = '20250727_193000'
down_revision = None # Replace with actual previous revision if exists
branch_labels = None
depends_on = None
def upgrade() -> None:
"""Convert TIMESTAMP WITHOUT TIME ZONE to TIMESTAMP WITH TIME ZONE"""
# Weather data table
op.execute("ALTER TABLE weather_data ALTER COLUMN date TYPE TIMESTAMP WITH TIME ZONE USING date AT TIME ZONE 'UTC'")
op.execute("ALTER TABLE weather_data ALTER COLUMN created_at TYPE TIMESTAMP WITH TIME ZONE USING created_at AT TIME ZONE 'UTC'")
# Weather forecasts table
op.execute("ALTER TABLE weather_forecasts ALTER COLUMN forecast_date TYPE TIMESTAMP WITH TIME ZONE USING forecast_date AT TIME ZONE 'UTC'")
op.execute("ALTER TABLE weather_forecasts ALTER COLUMN generated_at TYPE TIMESTAMP WITH TIME ZONE USING generated_at AT TIME ZONE 'UTC'")
# Traffic data table
op.execute("ALTER TABLE traffic_data ALTER COLUMN date TYPE TIMESTAMP WITH TIME ZONE USING date AT TIME ZONE 'UTC'")
op.execute("ALTER TABLE traffic_data ALTER COLUMN created_at TYPE TIMESTAMP WITH TIME ZONE USING created_at AT TIME ZONE 'UTC'")
def downgrade() -> None:
"""Convert TIMESTAMP WITH TIME ZONE back to TIMESTAMP WITHOUT TIME ZONE"""
# Weather data table
op.execute("ALTER TABLE weather_data ALTER COLUMN date TYPE TIMESTAMP WITHOUT TIME ZONE USING date AT TIME ZONE 'UTC'")
op.execute("ALTER TABLE weather_data ALTER COLUMN created_at TYPE TIMESTAMP WITHOUT TIME ZONE USING created_at AT TIME ZONE 'UTC'")
# Weather forecasts table
op.execute("ALTER TABLE weather_forecasts ALTER COLUMN forecast_date TYPE TIMESTAMP WITHOUT TIME ZONE USING forecast_date AT TIME ZONE 'UTC'")
op.execute("ALTER TABLE weather_forecasts ALTER COLUMN generated_at TYPE TIMESTAMP WITHOUT TIME ZONE USING generated_at AT TIME ZONE 'UTC'")
# Traffic data table
op.execute("ALTER TABLE traffic_data ALTER COLUMN date TYPE TIMESTAMP WITHOUT TIME ZONE USING date AT TIME ZONE 'UTC'")
op.execute("ALTER TABLE traffic_data ALTER COLUMN created_at TYPE TIMESTAMP WITHOUT TIME ZONE USING created_at AT TIME ZONE 'UTC'")