Fix MAdrid Open data file 2

services/data/app/external/madrid_opendata.py

@@ -59,8 +59,8 @@ MADRID_BOUNDS = {
 }
 
 # Constants
-MAX_HISTORICAL_DAYS = 90
+MAX_HISTORICAL_DAYS = 365
-MAX_CSV_PROCESSING_ROWS = 50000
+MAX_CSV_PROCESSING_ROWS = 5000000
 MEASUREMENT_POINTS_LIMIT = 20
 UTM_ZONE = 30  # Madrid is in UTM Zone 30N
 

services/data/tests/test_data.py

@@ -1,94 +0,0 @@
-# ================================================================
-# services/data/tests/test_data.py
-# ================================================================
-"""Integration tests for data service"""
-
-import pytest
-from fastapi.testclient import TestClient
-from sqlalchemy.ext.asyncio import AsyncSession
-from datetime import datetime, timedelta
-import uuid
-
-from app.services.sales_service import SalesService
-from app.services.weather_service import WeatherService
-from app.services.traffic_service import TrafficService
-from app.schemas.sales import SalesDataCreate, SalesDataQuery
-
-@pytest.mark.asyncio
-async def test_create_sales_record(db: AsyncSession, test_tenant_id, test_sales_data):
-    """Test creating a sales record"""
-    sales_data = SalesDataCreate(
-        tenant_id=test_tenant_id,
-        **test_sales_data
-    )
-    
-    record = await SalesService.create_sales_record(sales_data, db)
-    
-    assert record.id is not None
-    assert record.tenant_id == test_tenant_id
-    assert record.product_name == test_sales_data["product_name"]
-    assert record.quantity_sold == test_sales_data["quantity_sold"]
-
-@pytest.mark.asyncio
-async def test_query_sales_data(db: AsyncSession, test_tenant_id, test_sales_data):
-    """Test querying sales data"""
-    # Create test record
-    sales_data = SalesDataCreate(
-        tenant_id=test_tenant_id,
-        **test_sales_data
-    )
-    await SalesService.create_sales_record(sales_data, db)
-    
-    # Query data
-    query = SalesDataQuery(
-        tenant_id=test_tenant_id,
-        start_date=datetime.now() - timedelta(days=1),
-        end_date=datetime.now() + timedelta(days=1)
-    )
-    
-    records = await SalesService.get_sales_data(query, db)
-    
-    assert len(records) == 1
-    assert records[0].product_name == test_sales_data["product_name"]
-
-@pytest.mark.asyncio
-async def test_csv_import():
-    """Test CSV data import"""
-    csv_content = """date,product,quantity,revenue
-2024-01-15,Pan Integral,25,37.50
-2024-01-15,Croissant,15,22.50"""
-    
-    # This would normally use the database, but we're testing the parsing logic
-    import io
-    import pandas as pd
-    
-    csv_file = io.StringIO(csv_content)
-    df = pd.read_csv(csv_file)
-    
-    assert len(df) == 2
-    assert df.iloc[0]['product'] == 'Pan Integral'
-    assert df.iloc[0]['quantity'] == 25
-
-@pytest.mark.asyncio
-async def test_weather_service():
-    """Test weather service"""
-    weather_service = WeatherService()
-    
-    # Test current weather (should return synthetic data)
-    current = await weather_service.get_current_weather(40.4168, -3.7038)
-    
-    assert current is not None
-    assert current.temperature is not None
-    assert current.source in ['aemet', 'synthetic']
-
-@pytest.mark.asyncio
-async def test_traffic_service():
-    """Test traffic service"""
-    traffic_service = TrafficService()
-    
-    # Test current traffic (should return synthetic data)
-    current = await traffic_service.get_current_traffic(40.4168, -3.7038)
-    
-    assert current is not None
-    assert current.traffic_volume is not None
-    assert current.congestion_level in ['low', 'medium', 'high']

services/data/tests/test_external.py

@@ -1,87 +0,0 @@
-# ================================================================
-# services/data/tests/test_external.py
-# ================================================================
-"""Tests for external API clients"""
-
-import pytest
-from unittest.mock import AsyncMock, patch
-from datetime import datetime, timedelta
-
-from app.external.aemet import AEMETClient
-from app.external.madrid_opendata import MadridOpenDataClient
-
-@pytest.mark.asyncio
-async def test_aemet_client_synthetic_weather():
-    """Test AEMET client synthetic weather generation"""
-    client = AEMETClient()
-    
-    weather = await client._generate_synthetic_weather()
-    
-    assert weather is not None
-    assert 'temperature' in weather
-    assert 'precipitation' in weather
-    assert 'humidity' in weather
-    assert weather['source'] == 'synthetic'
-
-@pytest.mark.asyncio
-async def test_aemet_client_synthetic_forecast():
-    """Test AEMET client synthetic forecast"""
-    client = AEMETClient()
-    
-    forecast = await client._generate_synthetic_forecast(7)
-    
-    assert len(forecast) == 7
-    assert all('forecast_date' in item for item in forecast)
-    assert all('temperature' in item for item in forecast)
-    assert all(item['source'] == 'synthetic' for item in forecast)
-
-@pytest.mark.asyncio
-async def test_madrid_client_synthetic_traffic():
-    """Test Madrid Open Data client synthetic traffic"""
-    client = MadridOpenDataClient()
-    
-    traffic = await client._generate_synthetic_traffic(40.4168, -3.7038)
-    
-    assert traffic is not None
-    assert 'traffic_volume' in traffic
-    assert 'pedestrian_count' in traffic
-    assert 'congestion_level' in traffic
-    assert traffic['congestion_level'] in ['low', 'medium', 'high']
-
-@pytest.mark.asyncio
-async def test_madrid_client_historical_traffic():
-    """Test Madrid Open Data client historical traffic"""
-    client = MadridOpenDataClient()
-    
-    start_date = datetime.now() - timedelta(days=7)
-    end_date = datetime.now()
-    
-    historical = await client._generate_historical_traffic(
-        40.4168, -3.7038, start_date, end_date
-    )
-    
-    assert len(historical) > 0
-    assert all('traffic_volume' in item for item in historical)
-    assert all('date' in item for item in historical)
-
-@pytest.mark.asyncio
-async def test_distance_calculation():
-    """Test distance calculation in AEMET client"""
-    client = AEMETClient()
-    
-    # Distance between Madrid Centro and Madrid Norte (approximately)
-    distance = client._calculate_distance(40.4168, -3.7038, 40.4518, -3.7246)
-    
-    # Should be roughly 4-5 km
-    assert 3 < distance < 6
-
-@pytest.mark.asyncio
-async def test_madrid_area_detection():
-    """Test Madrid area detection"""
-    client = AEMETClient()
-    
-    # Madrid coordinates should be detected as Madrid area
-    assert client._is_in_madrid_area(40.4168, -3.7038) == True
-    
-    # Barcelona coordinates should not
-    assert client._is_in_madrid_area(41.3851, 2.1734) == False