REFACTOR data service

services/external/app/core/__init__.py

@@ -0,0 +1 @@
+# services/external/app/core/__init__.py

services/external/app/core/config.py

@@ -0,0 +1,64 @@
+# services/external/app/core/config.py
+
+from shared.config.base import BaseServiceSettings
+import os
+from pydantic import Field
+
+class DataSettings(BaseServiceSettings):
+    """Data service specific settings"""
+    
+    # Service Identity
+    SERVICE_NAME: str = "external-service"
+    VERSION: str = "1.0.0"
+    APP_NAME: str = "Bakery External Data Service"
+    DESCRIPTION: str = "External data collection service for weather and traffic data"
+    
+    # API Configuration
+    API_V1_STR: str = "/api/v1"
+    
+    # Override database URL to use EXTERNAL_DATABASE_URL
+    DATABASE_URL: str = Field(
+        default="postgresql+asyncpg://external_user:external_pass123@external-db:5432/external_db",
+        env="EXTERNAL_DATABASE_URL"
+    )
+    
+    # External API Configuration
+    AEMET_API_KEY: str = os.getenv("AEMET_API_KEY", "")
+    AEMET_BASE_URL: str = "https://opendata.aemet.es/opendata"
+    AEMET_TIMEOUT: int = int(os.getenv("AEMET_TIMEOUT", "30"))
+    AEMET_RETRY_ATTEMPTS: int = int(os.getenv("AEMET_RETRY_ATTEMPTS", "3"))
+    
+    MADRID_OPENDATA_API_KEY: str = os.getenv("MADRID_OPENDATA_API_KEY", "")
+    MADRID_OPENDATA_BASE_URL: str = "https://datos.madrid.es"
+    MADRID_OPENDATA_TIMEOUT: int = int(os.getenv("MADRID_OPENDATA_TIMEOUT", "30"))
+    
+    # Data Collection Configuration
+    WEATHER_COLLECTION_INTERVAL_HOURS: int = int(os.getenv("WEATHER_COLLECTION_INTERVAL_HOURS", "1"))
+    TRAFFIC_COLLECTION_INTERVAL_HOURS: int = int(os.getenv("TRAFFIC_COLLECTION_INTERVAL_HOURS", "1"))
+    EVENTS_COLLECTION_INTERVAL_HOURS: int = int(os.getenv("EVENTS_COLLECTION_INTERVAL_HOURS", "6"))
+    
+    # Cache TTL Configuration
+    WEATHER_CACHE_TTL_HOURS: int = int(os.getenv("WEATHER_CACHE_TTL_HOURS", "1"))
+    TRAFFIC_CACHE_TTL_HOURS: int = int(os.getenv("TRAFFIC_CACHE_TTL_HOURS", "1"))
+    EVENTS_CACHE_TTL_HOURS: int = int(os.getenv("EVENTS_CACHE_TTL_HOURS", "6"))
+    
+    # Data Quality Configuration
+    DATA_VALIDATION_ENABLED: bool = os.getenv("DATA_VALIDATION_ENABLED", "true").lower() == "true"
+    OUTLIER_DETECTION_ENABLED: bool = os.getenv("OUTLIER_DETECTION_ENABLED", "true").lower() == "true"
+    DATA_COMPLETENESS_THRESHOLD: float = float(os.getenv("DATA_COMPLETENESS_THRESHOLD", "0.8"))
+    
+    # Geolocation Settings (Madrid focus)
+    DEFAULT_LATITUDE: float = float(os.getenv("DEFAULT_LATITUDE", "40.4168"))  # Madrid
+    DEFAULT_LONGITUDE: float = float(os.getenv("DEFAULT_LONGITUDE", "-3.7038"))  # Madrid
+    LOCATION_RADIUS_KM: float = float(os.getenv("LOCATION_RADIUS_KM", "50.0"))
+    
+    # Data Retention
+    RAW_DATA_RETENTION_DAYS: int = int(os.getenv("RAW_DATA_RETENTION_DAYS", "90"))
+    PROCESSED_DATA_RETENTION_DAYS: int = int(os.getenv("PROCESSED_DATA_RETENTION_DAYS", "365"))
+    
+    # Batch Processing
+    BATCH_PROCESSING_ENABLED: bool = os.getenv("BATCH_PROCESSING_ENABLED", "true").lower() == "true"
+    BATCH_SIZE: int = int(os.getenv("BATCH_SIZE", "1000"))
+    PARALLEL_PROCESSING_WORKERS: int = int(os.getenv("PARALLEL_PROCESSING_WORKERS", "4"))
+
+settings = DataSettings()

services/external/app/core/database.py

@@ -0,0 +1,81 @@
+# services/external/app/core/database.py
+"""
+External Service Database Configuration using shared database manager
+"""
+
+import structlog
+from contextlib import asynccontextmanager
+from typing import AsyncGenerator
+
+from app.core.config import settings
+from shared.database.base import DatabaseManager, Base
+
+logger = structlog.get_logger()
+
+# Create database manager instance
+database_manager = DatabaseManager(
+    database_url=settings.DATABASE_URL,
+    service_name="external-service"
+)
+
+async def get_db():
+    """
+    Database dependency for FastAPI - using shared database manager
+    """
+    async for session in database_manager.get_db():
+        yield session
+
+
+async def init_db():
+    """Initialize database tables using shared database manager"""
+    try:
+        logger.info("Initializing External Service database...")
+        
+        # Import all models to ensure they're registered
+        from app.models import weather, traffic  # noqa: F401
+        
+        # Create all tables using database manager
+        await database_manager.create_tables(Base.metadata)
+            
+        logger.info("External Service database initialized successfully")
+        
+    except Exception as e:
+        logger.error("Failed to initialize database", error=str(e))
+        raise
+
+
+async def close_db():
+    """Close database connections using shared database manager"""
+    try:
+        await database_manager.close_connections()
+        logger.info("Database connections closed")
+    except Exception as e:
+        logger.error("Error closing database connections", error=str(e))
+
+
+@asynccontextmanager
+async def get_db_transaction():
+    """
+    Context manager for database transactions using shared database manager
+    """
+    async with database_manager.get_session() as session:
+        try:
+            async with session.begin():
+                yield session
+        except Exception as e:
+            logger.error("Transaction error", error=str(e))
+            raise
+
+
+@asynccontextmanager
+async def get_background_session():
+    """
+    Context manager for background tasks using shared database manager
+    """
+    async with database_manager.get_background_session() as session:
+        yield session
+
+
+async def health_check():
+    """Database health check using shared database manager"""
+    return await database_manager.health_check()