REFACTOR external service and improve websocket training
This commit is contained in:
Urtzi Alfaro
178
services/external/app/cache/redis_cache.py
vendored
Normal file
178
services/external/app/cache/redis_cache.py
vendored
Normal file
@@ -0,0 +1,178 @@
|
||||
# services/external/app/cache/redis_cache.py
|
||||
"""
|
||||
Redis cache layer for fast training data access
|
||||
"""
|
||||
|
||||
from typing import List, Dict, Any, Optional
|
||||
import json
|
||||
from datetime import datetime, timedelta
|
||||
import structlog
|
||||
import redis.asyncio as redis
|
||||
|
||||
from app.core.config import settings
|
||||
|
||||
logger = structlog.get_logger()
|
||||
|
||||
|
||||
class ExternalDataCache:
|
||||
"""Redis cache for external data service"""
|
||||
|
||||
def __init__(self):
|
||||
self.redis_client = redis.from_url(
|
||||
settings.REDIS_URL,
|
||||
encoding="utf-8",
|
||||
decode_responses=True
|
||||
)
|
||||
self.ttl = 86400 * 7
|
||||
|
||||
def _weather_cache_key(
|
||||
self,
|
||||
city_id: str,
|
||||
start_date: datetime,
|
||||
end_date: datetime
|
||||
) -> str:
|
||||
"""Generate cache key for weather data"""
|
||||
return f"weather:{city_id}:{start_date.date()}:{end_date.date()}"
|
||||
|
||||
async def get_cached_weather(
|
||||
self,
|
||||
city_id: str,
|
||||
start_date: datetime,
|
||||
end_date: datetime
|
||||
) -> Optional[List[Dict[str, Any]]]:
|
||||
"""Get cached weather data"""
|
||||
try:
|
||||
key = self._weather_cache_key(city_id, start_date, end_date)
|
||||
cached = await self.redis_client.get(key)
|
||||
|
||||
if cached:
|
||||
logger.debug("Weather cache hit", city_id=city_id, key=key)
|
||||
return json.loads(cached)
|
||||
|
||||
logger.debug("Weather cache miss", city_id=city_id, key=key)
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Error reading weather cache", error=str(e))
|
||||
return None
|
||||
|
||||
async def set_cached_weather(
|
||||
self,
|
||||
city_id: str,
|
||||
start_date: datetime,
|
||||
end_date: datetime,
|
||||
data: List[Dict[str, Any]]
|
||||
):
|
||||
"""Set cached weather data"""
|
||||
try:
|
||||
key = self._weather_cache_key(city_id, start_date, end_date)
|
||||
|
||||
serializable_data = []
|
||||
for record in data:
|
||||
# Handle both dict and Pydantic model objects
|
||||
if hasattr(record, 'model_dump'):
|
||||
record_dict = record.model_dump()
|
||||
elif hasattr(record, 'dict'):
|
||||
record_dict = record.dict()
|
||||
else:
|
||||
record_dict = record.copy() if isinstance(record, dict) else dict(record)
|
||||
|
||||
# Convert any datetime fields to ISO format strings
|
||||
for key_name, value in record_dict.items():
|
||||
if isinstance(value, datetime):
|
||||
record_dict[key_name] = value.isoformat()
|
||||
|
||||
serializable_data.append(record_dict)
|
||||
|
||||
await self.redis_client.setex(
|
||||
key,
|
||||
self.ttl,
|
||||
json.dumps(serializable_data)
|
||||
)
|
||||
|
||||
logger.debug("Weather data cached", city_id=city_id, records=len(data))
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Error caching weather data", error=str(e))
|
||||
|
||||
def _traffic_cache_key(
|
||||
self,
|
||||
city_id: str,
|
||||
start_date: datetime,
|
||||
end_date: datetime
|
||||
) -> str:
|
||||
"""Generate cache key for traffic data"""
|
||||
return f"traffic:{city_id}:{start_date.date()}:{end_date.date()}"
|
||||
|
||||
async def get_cached_traffic(
|
||||
self,
|
||||
city_id: str,
|
||||
start_date: datetime,
|
||||
end_date: datetime
|
||||
) -> Optional[List[Dict[str, Any]]]:
|
||||
"""Get cached traffic data"""
|
||||
try:
|
||||
key = self._traffic_cache_key(city_id, start_date, end_date)
|
||||
cached = await self.redis_client.get(key)
|
||||
|
||||
if cached:
|
||||
logger.debug("Traffic cache hit", city_id=city_id, key=key)
|
||||
return json.loads(cached)
|
||||
|
||||
logger.debug("Traffic cache miss", city_id=city_id, key=key)
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Error reading traffic cache", error=str(e))
|
||||
return None
|
||||
|
||||
async def set_cached_traffic(
|
||||
self,
|
||||
city_id: str,
|
||||
start_date: datetime,
|
||||
end_date: datetime,
|
||||
data: List[Dict[str, Any]]
|
||||
):
|
||||
"""Set cached traffic data"""
|
||||
try:
|
||||
key = self._traffic_cache_key(city_id, start_date, end_date)
|
||||
|
||||
serializable_data = []
|
||||
for record in data:
|
||||
# Handle both dict and Pydantic model objects
|
||||
if hasattr(record, 'model_dump'):
|
||||
record_dict = record.model_dump()
|
||||
elif hasattr(record, 'dict'):
|
||||
record_dict = record.dict()
|
||||
else:
|
||||
record_dict = record.copy() if isinstance(record, dict) else dict(record)
|
||||
|
||||
# Convert any datetime fields to ISO format strings
|
||||
for key_name, value in record_dict.items():
|
||||
if isinstance(value, datetime):
|
||||
record_dict[key_name] = value.isoformat()
|
||||
|
||||
serializable_data.append(record_dict)
|
||||
|
||||
await self.redis_client.setex(
|
||||
key,
|
||||
self.ttl,
|
||||
json.dumps(serializable_data)
|
||||
)
|
||||
|
||||
logger.debug("Traffic data cached", city_id=city_id, records=len(data))
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Error caching traffic data", error=str(e))
|
||||
|
||||
async def invalidate_city_cache(self, city_id: str):
|
||||
"""Invalidate all cache entries for a city"""
|
||||
try:
|
||||
pattern = f"*:{city_id}:*"
|
||||
async for key in self.redis_client.scan_iter(match=pattern):
|
||||
await self.redis_client.delete(key)
|
||||
|
||||
logger.info("City cache invalidated", city_id=city_id)
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Error invalidating cache", error=str(e))
|
||||
Reference in New Issue
Block a user