bakery-admin/bakery-ia
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

REFACTOR data service

Browse Source
This commit is contained in:
Urtzi Alfaro
2025-08-12 18:17:30 +02:00
parent 7c237c0acc
commit fbe7470ad9
149 changed files with 8528 additions and 7393 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
services/training/app/services/data_client.py
View File

@@ -9,7 +9,7 @@ from typing import Dict, Any, List, Optional
from datetime import datetime
# Import the shared clients
from shared.clients import get_data_client, get_service_clients
from shared.clients import get_sales_client, get_external_client, get_service_clients
from app.core.config import settings
logger = structlog.get_logger()
@@ -21,19 +21,20 @@ class DataClient:
"""
def __init__(self):
# Get the shared data client configured for this service
self.data_client = get_data_client(settings, "training")
# Get the new specialized clients
self.sales_client = get_sales_client(settings, "training")
self.external_client = get_external_client(settings, "training")
# Check if the new method is available for stored traffic data
if hasattr(self.data_client, 'get_stored_traffic_data_for_training'):
if hasattr(self.external_client, 'get_stored_traffic_data_for_training'):
self.supports_stored_traffic_data = True
else:
self.supports_stored_traffic_data = False
logger.warning("Stored traffic data method not available in data client")
logger.warning("Stored traffic data method not available in external client")
# Or alternatively, get all clients at once:
# self.clients = get_service_clients(settings, "training")
# Then use: self.clients.data.get_sales_data(...)
# Then use: self.clients.sales.get_sales_data(...) and self.clients.external.get_weather_forecast(...)
async def fetch_sales_data(
self,
@@ -57,18 +58,18 @@ class DataClient:
try:
if fetch_all:
# Use paginated method to get ALL records (original behavior)
sales_data = await self.data_client.get_all_sales_data(
sales_data = await self.sales_client.get_all_sales_data(
tenant_id=tenant_id,
start_date=start_date,
end_date=end_date,
product_id=product_id,
aggregation="daily",
page_size=5000, # Match original page size
page_size=1000, # Comply with API limit
max_pages=100 # Safety limit (500k records max)
)
else:
# Use standard method for limited results
sales_data = await self.data_client.get_sales_data(
sales_data = await self.sales_client.get_sales_data(
tenant_id=tenant_id,
start_date=start_date,
end_date=end_date,
@@ -102,7 +103,7 @@ class DataClient:
All the error handling and retry logic is now in the base client!
"""
try:
weather_data = await self.data_client.get_weather_historical(
weather_data = await self.external_client.get_weather_historical(
tenant_id=tenant_id,
start_date=start_date,
end_date=end_date,
@@ -134,7 +135,7 @@ class DataClient:
Fetch traffic data for training
"""
try:
traffic_data = await self.data_client.get_traffic_data(
traffic_data = await self.external_client.get_traffic_data(
tenant_id=tenant_id,
start_date=start_date,
end_date=end_date,
@@ -169,7 +170,7 @@ class DataClient:
try:
if self.supports_stored_traffic_data:
# Use the dedicated stored traffic data method
stored_traffic_data = await self.data_client.get_stored_traffic_data_for_training(
stored_traffic_data = await self.external_client.get_stored_traffic_data_for_training(
tenant_id=tenant_id,
start_date=start_date,
end_date=end_date,
@@ -209,11 +210,15 @@ class DataClient:
Validate data quality before training
"""
try:
validation_result = await self.data_client.validate_data_quality(
tenant_id=tenant_id,
start_date=start_date,
end_date=end_date
)
# Note: validation_data_quality may need to be implemented in one of the new services
# validation_result = await self.sales_client.validate_data_quality(
# tenant_id=tenant_id,
# start_date=start_date,
# end_date=end_date
# )
# Temporary implementation - assume data is valid for now
validation_result = {"is_valid": True, "message": "Validation temporarily disabled"}
if validation_result:
logger.info("Data validation completed",