bakery-admin/bakery-ia
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix new services implementation 3

Browse Source
This commit is contained in:
Urtzi Alfaro
2025-08-14 16:47:34 +02:00
parent 0951547e92
commit 03737430ee
51 changed files with 657 additions and 982 deletions
Download Patch File Download Diff File Expand all files Collapse all files

78
services/training/app/ml/trainer.py
View File

@@ -91,7 +91,7 @@ class EnhancedBakeryMLTrainer:
await self._validate_input_data(sales_df, tenant_id)
# Get unique products from the sales data
products = sales_df['product_name'].unique().tolist()
products = sales_df['inventory_product_id'].unique().tolist()
logger.info("Training enhanced models",
products_count=len(products),
products=products)
@@ -183,17 +183,17 @@ class EnhancedBakeryMLTrainer:
"""Process data for all products using enhanced processor with repository tracking"""
processed_data = {}
for product_name in products:
for inventory_product_id in products:
try:
logger.info("Processing data for product using enhanced processor",
product_name=product_name)
inventory_product_id=inventory_product_id)
# Filter sales data for this product
product_sales = sales_df[sales_df['product_name'] == product_name].copy()
product_sales = sales_df[sales_df['inventory_product_id'] == inventory_product_id].copy()
if product_sales.empty:
logger.warning("No sales data found for product",
product_name=product_name)
inventory_product_id=inventory_product_id)
continue
# Use enhanced data processor with repository tracking
@@ -201,19 +201,19 @@ class EnhancedBakeryMLTrainer:
sales_data=product_sales,
weather_data=weather_df,
traffic_data=traffic_df,
product_name=product_name,
inventory_product_id=inventory_product_id,
tenant_id=tenant_id,
job_id=job_id
)
processed_data[product_name] = processed_product_data
processed_data[inventory_product_id] = processed_product_data
logger.info("Enhanced processing completed",
product_name=product_name,
inventory_product_id=inventory_product_id,
data_points=len(processed_product_data))
except Exception as e:
logger.error("Failed to process data using enhanced processor",
product_name=product_name,
inventory_product_id=inventory_product_id,
error=str(e))
continue
@@ -231,15 +231,15 @@ class EnhancedBakeryMLTrainer:
base_progress = 45
max_progress = 85
for product_name, product_data in processed_data.items():
for inventory_product_id, product_data in processed_data.items():
product_start_time = time.time()
try:
logger.info("Training enhanced model",
product_name=product_name)
inventory_product_id=inventory_product_id)
# Check if we have enough data
if len(product_data) < settings.MIN_TRAINING_DATA_DAYS:
training_results[product_name] = {
training_results[inventory_product_id] = {
'status': 'skipped',
'reason': 'insufficient_data',
'data_points': len(product_data),
@@ -247,7 +247,7 @@ class EnhancedBakeryMLTrainer:
'message': f'Need at least {settings.MIN_TRAINING_DATA_DAYS} data points, got {len(product_data)}'
}
logger.warning("Skipping product due to insufficient data",
product_name=product_name,
inventory_product_id=inventory_product_id,
data_points=len(product_data),
min_required=settings.MIN_TRAINING_DATA_DAYS)
continue
@@ -255,24 +255,24 @@ class EnhancedBakeryMLTrainer:
# Train the model using Prophet manager
model_info = await self.prophet_manager.train_bakery_model(
tenant_id=tenant_id,
product_name=product_name,
inventory_product_id=inventory_product_id,
df=product_data,
job_id=job_id
)
# Store model record using repository
model_record = await self._create_model_record(
repos, tenant_id, product_name, model_info, job_id, product_data
repos, tenant_id, inventory_product_id, model_info, job_id, product_data
)
# Create performance metrics record
if model_info.get('training_metrics'):
await self._create_performance_metrics(
repos, model_record.id if model_record else None,
tenant_id, product_name, model_info['training_metrics']
tenant_id, inventory_product_id, model_info['training_metrics']
)
training_results[product_name] = {
training_results[inventory_product_id] = {
'status': 'success',
'model_info': model_info,
'model_record_id': model_record.id if model_record else None,
@@ -282,7 +282,7 @@ class EnhancedBakeryMLTrainer:
}
logger.info("Successfully trained enhanced model",
product_name=product_name,
inventory_product_id=inventory_product_id,
model_record_id=model_record.id if model_record else None)
completed_products = i + 1
@@ -295,15 +295,15 @@ class EnhancedBakeryMLTrainer:
await self.status_publisher.progress_update(
progress=progress,
step="model_training",
current_product=product_name,
step_details=f"Enhanced training completed for {product_name}"
current_product=inventory_product_id,
step_details=f"Enhanced training completed for {inventory_product_id}"
)
except Exception as e:
logger.error("Failed to train enhanced model",
product_name=product_name,
inventory_product_id=inventory_product_id,
error=str(e))
training_results[product_name] = {
training_results[inventory_product_id] = {
'status': 'error',
'error_message': str(e),
'data_points': len(product_data) if product_data is not None else 0,
@@ -320,8 +320,8 @@ class EnhancedBakeryMLTrainer:
await self.status_publisher.progress_update(
progress=progress,
step="model_training",
current_product=product_name,
step_details=f"Enhanced training failed for {product_name}: {str(e)}"
current_product=inventory_product_id,
step_details=f"Enhanced training failed for {inventory_product_id}: {str(e)}"
)
return training_results
@@ -329,7 +329,7 @@ class EnhancedBakeryMLTrainer:
async def _create_model_record(self,
repos: Dict,
tenant_id: str,
product_name: str,
inventory_product_id: str,
model_info: Dict,
job_id: str,
processed_data: pd.DataFrame):
@@ -337,7 +337,7 @@ class EnhancedBakeryMLTrainer:
try:
model_data = {
"tenant_id": tenant_id,
"product_name": product_name,
"inventory_product_id": inventory_product_id,
"job_id": job_id,
"model_type": "enhanced_prophet",
"model_path": model_info.get("model_path"),
@@ -357,7 +357,7 @@ class EnhancedBakeryMLTrainer:
model_record = await repos['model'].create_model(model_data)
logger.info("Created enhanced model record",
product_name=product_name,
inventory_product_id=inventory_product_id,
model_id=model_record.id)
# Create artifacts for model files
@@ -374,7 +374,7 @@ class EnhancedBakeryMLTrainer:
except Exception as e:
logger.error("Failed to create enhanced model record",
product_name=product_name,
inventory_product_id=inventory_product_id,
error=str(e))
return None
@@ -382,14 +382,14 @@ class EnhancedBakeryMLTrainer:
repos: Dict,
model_id: str,
tenant_id: str,
product_name: str,
inventory_product_id: str,
metrics: Dict):
"""Create performance metrics record using repository"""
try:
metric_data = {
"model_id": str(model_id),
"tenant_id": tenant_id,
"product_name": product_name,
"inventory_product_id": inventory_product_id,
"mae": metrics.get("mae"),
"mse": metrics.get("mse"),
"rmse": metrics.get("rmse"),
@@ -401,12 +401,12 @@ class EnhancedBakeryMLTrainer:
await repos['performance'].create_performance_metric(metric_data)
logger.info("Created enhanced performance metrics",
product_name=product_name,
inventory_product_id=inventory_product_id,
model_id=model_id)
except Exception as e:
logger.error("Failed to create enhanced performance metrics",
product_name=product_name,
inventory_product_id=inventory_product_id,
error=str(e))
async def _calculate_enhanced_training_summary(self,
@@ -532,7 +532,7 @@ class EnhancedBakeryMLTrainer:
async def evaluate_model_performance_enhanced(self,
tenant_id: str,
product_name: str,
inventory_product_id: str,
model_path: str,
test_dataset: TrainingDataSet) -> Dict[str, Any]:
"""
@@ -553,17 +553,17 @@ class EnhancedBakeryMLTrainer:
test_traffic_df = pd.DataFrame(test_dataset.traffic_data)
# Filter for specific product
product_test_sales = test_sales_df[test_sales_df['product_name'] == product_name].copy()
product_test_sales = test_sales_df[test_sales_df['inventory_product_id'] == inventory_product_id].copy()
if product_test_sales.empty:
raise ValueError(f"No test data found for product: {product_name}")
raise ValueError(f"No test data found for product: {inventory_product_id}")
# Process test data using enhanced processor
processed_test_data = await self.enhanced_data_processor.prepare_training_data(
sales_data=product_test_sales,
weather_data=test_weather_df,
traffic_data=test_traffic_df,
product_name=product_name,
inventory_product_id=inventory_product_id,
tenant_id=tenant_id
)
@@ -608,16 +608,16 @@ class EnhancedBakeryMLTrainer:
metrics["mape"] = 100.0
# Store evaluation metrics in repository
model_records = await repos['model'].get_models_by_product(tenant_id, product_name)
model_records = await repos['model'].get_models_by_product(tenant_id, inventory_product_id)
if model_records:
latest_model = max(model_records, key=lambda x: x.created_at)
await self._create_performance_metrics(
repos, latest_model.id, tenant_id, product_name, metrics
repos, latest_model.id, tenant_id, inventory_product_id, metrics
)
result = {
"tenant_id": tenant_id,
"product_name": product_name,
"inventory_product_id": inventory_product_id,
"enhanced_evaluation_metrics": metrics,
"test_samples": len(processed_test_data),
"prediction_samples": len(forecast),