bakery-admin/bakery-ia
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add improvements 2

Browse Source
This commit is contained in:
Urtzi Alfaro
2026-01-12 22:15:11 +01:00
parent 230bbe6a19
commit b931a5c45e
40 changed files with 1820 additions and 887 deletions
Download Patch File Download Diff File Expand all files Collapse all files

160
services/training/app/ml/hybrid_trainer.py
View File

@@ -216,17 +216,24 @@ class HybridProphetXGBoost:
Get Prophet predictions for given dataframe.
Args:
prophet_result: Prophet model result from training
prophet_result: Prophet model result from training (contains model_path)
df: DataFrame with 'ds' column
Returns:
Array of predictions
"""
# Get the Prophet model from result
prophet_model = prophet_result.get('model')
# Get the model path from result instead of expecting the model object directly
model_path = prophet_result.get('model_path')
if prophet_model is None:
raise ValueError("Prophet model not found in result")
if model_path is None:
raise ValueError("Prophet model path not found in result")
# Load the actual Prophet model from the stored path
try:
import joblib
prophet_model = joblib.load(model_path)
except Exception as e:
raise ValueError(f"Failed to load Prophet model from path {model_path}: {str(e)}")
# Prepare dataframe for prediction
pred_df = df[['ds']].copy()
@@ -273,7 +280,8 @@ class HybridProphetXGBoost:
'reg_lambda': 1.0, # L2 regularization
'objective': 'reg:squarederror',
'random_state': 42,
'n_jobs': -1
'n_jobs': -1,
'early_stopping_rounds': 10
}
# Initialize model
@@ -285,7 +293,6 @@ class HybridProphetXGBoost:
model.fit,
X_train, y_train,
eval_set=[(X_val, y_val)],
early_stopping_rounds=10,
verbose=False
)
@@ -303,109 +310,86 @@ class HybridProphetXGBoost:
train_prophet_pred: np.ndarray,
val_prophet_pred: np.ndarray,
prophet_result: Dict[str, Any]
) -> Dict[str, float]:
) -> Dict[str, Any]:
"""
Evaluate hybrid model vs Prophet-only on validation set.
Args:
train_df: Training data
val_df: Validation data
train_prophet_pred: Prophet predictions on training set
val_prophet_pred: Prophet predictions on validation set
prophet_result: Prophet training result
Returns:
Dictionary of metrics
Evaluate the overall performance of the hybrid model using threading for metrics.
"""
# Get actual values
train_actual = train_df['y'].values
val_actual = val_df['y'].values
# Get XGBoost predictions on residuals
import asyncio
# Get XGBoost predictions on training and validation
X_train = train_df[self.feature_columns].values
X_val = val_df[self.feature_columns].values
# ✅ FIX: Run blocking predict() in thread pool to avoid blocking event loop
import asyncio
train_xgb_pred = await asyncio.to_thread(self.xgb_model.predict, X_train)
val_xgb_pred = await asyncio.to_thread(self.xgb_model.predict, X_val)
# Hybrid predictions = Prophet + XGBoost residual correction
# Hybrid prediction = Prophet prediction + XGBoost residual prediction
train_hybrid_pred = train_prophet_pred + train_xgb_pred
val_hybrid_pred = val_prophet_pred + val_xgb_pred
# Calculate metrics for Prophet-only
prophet_train_mae = mean_absolute_error(train_actual, train_prophet_pred)
prophet_val_mae = mean_absolute_error(val_actual, val_prophet_pred)
prophet_train_mape = mean_absolute_percentage_error(train_actual, train_prophet_pred) * 100
prophet_val_mape = mean_absolute_percentage_error(val_actual, val_prophet_pred) * 100
# Calculate metrics for Hybrid
hybrid_train_mae = mean_absolute_error(train_actual, train_hybrid_pred)
hybrid_val_mae = mean_absolute_error(val_actual, val_hybrid_pred)
hybrid_train_mape = mean_absolute_percentage_error(train_actual, train_hybrid_pred) * 100
hybrid_val_mape = mean_absolute_percentage_error(val_actual, val_hybrid_pred) * 100
actual_train = train_df['y'].values
actual_val = val_df['y'].values
# Basic RMSE calculation
train_rmse = float(np.sqrt(np.mean((actual_train - train_hybrid_pred)**2)))
val_rmse = float(np.sqrt(np.mean((actual_val - val_hybrid_pred)**2)))
# MAE
train_mae = float(np.mean(np.abs(actual_train - train_hybrid_pred)))
val_mae = float(np.mean(np.abs(actual_val - val_hybrid_pred)))
# MAPE (with safety for zero sales)
train_mape = float(np.mean(np.abs((actual_train - train_hybrid_pred) / np.maximum(actual_train, 1))))
val_mape = float(np.mean(np.abs((actual_val - val_hybrid_pred) / np.maximum(actual_val, 1))))
# Calculate improvement
mae_improvement = ((prophet_val_mae - hybrid_val_mae) / prophet_val_mae) * 100
mape_improvement = ((prophet_val_mape - hybrid_val_mape) / prophet_val_mape) * 100
prophet_metrics = prophet_result.get("metrics", {})
prophet_val_mae = prophet_metrics.get("val_mae", val_mae) # Fallback to hybrid if missing
prophet_val_mape = prophet_metrics.get("val_mape", val_mape)
improvement_pct = 0.0
if prophet_val_mape > 0:
improvement_pct = ((prophet_val_mape - val_mape) / prophet_val_mape) * 100
metrics = {
'prophet_train_mae': float(prophet_train_mae),
'prophet_val_mae': float(prophet_val_mae),
'prophet_train_mape': float(prophet_train_mape),
'prophet_val_mape': float(prophet_val_mape),
'hybrid_train_mae': float(hybrid_train_mae),
'hybrid_val_mae': float(hybrid_val_mae),
'hybrid_train_mape': float(hybrid_train_mape),
'hybrid_val_mape': float(hybrid_val_mape),
'mae_improvement_pct': float(mae_improvement),
'mape_improvement_pct': float(mape_improvement),
'improvement_percentage': float(mape_improvement) # Primary metric
"train_rmse": train_rmse,
"val_rmse": val_rmse,
"train_mae": train_mae,
"val_mae": val_mae,
"train_mape": train_mape,
"val_mape": val_mape,
"prophet_val_mape": prophet_val_mape,
"hybrid_val_mape": val_mape,
"improvement_percentage": float(improvement_pct),
"prophet_metrics": prophet_metrics
}
logger.info(
"Hybrid model evaluation complete",
val_rmse=val_rmse,
val_mae=val_mae,
val_mape=val_mape,
improvement=improvement_pct
)
return metrics
def _package_hybrid_model(
self,
prophet_result: Dict[str, Any],
metrics: Dict[str, float],
metrics: Dict[str, Any],
tenant_id: str,
inventory_product_id: str
) -> Dict[str, Any]:
"""
Package hybrid model for storage.
Args:
prophet_result: Prophet model result
metrics: Hybrid model metrics
tenant_id: Tenant ID
inventory_product_id: Product ID
Returns:
Model package dictionary
"""
return {
'model_type': 'hybrid_prophet_xgboost',
'prophet_model': prophet_result.get('model'),
'prophet_model_path': prophet_result.get('model_path'),
'xgboost_model': self.xgb_model,
'feature_columns': self.feature_columns,
'prophet_metrics': {
'train_mae': metrics['prophet_train_mae'],
'val_mae': metrics['prophet_val_mae'],
'train_mape': metrics['prophet_train_mape'],
'val_mape': metrics['prophet_val_mape']
},
'hybrid_metrics': {
'train_mae': metrics['hybrid_train_mae'],
'val_mae': metrics['hybrid_val_mae'],
'train_mape': metrics['hybrid_train_mape'],
'val_mape': metrics['hybrid_val_mape']
},
'improvement_metrics': {
'mae_improvement_pct': metrics['mae_improvement_pct'],
'mape_improvement_pct': metrics['mape_improvement_pct']
},
'metrics': metrics,
'tenant_id': tenant_id,
'inventory_product_id': inventory_product_id,
'trained_at': datetime.now(timezone.utc).isoformat()
@@ -426,8 +410,18 @@ class HybridProphetXGBoost:
Returns:
DataFrame with predictions
"""
# Step 1: Get Prophet predictions
prophet_model = model_data['prophet_model']
# Step 1: Get Prophet model from path and make predictions
prophet_model_path = model_data.get('prophet_model_path')
if prophet_model_path is None:
raise ValueError("Prophet model path not found in model data")
# Load the Prophet model from the stored path
try:
import joblib
prophet_model = joblib.load(prophet_model_path)
except Exception as e:
raise ValueError(f"Failed to load Prophet model from path {prophet_model_path}: {str(e)}")
# ✅ FIX: Run blocking predict() in thread pool to avoid blocking event loop
import asyncio
prophet_forecast = await asyncio.to_thread(prophet_model.predict, future_df)

103
services/training/app/ml/poi_feature_integrator.py
View File

@@ -43,86 +43,79 @@ class POIFeatureIntegrator:
force_refresh: bool = False
) -> Optional[Dict[str, Any]]:
"""
Fetch POI features for tenant location.
Fetch POI features for tenant location (optimized for training).
First checks if POI context exists, if not, triggers detection.
First checks if POI context exists. If not, returns None without triggering detection.
POI detection should be triggered during tenant registration, not during training.
Args:
tenant_id: Tenant UUID
latitude: Bakery latitude
longitude: Bakery longitude
force_refresh: Force re-detection
force_refresh: Force re-detection (only use if POI context already exists)
Returns:
Dictionary with POI features or None if detection fails
Dictionary with POI features or None if not available
"""
try:
# Try to get existing POI context first
if not force_refresh:
existing_context = await self.external_client.get_poi_context(tenant_id)
if existing_context:
poi_context = existing_context.get("poi_context", {})
ml_features = poi_context.get("ml_features", {})
existing_context = await self.external_client.get_poi_context(tenant_id)
# Check if stale
is_stale = existing_context.get("is_stale", False)
if not is_stale:
if existing_context:
poi_context = existing_context.get("poi_context", {})
ml_features = poi_context.get("ml_features", {})
# Check if stale and force_refresh is requested
is_stale = existing_context.get("is_stale", False)
if not is_stale or not force_refresh:
logger.info(
"Using existing POI context",
tenant_id=tenant_id,
is_stale=is_stale,
feature_count=len(ml_features)
)
return ml_features
else:
logger.info(
"POI context is stale and force_refresh=True, refreshing",
tenant_id=tenant_id
)
# Only refresh if explicitly requested and context exists
detection_result = await self.external_client.detect_poi_for_tenant(
tenant_id=tenant_id,
latitude=latitude,
longitude=longitude,
force_refresh=True
)
if detection_result:
poi_context = detection_result.get("poi_context", {})
ml_features = poi_context.get("ml_features", {})
logger.info(
"Using existing POI context",
tenant_id=tenant_id
"POI refresh completed",
tenant_id=tenant_id,
feature_count=len(ml_features)
)
return ml_features
else:
logger.info(
"POI context is stale, refreshing",
logger.warning(
"POI refresh failed, returning existing features",
tenant_id=tenant_id
)
force_refresh = True
else:
logger.info(
"No existing POI context, will detect",
tenant_id=tenant_id
)
# Detect or refresh POIs
logger.info(
"Detecting POIs for tenant",
tenant_id=tenant_id,
location=(latitude, longitude)
)
detection_result = await self.external_client.detect_poi_for_tenant(
tenant_id=tenant_id,
latitude=latitude,
longitude=longitude,
force_refresh=force_refresh
)
if detection_result:
poi_context = detection_result.get("poi_context", {})
ml_features = poi_context.get("ml_features", {})
logger.info(
"POI detection completed",
tenant_id=tenant_id,
total_pois=poi_context.get("total_pois_detected", 0),
feature_count=len(ml_features)
)
return ml_features
return ml_features
else:
logger.error(
"POI detection failed",
logger.info(
"No existing POI context found - POI detection should be triggered during tenant registration",
tenant_id=tenant_id
)
return None
except Exception as e:
logger.error(
"Unexpected error fetching POI features",
logger.warning(
"Error fetching POI features - returning None",
tenant_id=tenant_id,
error=str(e),
exc_info=True
error=str(e)
)
return None