Refactor the traffic fetching system

services/training/app/ml/prophet_manager.py

@@ -162,7 +162,8 @@ class BakeryProphetManager:
                 'seasonality_mode': 'additive',
                 'daily_seasonality': False,
                 'weekly_seasonality': True,
-                'yearly_seasonality': False
+                'yearly_seasonality': False,
+                'uncertainty_samples': 100  # ✅ FIX: Minimal uncertainty sampling for very sparse data
             }
         elif zero_ratio > 0.6:
             logger.info(f"Moderate sparsity for {product_name}, using conservative optimization")
@@ -174,7 +175,8 @@ class BakeryProphetManager:
                 'seasonality_mode': 'additive',
                 'daily_seasonality': False,
                 'weekly_seasonality': True,
-                'yearly_seasonality': len(df) > 365  # Only if we have enough data
+                'yearly_seasonality': len(df) > 365,  # Only if we have enough data
+                'uncertainty_samples': 200  # ✅ FIX: Conservative uncertainty sampling for moderately sparse data
             }
         
         # Use unique seed for each product to avoid identical results
@@ -196,6 +198,16 @@ class BakeryProphetManager:
                     changepoint_scale_range = (0.001, 0.5)
                     seasonality_scale_range = (0.01, 10.0)
                 
+                # ✅ FIX: Determine appropriate uncertainty samples range based on product category
+                if product_category == 'high_volume':
+                    uncertainty_range = (300, 800)  # More samples for stable high-volume products
+                elif product_category == 'medium_volume':
+                    uncertainty_range = (200, 500)  # Moderate samples for medium volume
+                elif product_category == 'low_volume':
+                    uncertainty_range = (150, 300)  # Fewer samples for low volume
+                else:  # intermittent
+                    uncertainty_range = (100, 200)  # Minimal samples for intermittent demand
+                
                 params = {
                     'changepoint_prior_scale': trial.suggest_float(
                         'changepoint_prior_scale', 
@@ -214,7 +226,8 @@ class BakeryProphetManager:
                     'seasonality_mode': 'additive' if product_category == 'high_volume' else trial.suggest_categorical('seasonality_mode', ['additive', 'multiplicative']),
                     'daily_seasonality': trial.suggest_categorical('daily_seasonality', [True, False]),
                     'weekly_seasonality': True,  # Always keep weekly
-                    'yearly_seasonality': trial.suggest_categorical('yearly_seasonality', [True, False])
+                    'yearly_seasonality': trial.suggest_categorical('yearly_seasonality', [True, False]),
+                    'uncertainty_samples': trial.suggest_int('uncertainty_samples', uncertainty_range[0], uncertainty_range[1])  # ✅ FIX: Adaptive uncertainty sampling
                 }
                 
                 # Simple 2-fold cross-validation for speed
@@ -229,8 +242,10 @@ class BakeryProphetManager:
                         continue
                     
                     try:
-                        # Create and train model
-                        model = Prophet(**params, interval_width=0.8, uncertainty_samples=100)
+                        # Create and train model with adaptive uncertainty sampling
+                        uncertainty_samples = params.get('uncertainty_samples', 200)  # ✅ FIX: Use adaptive uncertainty samples
+                        model = Prophet(**{k: v for k, v in params.items() if k != 'uncertainty_samples'}, 
+                                      interval_width=0.8, uncertainty_samples=uncertainty_samples)
                         
                         for regressor in regressor_columns:
                             if regressor in train_data.columns:
@@ -291,6 +306,12 @@ class BakeryProphetManager:
         
         logger.info(f"Optimization completed for {product_name}. Best score: {best_score:.2f}%. "
                    f"Parameters: {best_params}")
+        
+        # ✅ FIX: Log uncertainty sampling configuration for debugging confidence intervals
+        uncertainty_samples = best_params.get('uncertainty_samples', 500)
+        logger.info(f"Prophet model will use {uncertainty_samples} uncertainty samples for {product_name} "
+                   f"(category: {product_category}, zero_ratio: {zero_ratio:.2f})")
+        
         return best_params
     
     def _classify_product(self, product_name: str, sales_data: pd.DataFrame) -> str:
@@ -329,9 +350,12 @@ class BakeryProphetManager:
             return 'intermittent'
     
     def _create_optimized_prophet_model(self, optimized_params: Dict[str, Any], regressor_columns: List[str]) -> Prophet:
-        """Create Prophet model with optimized parameters"""
+        """Create Prophet model with optimized parameters and adaptive uncertainty sampling"""
         holidays = self._get_spanish_holidays()
         
+        # Determine uncertainty samples based on data characteristics
+        uncertainty_samples = optimized_params.get('uncertainty_samples', 500)
+        
         model = Prophet(
             holidays=holidays if not holidays.empty else None,
             daily_seasonality=optimized_params.get('daily_seasonality', True),
@@ -344,7 +368,7 @@ class BakeryProphetManager:
             changepoint_range=optimized_params.get('changepoint_range', 0.8),
             interval_width=0.8,
             mcmc_samples=0,
-            uncertainty_samples=1000
+            uncertainty_samples=uncertainty_samples
         )
         
         return model