bakery-admin/bakery-ia
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve AI logic

Browse Source
This commit is contained in:
Urtzi Alfaro
2025-11-05 13:34:56 +01:00
parent 5c87fbcf48
commit 394ad3aea4
218 changed files with 30627 additions and 7658 deletions
Download Patch File Download Diff File Expand all files Collapse all files

355
services/training/app/ml/prophet_manager.py
View File

@@ -19,6 +19,8 @@ import json
from pathlib import Path
import math
import warnings
import shutil
import errno
warnings.filterwarnings('ignore')
from sqlalchemy.ext.asyncio import AsyncSession
@@ -39,6 +41,38 @@ from app.utils.distributed_lock import get_training_lock, LockAcquisitionError
logger = logging.getLogger(__name__)
def check_disk_space(path='/tmp', min_free_gb=1.0):
"""
Check if there's enough disk space available.
Args:
path: Path to check disk space for
min_free_gb: Minimum required free space in GB
Returns:
tuple: (bool: has_space, float: free_gb, float: total_gb, float: used_percent)
"""
try:
stat = shutil.disk_usage(path)
total_gb = stat.total / (1024**3)
free_gb = stat.free / (1024**3)
used_gb = stat.used / (1024**3)
used_percent = (stat.used / stat.total) * 100
has_space = free_gb >= min_free_gb
logger.info(f"Disk space check for {path}: "
f"total={total_gb:.2f}GB, free={free_gb:.2f}GB, "
f"used={used_gb:.2f}GB ({used_percent:.1f}%)")
if used_percent > 85:
logger.warning(f"Disk usage is high: {used_percent:.1f}% - this may cause issues")
return has_space, free_gb, total_gb, used_percent
except Exception as e:
logger.error(f"Failed to check disk space: {e}")
return True, 0, 0, 0 # Assume OK if we can't check
class BakeryProphetManager:
"""
Simplified Prophet Manager with built-in hyperparameter optimization.
@@ -58,10 +92,27 @@ class BakeryProphetManager:
tenant_id: str,
inventory_product_id: str,
df: pd.DataFrame,
job_id: str) -> Dict[str, Any]:
job_id: str,
product_category: 'ProductCategory' = None,
category_hyperparameters: Dict[str, Any] = None) -> Dict[str, Any]:
"""
Train a Prophet model with automatic hyperparameter optimization and distributed locking.
Args:
tenant_id: Tenant identifier
inventory_product_id: Product identifier
df: Training data DataFrame
job_id: Training job identifier
product_category: Optional product category for category-specific settings
category_hyperparameters: Optional category-specific Prophet hyperparameters
"""
# Check disk space before starting training
has_space, free_gb, total_gb, used_percent = check_disk_space('/tmp', min_free_gb=0.5)
if not has_space:
error_msg = f"Insufficient disk space: {free_gb:.2f}GB free ({used_percent:.1f}% used). Need at least 0.5GB free."
logger.error(error_msg)
raise RuntimeError(error_msg)
# Acquire distributed lock to prevent concurrent training of same product
lock = get_training_lock(tenant_id, inventory_product_id, use_advisory=True)
@@ -79,9 +130,33 @@ class BakeryProphetManager:
# Get regressor columns
regressor_columns = self._extract_regressor_columns(prophet_data)
# Automatically optimize hyperparameters
logger.info(f"Optimizing hyperparameters for {inventory_product_id}...")
best_params = await self._optimize_hyperparameters(prophet_data, inventory_product_id, regressor_columns)
# Use category-specific hyperparameters if provided, otherwise optimize
if category_hyperparameters:
logger.info(f"Using category-specific hyperparameters for {inventory_product_id} (category: {product_category.value if product_category else 'unknown'})")
best_params = category_hyperparameters.copy()
use_optimized = False # Not optimized, but category-specific
else:
# Automatically optimize hyperparameters
logger.info(f"Optimizing hyperparameters for {inventory_product_id}...")
try:
best_params = await self._optimize_hyperparameters(prophet_data, inventory_product_id, regressor_columns)
use_optimized = True
except Exception as opt_error:
logger.warning(f"Hyperparameter optimization failed for {inventory_product_id}: {opt_error}")
logger.warning("Falling back to default Prophet parameters")
# Use conservative default parameters
best_params = {
'changepoint_prior_scale': 0.05,
'seasonality_prior_scale': 10.0,
'holidays_prior_scale': 10.0,
'changepoint_range': 0.8,
'seasonality_mode': 'additive',
'daily_seasonality': False,
'weekly_seasonality': True,
'yearly_seasonality': len(prophet_data) > 365,
'uncertainty_samples': 0 # Disable uncertainty sampling to avoid cmdstan
}
use_optimized = False
# Create optimized Prophet model
model = self._create_optimized_prophet_model(best_params, regressor_columns)
@@ -91,8 +166,38 @@ class BakeryProphetManager:
if regressor in prophet_data.columns:
model.add_regressor(regressor)
# Fit the model
model.fit(prophet_data)
# Set environment variable for cmdstan tmp directory
import os
tmpdir = os.environ.get('TMPDIR', '/tmp/cmdstan')
os.makedirs(tmpdir, mode=0o777, exist_ok=True)
os.environ['TMPDIR'] = tmpdir
# Verify tmp directory is writable
test_file = os.path.join(tmpdir, f'test_write_{inventory_product_id}.tmp')
try:
with open(test_file, 'w') as f:
f.write('test')
os.remove(test_file)
logger.debug(f"Verified {tmpdir} is writable")
except Exception as e:
logger.error(f"TMPDIR {tmpdir} is not writable: {e}")
raise RuntimeError(f"Cannot write to {tmpdir}: {e}")
# Fit the model with enhanced error handling
try:
logger.info(f"Starting Prophet model fit for {inventory_product_id}")
model.fit(prophet_data)
logger.info(f"Prophet model fit completed successfully for {inventory_product_id}")
except Exception as fit_error:
error_details = {
'error_type': type(fit_error).__name__,
'error_message': str(fit_error),
'errno': getattr(fit_error, 'errno', None),
'tmpdir': tmpdir,
'disk_space': check_disk_space(tmpdir, 0)
}
logger.error(f"Prophet model fit failed for {inventory_product_id}: {error_details}")
raise RuntimeError(f"Prophet training failed: {error_details['error_message']}") from fit_error
# Calculate enhanced training metrics first
training_metrics = await self._calculate_training_metrics(model, prophet_data, best_params)
@@ -104,18 +209,39 @@ class BakeryProphetManager:
)
# Return same format as before, but with optimization info
# Ensure hyperparameters are JSON-serializable
def _serialize_hyperparameters(params):
"""Helper to ensure hyperparameters are JSON serializable"""
if not params:
return {}
safe_params = {}
for k, v in params.items():
try:
if isinstance(v, (int, float, str, bool, type(None))):
safe_params[k] = v
elif hasattr(v, 'item'): # numpy scalars
safe_params[k] = v.item()
elif isinstance(v, (list, tuple)):
safe_params[k] = [x.item() if hasattr(x, 'item') else x for x in v]
else:
safe_params[k] = float(v) if isinstance(v, (np.integer, np.floating)) else str(v)
except:
safe_params[k] = str(v) # fallback to string conversion
return safe_params
model_info = {
"model_id": model_id,
"model_path": model_path,
"type": "prophet_optimized",
"training_samples": len(prophet_data),
"features": regressor_columns,
"hyperparameters": best_params,
"hyperparameters": _serialize_hyperparameters(best_params),
"training_metrics": training_metrics,
"product_category": product_category.value if product_category else "unknown",
"trained_at": datetime.now().isoformat(),
"data_period": {
"start_date": prophet_data['ds'].min().isoformat(),
"end_date": prophet_data['ds'].max().isoformat(),
"start_date": pd.Timestamp(prophet_data['ds'].min()).isoformat(),
"end_date": pd.Timestamp(prophet_data['ds'].max()).isoformat(),
"total_days": len(prophet_data)
}
}
@@ -238,7 +364,7 @@ class BakeryProphetManager:
'daily_seasonality': trial.suggest_categorical('daily_seasonality', [True, False]),
'weekly_seasonality': True, # Always keep weekly
'yearly_seasonality': trial.suggest_categorical('yearly_seasonality', [True, False]),
'uncertainty_samples': trial.suggest_int('uncertainty_samples', uncertainty_range[0], uncertainty_range[1]) # ✅ FIX: Adaptive uncertainty sampling
'uncertainty_samples': int(trial.suggest_int('uncertainty_samples', int(uncertainty_range[0]), int(uncertainty_range[1]))) # ✅ FIX: Explicit int casting for all values
}
# Simple 2-fold cross-validation for speed
@@ -254,17 +380,32 @@ class BakeryProphetManager:
try:
# Create and train model with adaptive uncertainty sampling
uncertainty_samples = params.get('uncertainty_samples', 200) # ✅ FIX: Use adaptive uncertainty samples
model = Prophet(**{k: v for k, v in params.items() if k != 'uncertainty_samples'},
uncertainty_samples = int(params.get('uncertainty_samples', 200)) # ✅ FIX: Explicit int casting to prevent type errors
# Set environment variable for cmdstan tmp directory
import os
tmpdir = os.environ.get('TMPDIR', '/tmp/cmdstan')
os.makedirs(tmpdir, mode=0o777, exist_ok=True)
os.environ['TMPDIR'] = tmpdir
model = Prophet(**{k: v for k, v in params.items() if k != 'uncertainty_samples'},
interval_width=0.8, uncertainty_samples=uncertainty_samples)
for regressor in regressor_columns:
if regressor in train_data.columns:
model.add_regressor(regressor)
with warnings.catch_warnings():
warnings.simplefilter("ignore")
model.fit(train_data)
try:
model.fit(train_data)
except OSError as e:
# Log errno for "Operation not permitted" errors
if e.errno == errno.EPERM:
logger.error(f"Permission denied during Prophet fit (errno={e.errno}): {e}")
logger.error(f"TMPDIR: {tmpdir}, exists: {os.path.exists(tmpdir)}, "
f"writable: {os.access(tmpdir, os.W_OK)}")
raise
# Predict on validation set
future_df = model.make_future_dataframe(periods=0)
@@ -317,9 +458,9 @@ class BakeryProphetManager:
logger.info(f"Optimization completed for {inventory_product_id}. Best score: {best_score:.2f}%. "
f"Parameters: {best_params}")
# ✅ FIX: Log uncertainty sampling configuration for debugging confidence intervals
uncertainty_samples = best_params.get('uncertainty_samples', 500)
# ✅ FIX: Log uncertainty sampling configuration for debugging confidence intervals with explicit int casting
uncertainty_samples = int(best_params.get('uncertainty_samples', 500))
logger.info(f"Prophet model will use {uncertainty_samples} uncertainty samples for {inventory_product_id} "
f"(category: {product_category}, zero_ratio: {zero_ratio:.2f})")
@@ -363,25 +504,43 @@ class BakeryProphetManager:
def _create_optimized_prophet_model(self, optimized_params: Dict[str, Any], regressor_columns: List[str]) -> Prophet:
"""Create Prophet model with optimized parameters and adaptive uncertainty sampling"""
holidays = self._get_spanish_holidays()
# Determine uncertainty samples based on data characteristics
uncertainty_samples = optimized_params.get('uncertainty_samples', 500)
model = Prophet(
holidays=holidays if not holidays.empty else None,
daily_seasonality=optimized_params.get('daily_seasonality', True),
weekly_seasonality=optimized_params.get('weekly_seasonality', True),
yearly_seasonality=optimized_params.get('yearly_seasonality', True),
seasonality_mode=optimized_params.get('seasonality_mode', 'additive'),
changepoint_prior_scale=optimized_params.get('changepoint_prior_scale', 0.05),
seasonality_prior_scale=optimized_params.get('seasonality_prior_scale', 10.0),
holidays_prior_scale=optimized_params.get('holidays_prior_scale', 10.0),
changepoint_range=optimized_params.get('changepoint_range', 0.8),
interval_width=0.8,
mcmc_samples=0,
uncertainty_samples=uncertainty_samples
)
# Determine uncertainty samples based on data characteristics with explicit int casting
uncertainty_samples = int(optimized_params.get('uncertainty_samples', 500)) if optimized_params.get('uncertainty_samples') is not None else 500
# If uncertainty_samples is 0, we're in fallback mode (no cmdstan)
if uncertainty_samples == 0:
logger.info("Creating Prophet model without uncertainty sampling (fallback mode)")
model = Prophet(
holidays=holidays if not holidays.empty else None,
daily_seasonality=optimized_params.get('daily_seasonality', True),
weekly_seasonality=optimized_params.get('weekly_seasonality', True),
yearly_seasonality=optimized_params.get('yearly_seasonality', True),
seasonality_mode=optimized_params.get('seasonality_mode', 'additive'),
changepoint_prior_scale=float(optimized_params.get('changepoint_prior_scale', 0.05)),
seasonality_prior_scale=float(optimized_params.get('seasonality_prior_scale', 10.0)),
holidays_prior_scale=float(optimized_params.get('holidays_prior_scale', 10.0)),
changepoint_range=float(optimized_params.get('changepoint_range', 0.8)),
interval_width=0.8,
mcmc_samples=0,
uncertainty_samples=1 # Minimum value to avoid errors
)
else:
model = Prophet(
holidays=holidays if not holidays.empty else None,
daily_seasonality=optimized_params.get('daily_seasonality', True),
weekly_seasonality=optimized_params.get('weekly_seasonality', True),
yearly_seasonality=optimized_params.get('yearly_seasonality', True),
seasonality_mode=optimized_params.get('seasonality_mode', 'additive'),
changepoint_prior_scale=float(optimized_params.get('changepoint_prior_scale', 0.05)),
seasonality_prior_scale=float(optimized_params.get('seasonality_prior_scale', 10.0)),
holidays_prior_scale=float(optimized_params.get('holidays_prior_scale', 10.0)),
changepoint_range=float(optimized_params.get('changepoint_range', 0.8)),
interval_width=0.8,
mcmc_samples=0,
uncertainty_samples=uncertainty_samples
)
return model
# All the existing methods remain the same, just with enhanced metrics
@@ -539,8 +698,8 @@ class BakeryProphetManager:
"regressor_columns": regressor_columns,
"training_samples": len(training_data),
"data_period": {
"start_date": training_data['ds'].min().isoformat(),
"end_date": training_data['ds'].max().isoformat()
"start_date": pd.Timestamp(training_data['ds'].min()).isoformat(),
"end_date": pd.Timestamp(training_data['ds'].max()).isoformat()
},
"optimized": True,
"optimized_parameters": optimized_params or {},
@@ -566,6 +725,25 @@ class BakeryProphetManager:
# Deactivate previous models for this product
await self._deactivate_previous_models_with_session(db_session, tenant_id, inventory_product_id)
# Helper to ensure hyperparameters are JSON serializable
def _serialize_hyperparameters(params):
if not params:
return {}
safe_params = {}
for k, v in params.items():
try:
if isinstance(v, (int, float, str, bool, type(None))):
safe_params[k] = v
elif hasattr(v, 'item'): # numpy scalars
safe_params[k] = v.item()
elif isinstance(v, (list, tuple)):
safe_params[k] = [x.item() if hasattr(x, 'item') else x for x in v]
else:
safe_params[k] = float(v) if isinstance(v, (np.integer, np.floating)) else str(v)
except:
safe_params[k] = str(v) # fallback to string conversion
return safe_params
# Create new database record
db_model = TrainedModel(
id=model_id,
@@ -575,22 +753,22 @@ class BakeryProphetManager:
job_id=model_id.split('_')[0], # Extract job_id from model_id
model_path=str(model_path),
metadata_path=str(metadata_path),
hyperparameters=optimized_params or {},
features_used=regressor_columns,
hyperparameters=_serialize_hyperparameters(optimized_params or {}),
features_used=[str(f) for f in regressor_columns] if regressor_columns else [],
is_active=True,
is_production=True, # New models are production-ready
training_start_date=training_data['ds'].min().to_pydatetime().replace(tzinfo=None) if training_data['ds'].min().tz is None else training_data['ds'].min().to_pydatetime(),
training_end_date=training_data['ds'].max().to_pydatetime().replace(tzinfo=None) if training_data['ds'].max().tz is None else training_data['ds'].max().to_pydatetime(),
training_start_date=pd.Timestamp(training_data['ds'].min()).to_pydatetime().replace(tzinfo=None),
training_end_date=pd.Timestamp(training_data['ds'].max()).to_pydatetime().replace(tzinfo=None),
training_samples=len(training_data)
)
# Add training metrics if available
if training_metrics:
db_model.mape = training_metrics.get('mape')
db_model.mae = training_metrics.get('mae')
db_model.rmse = training_metrics.get('rmse')
db_model.r2_score = training_metrics.get('r2')
db_model.data_quality_score = training_metrics.get('data_quality_score')
db_model.mape = float(training_metrics.get('mape')) if training_metrics.get('mape') is not None else None
db_model.mae = float(training_metrics.get('mae')) if training_metrics.get('mae') is not None else None
db_model.rmse = float(training_metrics.get('rmse')) if training_metrics.get('rmse') is not None else None
db_model.r2_score = float(training_metrics.get('r2')) if training_metrics.get('r2') is not None else None
db_model.data_quality_score = float(training_metrics.get('data_quality_score')) if training_metrics.get('data_quality_score') is not None else None
db_session.add(db_model)
await db_session.commit()
@@ -698,7 +876,7 @@ class BakeryProphetManager:
# Ensure y values are non-negative
prophet_data['y'] = prophet_data['y'].clip(lower=0)
logger.info(f"Prepared Prophet data: {len(prophet_data)} rows, date range: {prophet_data['ds'].min()} to {prophet_data['ds'].max()}")
logger.info(f"Prepared Prophet data: {len(prophet_data)} rows, date range: {pd.Timestamp(prophet_data['ds'].min())} to {pd.Timestamp(prophet_data['ds'].max())}")
return prophet_data
@@ -714,12 +892,69 @@ class BakeryProphetManager:
logger.info(f"Identified regressor columns: {regressor_columns}")
return regressor_columns
def _get_spanish_holidays(self) -> pd.DataFrame:
"""Get Spanish holidays (unchanged)"""
def _get_spanish_holidays(self, region: str = None) -> pd.DataFrame:
"""
Get Spanish holidays dynamically using holidays library.
Supports national and regional holidays, including dynamic Easter calculation.
Args:
region: Region code (e.g., 'MD' for Madrid, 'PV' for Basque Country)
Returns:
DataFrame with holiday dates and names
"""
try:
import holidays
holidays_list = []
years = range(2020, 2035) # Extended range for better coverage
# Get Spanish holidays for each year
for year in years:
# National holidays
spain_holidays = holidays.Spain(years=year, prov=region)
for date, name in spain_holidays.items():
holidays_list.append({
'holiday': self._normalize_holiday_name(name),
'ds': pd.Timestamp(date),
'lower_window': 0,
'upper_window': 0 # Can be adjusted for multi-day holidays
})
if holidays_list:
holidays_df = pd.DataFrame(holidays_list)
# Remove duplicates (some holidays may repeat)
holidays_df = holidays_df.drop_duplicates(subset=['ds', 'holiday'])
holidays_df = holidays_df.sort_values('ds').reset_index(drop=True)
logger.info(f"Loaded {len(holidays_df)} Spanish holidays dynamically",
region=region or 'National',
years=f"{min(years)}-{max(years)}")
return holidays_df
else:
return pd.DataFrame()
except Exception as e:
logger.warning(f"Could not load Spanish holidays dynamically: {str(e)}")
# Fallback to minimal hardcoded holidays
return self._get_fallback_holidays()
def _normalize_holiday_name(self, name: str) -> str:
"""Normalize holiday name to a consistent format for Prophet"""
# Convert to lowercase and replace spaces with underscores
normalized = name.lower().replace(' ', '_').replace("'", '')
# Remove special characters
normalized = ''.join(c for c in normalized if c.isalnum() or c == '_')
return normalized
def _get_fallback_holidays(self) -> pd.DataFrame:
"""Fallback to basic hardcoded holidays if dynamic loading fails"""
try:
holidays_list = []
years = range(2020, 2030)
years = range(2020, 2035)
for year in years:
holidays_list.extend([
{'holiday': 'new_year', 'ds': f'{year}-01-01'},
@@ -732,14 +967,10 @@ class BakeryProphetManager:
{'holiday': 'immaculate_conception', 'ds': f'{year}-12-08'},
{'holiday': 'christmas', 'ds': f'{year}-12-25'}
])
if holidays_list:
holidays_df = pd.DataFrame(holidays_list)
holidays_df['ds'] = pd.to_datetime(holidays_df['ds'])
return holidays_df
else:
return pd.DataFrame()
holidays_df = pd.DataFrame(holidays_list)
holidays_df['ds'] = pd.to_datetime(holidays_df['ds'])
return holidays_df
except Exception as e:
logger.warning(f"Could not load Spanish holidays: {str(e)}")
logger.error(f"Fallback holidays failed: {e}")
return pd.DataFrame()