533 lines
19 KiB
Python
533 lines
19 KiB
Python
# services/forecasting/app/services/forecast_feedback_service.py
|
|
"""
|
|
Forecast Feedback Service
|
|
Business logic for collecting and analyzing forecast feedback
|
|
"""
|
|
|
|
from typing import List, Dict, Any, Optional
|
|
from datetime import datetime, timedelta, date
|
|
import uuid
|
|
import structlog
|
|
from dataclasses import dataclass
|
|
|
|
logger = structlog.get_logger()
|
|
|
|
|
|
@dataclass
|
|
class ForecastFeedback:
|
|
"""Data class for forecast feedback"""
|
|
feedback_id: uuid.UUID
|
|
forecast_id: uuid.UUID
|
|
tenant_id: str
|
|
feedback_type: str
|
|
confidence: str
|
|
actual_value: Optional[float]
|
|
notes: Optional[str]
|
|
feedback_data: Dict[str, Any]
|
|
created_at: datetime
|
|
created_by: Optional[str]
|
|
|
|
|
|
@dataclass
|
|
class ForecastAccuracyMetrics:
|
|
"""Data class for forecast accuracy metrics"""
|
|
forecast_id: str
|
|
total_feedback_count: int
|
|
accuracy_score: float
|
|
feedback_distribution: Dict[str, int]
|
|
average_confidence: float
|
|
last_feedback_date: Optional[datetime]
|
|
|
|
|
|
@dataclass
|
|
class ForecasterPerformanceMetrics:
|
|
"""Data class for forecaster performance metrics"""
|
|
overall_accuracy: float
|
|
total_forecasts_with_feedback: int
|
|
accuracy_by_product: Dict[str, float]
|
|
accuracy_trend: str
|
|
improvement_suggestions: List[str]
|
|
|
|
|
|
class ForecastFeedbackService:
|
|
"""
|
|
Service for managing forecast feedback and accuracy tracking
|
|
"""
|
|
|
|
def __init__(self, database_manager):
|
|
self.database_manager = database_manager
|
|
|
|
async def forecast_exists(self, tenant_id: str, forecast_id: str) -> bool:
|
|
"""
|
|
Check if a forecast exists
|
|
"""
|
|
try:
|
|
async with self.database_manager.get_session() as session:
|
|
from app.models.forecasts import Forecast
|
|
|
|
result = await session.execute(
|
|
"""
|
|
SELECT 1 FROM forecasts
|
|
WHERE tenant_id = :tenant_id AND id = :forecast_id
|
|
""",
|
|
{"tenant_id": tenant_id, "forecast_id": forecast_id}
|
|
)
|
|
return result.scalar() is not None
|
|
|
|
except Exception as e:
|
|
logger.error("Failed to check forecast existence", error=str(e))
|
|
raise Exception(f"Failed to check forecast existence: {str(e)}")
|
|
|
|
async def submit_feedback(
|
|
self,
|
|
tenant_id: str,
|
|
forecast_id: str,
|
|
feedback_type: str,
|
|
confidence: str,
|
|
actual_value: Optional[float] = None,
|
|
notes: Optional[str] = None,
|
|
feedback_data: Optional[Dict[str, Any]] = None
|
|
) -> ForecastFeedback:
|
|
"""
|
|
Submit feedback on forecast accuracy
|
|
"""
|
|
try:
|
|
async with self.database_manager.get_session() as session:
|
|
# Create feedback record
|
|
feedback_id = uuid.uuid4()
|
|
created_at = datetime.now()
|
|
|
|
# In a real implementation, this would insert into a forecast_feedback table
|
|
# For demo purposes, we'll simulate the database operation
|
|
|
|
feedback = ForecastFeedback(
|
|
feedback_id=feedback_id,
|
|
forecast_id=uuid.UUID(forecast_id),
|
|
tenant_id=tenant_id,
|
|
feedback_type=feedback_type,
|
|
confidence=confidence,
|
|
actual_value=actual_value,
|
|
notes=notes,
|
|
feedback_data=feedback_data or {},
|
|
created_at=created_at,
|
|
created_by="system" # In real implementation, this would be the user ID
|
|
)
|
|
|
|
# Simulate database insert
|
|
logger.info("Feedback submitted",
|
|
feedback_id=str(feedback_id),
|
|
forecast_id=forecast_id,
|
|
feedback_type=feedback_type)
|
|
|
|
return feedback
|
|
|
|
except Exception as e:
|
|
logger.error("Failed to submit feedback", error=str(e))
|
|
raise Exception(f"Failed to submit feedback: {str(e)}")
|
|
|
|
async def get_feedback_for_forecast(
|
|
self,
|
|
tenant_id: str,
|
|
forecast_id: str,
|
|
limit: int = 50,
|
|
offset: int = 0
|
|
) -> List[ForecastFeedback]:
|
|
"""
|
|
Get all feedback for a specific forecast
|
|
"""
|
|
try:
|
|
# In a real implementation, this would query the forecast_feedback table
|
|
# For demo purposes, we'll return simulated data
|
|
|
|
# Simulate some feedback data
|
|
simulated_feedback = []
|
|
|
|
for i in range(min(limit, 3)): # Return up to 3 simulated feedback items
|
|
feedback = ForecastFeedback(
|
|
feedback_id=uuid.uuid4(),
|
|
forecast_id=uuid.UUID(forecast_id),
|
|
tenant_id=tenant_id,
|
|
feedback_type=["too_high", "too_low", "accurate"][i % 3],
|
|
confidence=["medium", "high", "low"][i % 3],
|
|
actual_value=150.0 + i * 20 if i < 2 else None,
|
|
notes=f"Feedback sample {i+1}" if i == 0 else None,
|
|
feedback_data={"sample": i+1, "demo": True},
|
|
created_at=datetime.now() - timedelta(days=i),
|
|
created_by="demo_user"
|
|
)
|
|
simulated_feedback.append(feedback)
|
|
|
|
return simulated_feedback
|
|
|
|
except Exception as e:
|
|
logger.error("Failed to get feedback for forecast", error=str(e))
|
|
raise Exception(f"Failed to get feedback: {str(e)}")
|
|
|
|
async def calculate_accuracy_metrics(
|
|
self,
|
|
tenant_id: str,
|
|
forecast_id: str
|
|
) -> ForecastAccuracyMetrics:
|
|
"""
|
|
Calculate accuracy metrics for a forecast
|
|
"""
|
|
try:
|
|
# Get feedback for this forecast
|
|
feedback_list = await self.get_feedback_for_forecast(tenant_id, forecast_id)
|
|
|
|
if not feedback_list:
|
|
return None
|
|
|
|
# Calculate metrics
|
|
total_feedback = len(feedback_list)
|
|
|
|
# Count feedback distribution
|
|
feedback_distribution = {
|
|
"too_high": 0,
|
|
"too_low": 0,
|
|
"accurate": 0,
|
|
"uncertain": 0
|
|
}
|
|
|
|
confidence_scores = {
|
|
"low": 1,
|
|
"medium": 2,
|
|
"high": 3
|
|
}
|
|
|
|
total_confidence = 0
|
|
|
|
for feedback in feedback_list:
|
|
feedback_distribution[feedback.feedback_type] += 1
|
|
total_confidence += confidence_scores.get(feedback.confidence, 1)
|
|
|
|
# Calculate accuracy score (simplified)
|
|
accurate_count = feedback_distribution["accurate"]
|
|
accuracy_score = (accurate_count / total_feedback) * 100
|
|
|
|
# Adjust for confidence
|
|
avg_confidence = total_confidence / total_feedback
|
|
adjusted_accuracy = accuracy_score * (avg_confidence / 3) # Normalize confidence to 0-1 range
|
|
|
|
return ForecastAccuracyMetrics(
|
|
forecast_id=forecast_id,
|
|
total_feedback_count=total_feedback,
|
|
accuracy_score=round(adjusted_accuracy, 1),
|
|
feedback_distribution=feedback_distribution,
|
|
average_confidence=round(avg_confidence, 1),
|
|
last_feedback_date=max(f.created_at for f in feedback_list)
|
|
)
|
|
|
|
except Exception as e:
|
|
logger.error("Failed to calculate accuracy metrics", error=str(e))
|
|
raise Exception(f"Failed to calculate metrics: {str(e)}")
|
|
|
|
async def calculate_performance_summary(
|
|
self,
|
|
tenant_id: str,
|
|
start_date: Optional[date] = None,
|
|
end_date: Optional[date] = None,
|
|
product_id: Optional[str] = None
|
|
) -> ForecasterPerformanceMetrics:
|
|
"""
|
|
Calculate overall forecaster performance summary
|
|
"""
|
|
try:
|
|
# In a real implementation, this would aggregate data across multiple forecasts
|
|
# For demo purposes, we'll return simulated metrics
|
|
|
|
# Simulate performance data
|
|
accuracy_by_product = {
|
|
"baguette": 85.5,
|
|
"croissant": 78.2,
|
|
"pain_au_chocolat": 92.1
|
|
}
|
|
|
|
if product_id and product_id in accuracy_by_product:
|
|
# Return metrics for specific product
|
|
product_accuracy = accuracy_by_product[product_id]
|
|
accuracy_by_product = {product_id: product_accuracy}
|
|
|
|
# Calculate overall accuracy
|
|
overall_accuracy = sum(accuracy_by_product.values()) / len(accuracy_by_product)
|
|
|
|
# Determine trend (simulated)
|
|
trend_data = [82.3, 84.1, 85.5, 86.8, 88.2] # Last 5 periods
|
|
if trend_data[-1] > trend_data[0]:
|
|
trend = "improving"
|
|
elif trend_data[-1] < trend_data[0]:
|
|
trend = "declining"
|
|
else:
|
|
trend = "stable"
|
|
|
|
# Generate improvement suggestions
|
|
suggestions = []
|
|
|
|
for product, accuracy in accuracy_by_product.items():
|
|
if accuracy < 80:
|
|
suggestions.append(f"Improve {product} forecast accuracy (current: {accuracy}%)")
|
|
elif accuracy < 90:
|
|
suggestions.append(f"Consider fine-tuning {product} forecast model (current: {accuracy}%)")
|
|
|
|
if not suggestions:
|
|
suggestions.append("Overall forecast accuracy is excellent - maintain current approach")
|
|
|
|
return ForecasterPerformanceMetrics(
|
|
overall_accuracy=round(overall_accuracy, 1),
|
|
total_forecasts_with_feedback=42,
|
|
accuracy_by_product=accuracy_by_product,
|
|
accuracy_trend=trend,
|
|
improvement_suggestions=suggestions
|
|
)
|
|
|
|
except Exception as e:
|
|
logger.error("Failed to calculate performance summary", error=str(e))
|
|
raise Exception(f"Failed to calculate summary: {str(e)}")
|
|
|
|
async def get_feedback_trends(
|
|
self,
|
|
tenant_id: str,
|
|
days: int = 30,
|
|
product_id: Optional[str] = None
|
|
) -> List[Dict[str, Any]]:
|
|
"""
|
|
Get feedback trends over time
|
|
"""
|
|
try:
|
|
# Simulate trend data
|
|
trends = []
|
|
end_date = datetime.now()
|
|
|
|
# Generate daily trend data
|
|
for i in range(days):
|
|
date = end_date - timedelta(days=i)
|
|
|
|
# Simulate varying accuracy with weekly pattern
|
|
base_accuracy = 85.0
|
|
weekly_variation = 3.0 * (i % 7 / 6 - 0.5) # Weekly pattern
|
|
daily_noise = (i % 3 - 1) * 1.5 # Daily noise
|
|
|
|
accuracy = max(70, min(95, base_accuracy + weekly_variation + daily_noise))
|
|
|
|
trends.append({
|
|
'date': date.strftime('%Y-%m-%d'),
|
|
'accuracy_score': round(accuracy, 1),
|
|
'feedback_count': max(1, int(5 + i % 10)),
|
|
'confidence_score': round(2.5 + (i % 5 - 2) * 0.2, 1)
|
|
})
|
|
|
|
# Sort by date (oldest first)
|
|
trends.sort(key=lambda x: x['date'])
|
|
|
|
return trends
|
|
|
|
except Exception as e:
|
|
logger.error("Failed to get feedback trends", error=str(e))
|
|
raise Exception(f"Failed to get trends: {str(e)}")
|
|
|
|
async def trigger_retraining_from_feedback(
|
|
self,
|
|
tenant_id: str,
|
|
forecast_id: str
|
|
) -> Dict[str, Any]:
|
|
"""
|
|
Trigger model retraining based on feedback
|
|
"""
|
|
try:
|
|
# In a real implementation, this would:
|
|
# 1. Collect recent feedback data
|
|
# 2. Prepare training dataset
|
|
# 3. Submit retraining job to ML service
|
|
# 4. Return job ID
|
|
|
|
# For demo purposes, simulate a retraining job
|
|
job_id = str(uuid.uuid4())
|
|
|
|
logger.info("Retraining job triggered",
|
|
job_id=job_id,
|
|
tenant_id=tenant_id,
|
|
forecast_id=forecast_id)
|
|
|
|
return {
|
|
'job_id': job_id,
|
|
'forecasts_included': 15,
|
|
'feedback_samples_used': 42,
|
|
'status': 'queued',
|
|
'estimated_completion': (datetime.now() + timedelta(minutes=30)).isoformat()
|
|
}
|
|
|
|
except Exception as e:
|
|
logger.error("Failed to trigger retraining", error=str(e))
|
|
raise Exception(f"Failed to trigger retraining: {str(e)}")
|
|
|
|
async def get_improvement_suggestions(
|
|
self,
|
|
tenant_id: str,
|
|
forecast_id: str
|
|
) -> List[Dict[str, Any]]:
|
|
"""
|
|
Get AI-generated improvement suggestions
|
|
"""
|
|
try:
|
|
# Get accuracy metrics for this forecast
|
|
metrics = await self.calculate_accuracy_metrics(tenant_id, forecast_id)
|
|
|
|
if not metrics:
|
|
return [
|
|
{
|
|
'suggestion': 'Insufficient feedback data to generate suggestions',
|
|
'type': 'data',
|
|
'priority': 'low',
|
|
'confidence': 0.7
|
|
}
|
|
]
|
|
|
|
# Generate suggestions based on metrics
|
|
suggestions = []
|
|
|
|
# Analyze feedback distribution
|
|
feedback_dist = metrics.feedback_distribution
|
|
total_feedback = metrics.total_feedback_count
|
|
|
|
if feedback_dist['too_high'] > total_feedback * 0.4:
|
|
suggestions.append({
|
|
'suggestion': 'Forecasts are consistently too high - consider adjusting demand estimation parameters',
|
|
'type': 'bias',
|
|
'priority': 'high',
|
|
'confidence': 0.9,
|
|
'details': {
|
|
'too_high_percentage': feedback_dist['too_high'] / total_feedback * 100,
|
|
'recommended_action': 'Reduce demand estimation by 10-15%'
|
|
}
|
|
})
|
|
|
|
if feedback_dist['too_low'] > total_feedback * 0.4:
|
|
suggestions.append({
|
|
'suggestion': 'Forecasts are consistently too low - consider increasing demand estimation parameters',
|
|
'type': 'bias',
|
|
'priority': 'high',
|
|
'confidence': 0.9,
|
|
'details': {
|
|
'too_low_percentage': feedback_dist['too_low'] / total_feedback * 100,
|
|
'recommended_action': 'Increase demand estimation by 10-15%'
|
|
}
|
|
})
|
|
|
|
if metrics.accuracy_score < 70:
|
|
suggestions.append({
|
|
'suggestion': 'Low overall accuracy - consider comprehensive model review and retraining',
|
|
'type': 'model',
|
|
'priority': 'critical',
|
|
'confidence': 0.85,
|
|
'details': {
|
|
'current_accuracy': metrics.accuracy_score,
|
|
'recommended_action': 'Full model retraining with expanded feature set'
|
|
}
|
|
})
|
|
elif metrics.accuracy_score < 85:
|
|
suggestions.append({
|
|
'suggestion': 'Moderate accuracy - consider feature engineering improvements',
|
|
'type': 'features',
|
|
'priority': 'medium',
|
|
'confidence': 0.8,
|
|
'details': {
|
|
'current_accuracy': metrics.accuracy_score,
|
|
'recommended_action': 'Add weather data, promotions, and seasonal features'
|
|
}
|
|
})
|
|
|
|
if metrics.average_confidence < 2.0: # Average of medium (2) and high (3)
|
|
suggestions.append({
|
|
'suggestion': 'Low confidence in feedback - consider improving feedback collection process',
|
|
'type': 'process',
|
|
'priority': 'medium',
|
|
'confidence': 0.75,
|
|
'details': {
|
|
'average_confidence': metrics.average_confidence,
|
|
'recommended_action': 'Provide clearer guidance to users on feedback submission'
|
|
}
|
|
})
|
|
|
|
if not suggestions:
|
|
suggestions.append({
|
|
'suggestion': 'Forecast accuracy is good - consider expanding to additional products',
|
|
'type': 'expansion',
|
|
'priority': 'low',
|
|
'confidence': 0.85,
|
|
'details': {
|
|
'current_accuracy': metrics.accuracy_score,
|
|
'recommended_action': 'Extend forecasting to new product categories'
|
|
}
|
|
})
|
|
|
|
return suggestions
|
|
|
|
except Exception as e:
|
|
logger.error("Failed to generate improvement suggestions", error=str(e))
|
|
raise Exception(f"Failed to generate suggestions: {str(e)}")
|
|
|
|
|
|
# Helper class for feedback analysis
|
|
class FeedbackAnalyzer:
|
|
"""
|
|
Helper class for analyzing feedback patterns
|
|
"""
|
|
|
|
@staticmethod
|
|
def detect_feedback_patterns(feedback_list: List[ForecastFeedback]) -> Dict[str, Any]:
|
|
"""
|
|
Detect patterns in feedback data
|
|
"""
|
|
if not feedback_list:
|
|
return {'patterns': [], 'anomalies': []}
|
|
|
|
patterns = []
|
|
anomalies = []
|
|
|
|
# Simple pattern detection (in real implementation, this would be more sophisticated)
|
|
feedback_types = [f.feedback_type for f in feedback_list]
|
|
|
|
if len(set(feedback_types)) == 1:
|
|
patterns.append({
|
|
'type': 'consistent_feedback',
|
|
'pattern': f'All feedback is "{feedback_types[0]}"',
|
|
'confidence': 0.9
|
|
})
|
|
|
|
return {'patterns': patterns, 'anomalies': anomalies}
|
|
|
|
|
|
# Helper class for accuracy calculation
|
|
class AccuracyCalculator:
|
|
"""
|
|
Helper class for calculating forecast accuracy metrics
|
|
"""
|
|
|
|
@staticmethod
|
|
def calculate_mape(actual: float, predicted: float) -> float:
|
|
"""
|
|
Calculate Mean Absolute Percentage Error
|
|
"""
|
|
if actual == 0:
|
|
return 0.0
|
|
return abs((actual - predicted) / actual) * 100
|
|
|
|
@staticmethod
|
|
def calculate_rmse(actual: float, predicted: float) -> float:
|
|
"""
|
|
Calculate Root Mean Squared Error
|
|
"""
|
|
return (actual - predicted) ** 2
|
|
|
|
@staticmethod
|
|
def feedback_to_accuracy_score(feedback_type: str) -> float:
|
|
"""
|
|
Convert feedback type to accuracy score
|
|
"""
|
|
feedback_scores = {
|
|
'accurate': 100,
|
|
'too_high': 50,
|
|
'too_low': 50,
|
|
'uncertain': 75
|
|
}
|
|
return feedback_scores.get(feedback_type, 75) |