417 lines
16 KiB
Python
417 lines
16 KiB
Python
|
# services/forecasting/app/api/forecast_feedback.py
|
||
|
|
"""
|
||
|
|
Forecast Feedback API - Endpoints for collecting and analyzing forecast feedback
|
||
|
|
"""
|
||
|
|
|
||
|
|
import structlog
|
||
|
|
from fastapi import APIRouter, Depends, HTTPException, status, Query, Path, Body
|
||
|
|
from typing import List, Optional, Dict, Any
|
||
|
|
from datetime import date, datetime
|
||
|
|
import uuid
|
||
|
|
import enum
|
||
|
|
from pydantic import BaseModel, Field
|
||
|
|
|
||
|
|
from app.services.forecast_feedback_service import ForecastFeedbackService
|
||
|
|
from shared.database.base import create_database_manager
|
||
|
|
from app.core.config import settings
|
||
|
|
from shared.routing import RouteBuilder
|
||
|
|
from shared.auth.tenant_access import verify_tenant_permission_dep
|
||
|
|
|
||
|
|
route_builder = RouteBuilder('forecasting')
|
||
|
|
logger = structlog.get_logger()
|
||
|
|
router = APIRouter(tags=["forecast-feedback"])
|
||
|
|
|
||
|
|
|
||
|
|
# Enums for feedback types
|
||
|
|
class FeedbackType(str, enum.Enum):
|
||
|
|
"""Type of feedback on forecast accuracy"""
|
||
|
|
TOO_HIGH = "too_high"
|
||
|
|
TOO_LOW = "too_low"
|
||
|
|
ACCURATE = "accurate"
|
||
|
|
UNCERTAIN = "uncertain"
|
||
|
|
|
||
|
|
|
||
|
|
class FeedbackConfidence(str, enum.Enum):
|
||
|
|
"""Confidence level of the feedback provider"""
|
||
|
|
LOW = "low"
|
||
|
|
MEDIUM = "medium"
|
||
|
|
HIGH = "high"
|
||
|
|
|
||
|
|
|
||
|
|
# Pydantic models
|
||
|
|
from pydantic import BaseModel, Field
|
||
|
|
|
||
|
|
class ForecastFeedbackRequest(BaseModel):
|
||
|
|
"""Request model for submitting forecast feedback"""
|
||
|
|
feedback_type: FeedbackType = Field(..., description="Type of feedback on forecast accuracy")
|
||
|
|
confidence: FeedbackConfidence = Field(..., description="Confidence level of the feedback provider")
|
||
|
|
actual_value: Optional[float] = Field(None, description="Actual observed value")
|
||
|
|
notes: Optional[str] = Field(None, description="Additional notes about the feedback")
|
||
|
|
feedback_data: Optional[Dict[str, Any]] = Field(None, description="Additional feedback data")
|
||
|
|
|
||
|
|
|
||
|
|
class ForecastFeedbackResponse(BaseModel):
|
||
|
|
"""Response model for forecast feedback"""
|
||
|
|
feedback_id: str = Field(..., description="Unique feedback ID")
|
||
|
|
forecast_id: str = Field(..., description="Forecast ID this feedback relates to")
|
||
|
|
tenant_id: str = Field(..., description="Tenant ID")
|
||
|
|
feedback_type: FeedbackType = Field(..., description="Type of feedback")
|
||
|
|
confidence: FeedbackConfidence = Field(..., description="Confidence level")
|
||
|
|
actual_value: Optional[float] = Field(None, description="Actual value observed")
|
||
|
|
notes: Optional[str] = Field(None, description="Feedback notes")
|
||
|
|
feedback_data: Dict[str, Any] = Field(..., description="Additional feedback data")
|
||
|
|
created_at: datetime = Field(..., description="When feedback was created")
|
||
|
|
created_by: Optional[str] = Field(None, description="Who created the feedback")
|
||
|
|
|
||
|
|
|
||
|
|
class ForecastAccuracyMetrics(BaseModel):
|
||
|
|
"""Accuracy metrics for a forecast"""
|
||
|
|
forecast_id: str = Field(..., description="Forecast ID")
|
||
|
|
total_feedback_count: int = Field(..., description="Total feedback received")
|
||
|
|
accuracy_score: float = Field(..., description="Calculated accuracy score (0-100)")
|
||
|
|
feedback_distribution: Dict[str, int] = Field(..., description="Distribution of feedback types")
|
||
|
|
average_confidence: float = Field(..., description="Average confidence score")
|
||
|
|
last_feedback_date: Optional[datetime] = Field(None, description="Most recent feedback date")
|
||
|
|
|
||
|
|
|
||
|
|
class ForecasterPerformanceMetrics(BaseModel):
|
||
|
|
"""Performance metrics for the forecasting system"""
|
||
|
|
overall_accuracy: float = Field(..., description="Overall system accuracy score")
|
||
|
|
total_forecasts_with_feedback: int = Field(..., description="Total forecasts with feedback")
|
||
|
|
accuracy_by_product: Dict[str, float] = Field(..., description="Accuracy by product type")
|
||
|
|
accuracy_trend: str = Field(..., description="Trend direction: improving, declining, stable")
|
||
|
|
improvement_suggestions: List[str] = Field(..., description="AI-generated improvement suggestions")
|
||
|
|
|
||
|
|
|
||
|
|
def get_forecast_feedback_service():
|
||
|
|
"""Dependency injection for ForecastFeedbackService"""
|
||
|
|
database_manager = create_database_manager(settings.DATABASE_URL, "forecasting-service")
|
||
|
|
return ForecastFeedbackService(database_manager)
|
||
|
|
|
||
|
|
|
||
|
|
@router.post(
|
||
|
|
route_builder.build_nested_resource_route("forecasts", "forecast_id", "feedback"),
|
||
|
|
response_model=ForecastFeedbackResponse,
|
||
|
|
status_code=status.HTTP_201_CREATED
|
||
|
|
)
|
||
|
|
async def submit_forecast_feedback(
|
||
|
|
tenant_id: str = Path(..., description="Tenant ID"),
|
||
|
|
forecast_id: str = Path(..., description="Forecast ID"),
|
||
|
|
feedback_request: ForecastFeedbackRequest = Body(...),
|
||
|
|
forecast_feedback_service: ForecastFeedbackService = Depends(get_forecast_feedback_service),
|
||
|
|
verified_tenant: str = Depends(verify_tenant_permission_dep)
|
||
|
|
):
|
||
|
|
"""
|
||
|
|
Submit feedback on forecast accuracy
|
||
|
|
|
||
|
|
Allows users to provide feedback on whether forecasts were accurate, too high, or too low.
|
||
|
|
This feedback is used to improve future forecast accuracy through continuous learning.
|
||
|
|
"""
|
||
|
|
try:
|
||
|
|
logger.info("Submitting forecast feedback",
|
||
|
|
tenant_id=tenant_id, forecast_id=forecast_id,
|
||
|
|
feedback_type=feedback_request.feedback_type)
|
||
|
|
|
||
|
|
# Validate forecast exists
|
||
|
|
forecast_exists = await forecast_feedback_service.forecast_exists(tenant_id, forecast_id)
|
||
|
|
if not forecast_exists:
|
||
|
|
raise HTTPException(
|
||
|
|
status_code=status.HTTP_404_NOT_FOUND,
|
||
|
|
detail="Forecast not found"
|
||
|
|
)
|
||
|
|
|
||
|
|
# Submit feedback
|
||
|
|
feedback = await forecast_feedback_service.submit_feedback(
|
||
|
|
tenant_id=tenant_id,
|
||
|
|
forecast_id=forecast_id,
|
||
|
|
feedback_type=feedback_request.feedback_type,
|
||
|
|
confidence=feedback_request.confidence,
|
||
|
|
actual_value=feedback_request.actual_value,
|
||
|
|
notes=feedback_request.notes,
|
||
|
|
feedback_data=feedback_request.feedback_data
|
||
|
|
)
|
||
|
|
|
||
|
|
return {
|
||
|
|
'feedback_id': str(feedback.feedback_id),
|
||
|
|
'forecast_id': str(feedback.forecast_id),
|
||
|
|
'tenant_id': feedback.tenant_id,
|
||
|
|
'feedback_type': feedback.feedback_type,
|
||
|
|
'confidence': feedback.confidence,
|
||
|
|
'actual_value': feedback.actual_value,
|
||
|
|
'notes': feedback.notes,
|
||
|
|
'feedback_data': feedback.feedback_data or {},
|
||
|
|
'created_at': feedback.created_at,
|
||
|
|
'created_by': feedback.created_by
|
||
|
|
}
|
||
|
|
|
||
|
|
except HTTPException:
|
||
|
|
raise
|
||
|
|
except ValueError as e:
|
||
|
|
logger.error("Invalid forecast ID", error=str(e))
|
||
|
|
raise HTTPException(
|
||
|
|
status_code=status.HTTP_400_BAD_REQUEST,
|
||
|
|
detail="Invalid forecast ID format"
|
||
|
|
)
|
||
|
|
except Exception as e:
|
||
|
|
logger.error("Failed to submit forecast feedback", error=str(e), tenant_id=tenant_id)
|
||
|
|
raise HTTPException(
|
||
|
|
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||
|
|
detail="Failed to submit feedback"
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
@router.get(
|
||
|
|
route_builder.build_nested_resource_route("forecasts", "forecast_id", "feedback"),
|
||
|
|
response_model=List[ForecastFeedbackResponse]
|
||
|
|
)
|
||
|
|
async def get_forecast_feedback(
|
||
|
|
tenant_id: str = Path(..., description="Tenant ID"),
|
||
|
|
forecast_id: str = Path(..., description="Forecast ID"),
|
||
|
|
limit: int = Query(50, ge=1, le=1000),
|
||
|
|
offset: int = Query(0, ge=0),
|
||
|
|
forecast_feedback_service: ForecastFeedbackService = Depends(get_forecast_feedback_service),
|
||
|
|
verified_tenant: str = Depends(verify_tenant_permission_dep)
|
||
|
|
):
|
||
|
|
"""
|
||
|
|
Get all feedback for a specific forecast
|
||
|
|
|
||
|
|
Retrieves historical feedback submissions for analysis and auditing.
|
||
|
|
"""
|
||
|
|
try:
|
||
|
|
logger.info("Getting forecast feedback", tenant_id=tenant_id, forecast_id=forecast_id)
|
||
|
|
|
||
|
|
feedback_list = await forecast_feedback_service.get_feedback_for_forecast(
|
||
|
|
tenant_id=tenant_id,
|
||
|
|
forecast_id=forecast_id,
|
||
|
|
limit=limit,
|
||
|
|
offset=offset
|
||
|
|
)
|
||
|
|
|
||
|
|
return [
|
||
|
|
ForecastFeedbackResponse(
|
||
|
|
feedback_id=str(f.feedback_id),
|
||
|
|
forecast_id=str(f.forecast_id),
|
||
|
|
tenant_id=f.tenant_id,
|
||
|
|
feedback_type=f.feedback_type,
|
||
|
|
confidence=f.confidence,
|
||
|
|
actual_value=f.actual_value,
|
||
|
|
notes=f.notes,
|
||
|
|
feedback_data=f.feedback_data or {},
|
||
|
|
created_at=f.created_at,
|
||
|
|
created_by=f.created_by
|
||
|
|
) for f in feedback_list
|
||
|
|
]
|
||
|
|
|
||
|
|
except Exception as e:
|
||
|
|
logger.error("Failed to get forecast feedback", error=str(e), tenant_id=tenant_id)
|
||
|
|
raise HTTPException(
|
||
|
|
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||
|
|
detail="Failed to retrieve feedback"
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
@router.get(
|
||
|
|
route_builder.build_nested_resource_route("forecasts", "forecast_id", "accuracy"),
|
||
|
|
response_model=ForecastAccuracyMetrics
|
||
|
|
)
|
||
|
|
async def get_forecast_accuracy_metrics(
|
||
|
|
tenant_id: str = Path(..., description="Tenant ID"),
|
||
|
|
forecast_id: str = Path(..., description="Forecast ID"),
|
||
|
|
forecast_feedback_service: ForecastFeedbackService = Depends(get_forecast_feedback_service),
|
||
|
|
verified_tenant: str = Depends(verify_tenant_permission_dep)
|
||
|
|
):
|
||
|
|
"""
|
||
|
|
Get accuracy metrics for a specific forecast
|
||
|
|
|
||
|
|
Calculates accuracy scores based on feedback and actual vs predicted values.
|
||
|
|
"""
|
||
|
|
try:
|
||
|
|
logger.info("Getting forecast accuracy metrics", tenant_id=tenant_id, forecast_id=forecast_id)
|
||
|
|
|
||
|
|
metrics = await forecast_feedback_service.calculate_accuracy_metrics(
|
||
|
|
tenant_id=tenant_id,
|
||
|
|
forecast_id=forecast_id
|
||
|
|
)
|
||
|
|
|
||
|
|
if not metrics:
|
||
|
|
raise HTTPException(
|
||
|
|
status_code=status.HTTP_404_NOT_FOUND,
|
||
|
|
detail="No accuracy metrics available for this forecast"
|
||
|
|
)
|
||
|
|
|
||
|
|
return {
|
||
|
|
'forecast_id': metrics.forecast_id,
|
||
|
|
'total_feedback_count': metrics.total_feedback_count,
|
||
|
|
'accuracy_score': metrics.accuracy_score,
|
||
|
|
'feedback_distribution': metrics.feedback_distribution,
|
||
|
|
'average_confidence': metrics.average_confidence,
|
||
|
|
'last_feedback_date': metrics.last_feedback_date
|
||
|
|
}
|
||
|
|
|
||
|
|
except Exception as e:
|
||
|
|
logger.error("Failed to get forecast accuracy metrics", error=str(e), tenant_id=tenant_id)
|
||
|
|
raise HTTPException(
|
||
|
|
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||
|
|
detail="Failed to calculate accuracy metrics"
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
@router.get(
|
||
|
|
route_builder.build_base_route("forecasts", "accuracy-summary"),
|
||
|
|
response_model=ForecasterPerformanceMetrics
|
||
|
|
)
|
||
|
|
async def get_forecaster_performance_summary(
|
||
|
|
tenant_id: str = Path(..., description="Tenant ID"),
|
||
|
|
start_date: Optional[date] = Query(None, description="Start date filter"),
|
||
|
|
end_date: Optional[date] = Query(None, description="End date filter"),
|
||
|
|
product_id: Optional[str] = Query(None, description="Filter by product ID"),
|
||
|
|
forecast_feedback_service: ForecastFeedbackService = Depends(get_forecast_feedback_service),
|
||
|
|
verified_tenant: str = Depends(verify_tenant_permission_dep)
|
||
|
|
):
|
||
|
|
"""
|
||
|
|
Get overall forecaster performance summary
|
||
|
|
|
||
|
|
Aggregates accuracy metrics across all forecasts to assess overall system performance
|
||
|
|
and identify areas for improvement.
|
||
|
|
"""
|
||
|
|
try:
|
||
|
|
logger.info("Getting forecaster performance summary", tenant_id=tenant_id)
|
||
|
|
|
||
|
|
metrics = await forecast_feedback_service.calculate_performance_summary(
|
||
|
|
tenant_id=tenant_id,
|
||
|
|
start_date=start_date,
|
||
|
|
end_date=end_date,
|
||
|
|
product_id=product_id
|
||
|
|
)
|
||
|
|
|
||
|
|
return {
|
||
|
|
'overall_accuracy': metrics.overall_accuracy,
|
||
|
|
'total_forecasts_with_feedback': metrics.total_forecasts_with_feedback,
|
||
|
|
'accuracy_by_product': metrics.accuracy_by_product,
|
||
|
|
'accuracy_trend': metrics.accuracy_trend,
|
||
|
|
'improvement_suggestions': metrics.improvement_suggestions
|
||
|
|
}
|
||
|
|
|
||
|
|
except Exception as e:
|
||
|
|
logger.error("Failed to get forecaster performance summary", error=str(e), tenant_id=tenant_id)
|
||
|
|
raise HTTPException(
|
||
|
|
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||
|
|
detail="Failed to calculate performance summary"
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
@router.get(
|
||
|
|
route_builder.build_base_route("forecasts", "feedback-trends")
|
||
|
|
)
|
||
|
|
async def get_feedback_trends(
|
||
|
|
tenant_id: str = Path(..., description="Tenant ID"),
|
||
|
|
days: int = Query(30, ge=7, le=365, description="Number of days to analyze"),
|
||
|
|
product_id: Optional[str] = Query(None, description="Filter by product ID"),
|
||
|
|
forecast_feedback_service: ForecastFeedbackService = Depends(get_forecast_feedback_service),
|
||
|
|
verified_tenant: str = Depends(verify_tenant_permission_dep)
|
||
|
|
):
|
||
|
|
"""
|
||
|
|
Get feedback trends over time
|
||
|
|
|
||
|
|
Analyzes how forecast accuracy and feedback patterns change over time.
|
||
|
|
"""
|
||
|
|
try:
|
||
|
|
logger.info("Getting feedback trends", tenant_id=tenant_id, days=days)
|
||
|
|
|
||
|
|
trends = await forecast_feedback_service.get_feedback_trends(
|
||
|
|
tenant_id=tenant_id,
|
||
|
|
days=days,
|
||
|
|
product_id=product_id
|
||
|
|
)
|
||
|
|
|
||
|
|
return {
|
||
|
|
'success': True,
|
||
|
|
'trends': trends,
|
||
|
|
'period': f'Last {days} days'
|
||
|
|
}
|
||
|
|
|
||
|
|
except Exception as e:
|
||
|
|
logger.error("Failed to get feedback trends", error=str(e), tenant_id=tenant_id)
|
||
|
|
raise HTTPException(
|
||
|
|
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||
|
|
detail="Failed to retrieve feedback trends"
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
@router.post(
|
||
|
|
route_builder.build_resource_action_route("forecasts", "forecast_id", "retrain")
|
||
|
|
)
|
||
|
|
async def trigger_retraining_from_feedback(
|
||
|
|
tenant_id: str = Path(..., description="Tenant ID"),
|
||
|
|
forecast_id: str = Path(..., description="Forecast ID"),
|
||
|
|
forecast_feedback_service: ForecastFeedbackService = Depends(get_forecast_feedback_service),
|
||
|
|
verified_tenant: str = Depends(verify_tenant_permission_dep)
|
||
|
|
):
|
||
|
|
"""
|
||
|
|
Trigger model retraining based on feedback
|
||
|
|
|
||
|
|
Initiates a retraining job using recent feedback to improve forecast accuracy.
|
||
|
|
"""
|
||
|
|
try:
|
||
|
|
logger.info("Triggering retraining from feedback", tenant_id=tenant_id, forecast_id=forecast_id)
|
||
|
|
|
||
|
|
result = await forecast_feedback_service.trigger_retraining_from_feedback(
|
||
|
|
tenant_id=tenant_id,
|
||
|
|
forecast_id=forecast_id
|
||
|
|
)
|
||
|
|
|
||
|
|
return {
|
||
|
|
'success': True,
|
||
|
|
'message': 'Retraining job initiated successfully',
|
||
|
|
'job_id': result.job_id,
|
||
|
|
'forecasts_included': result.forecasts_included,
|
||
|
|
'feedback_samples_used': result.feedback_samples_used
|
||
|
|
}
|
||
|
|
|
||
|
|
except Exception as e:
|
||
|
|
logger.error("Failed to trigger retraining", error=str(e), tenant_id=tenant_id)
|
||
|
|
raise HTTPException(
|
||
|
|
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||
|
|
detail="Failed to initiate retraining"
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
@router.get(
|
||
|
|
route_builder.build_resource_action_route("forecasts", "forecast_id", "suggestions")
|
||
|
|
)
|
||
|
|
async def get_improvement_suggestions(
|
||
|
|
tenant_id: str = Path(..., description="Tenant ID"),
|
||
|
|
forecast_id: str = Path(..., description="Forecast ID"),
|
||
|
|
forecast_feedback_service: ForecastFeedbackService = Depends(get_forecast_feedback_service),
|
||
|
|
verified_tenant: str = Depends(verify_tenant_permission_dep)
|
||
|
|
):
|
||
|
|
"""
|
||
|
|
Get AI-generated improvement suggestions for a forecast
|
||
|
|
|
||
|
|
Analyzes feedback patterns and suggests specific improvements for forecast accuracy.
|
||
|
|
"""
|
||
|
|
try:
|
||
|
|
logger.info("Getting improvement suggestions", tenant_id=tenant_id, forecast_id=forecast_id)
|
||
|
|
|
||
|
|
suggestions = await forecast_feedback_service.get_improvement_suggestions(
|
||
|
|
tenant_id=tenant_id,
|
||
|
|
forecast_id=forecast_id
|
||
|
|
)
|
||
|
|
|
||
|
|
return {
|
||
|
|
'success': True,
|
||
|
|
'forecast_id': forecast_id,
|
||
|
|
'suggestions': suggestions,
|
||
|
|
'confidence_scores': [s.get('confidence', 0.8) for s in suggestions]
|
||
|
|
}
|
||
|
|
|
||
|
|
except Exception as e:
|
||
|
|
logger.error("Failed to get improvement suggestions", error=str(e), tenant_id=tenant_id)
|
||
|
|
raise HTTPException(
|
||
|
|
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||
|
|
detail="Failed to generate suggestions"
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
# Import datetime at runtime to avoid circular imports
|
||
|
|
from datetime import datetime, timedelta
|