bakery-admin/bakery-ia
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

REFACTOR - Database logic

Browse Source
This commit is contained in:
Urtzi Alfaro
2025-08-08 09:08:41 +02:00
parent 0154365bfc
commit 488bb3ef93
113 changed files with 22842 additions and 6503 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
services/data/app/api/__init__.py
View File

@@ -0,0 +1,14 @@
"""
Data Service API Layer
API endpoints for data operations
"""
from .sales import router as sales_router
from .traffic import router as traffic_router
from .weather import router as weather_router
__all__ = [
"sales_router",
"traffic_router",
"weather_router"
]

493
services/data/app/api/sales.py
View File

@@ -1,18 +1,15 @@
# ================================================================
# services/data/app/api/sales.py - FIXED FOR NEW TENANT-SCOPED ARCHITECTURE
# ================================================================
"""Sales data API endpoints with tenant-scoped URLs"""
"""
Enhanced Sales API Endpoints
Updated to use repository pattern and enhanced services with dependency injection
"""
from fastapi import APIRouter, Depends, HTTPException, UploadFile, File, Form, Query, Response, Path
from fastapi.responses import StreamingResponse
from sqlalchemy.ext.asyncio import AsyncSession
from typing import List, Optional, Dict, Any
from uuid import UUID
from datetime import datetime
import base64
import structlog
from app.core.database import get_db
from app.schemas.sales import (
SalesDataCreate,
SalesDataResponse,
@@ -20,50 +17,61 @@ from app.schemas.sales import (
SalesDataImport,
SalesImportResult,
SalesValidationResult,
SalesValidationRequest,
SalesExportRequest
)
from app.services.sales_service import SalesService
from app.services.data_import_service import DataImportService
from app.services.data_import_service import EnhancedDataImportService
from app.services.messaging import (
publish_sales_created,
publish_data_imported,
publish_export_completed
)
# Import unified authentication from shared library
from shared.database.base import create_database_manager
from shared.auth.decorators import get_current_user_dep
router = APIRouter(tags=["sales"])
router = APIRouter(tags=["enhanced-sales"])
logger = structlog.get_logger()
# ================================================================
# TENANT-SCOPED SALES ENDPOINTS
# ================================================================
def get_sales_service():
"""Dependency injection for SalesService"""
from app.core.config import settings
database_manager = create_database_manager(settings.DATABASE_URL, "data-service")
return SalesService(database_manager)
def get_import_service():
"""Dependency injection for EnhancedDataImportService"""
from app.core.config import settings
database_manager = create_database_manager(settings.DATABASE_URL, "data-service")
return EnhancedDataImportService(database_manager)
@router.post("/tenants/{tenant_id}/sales", response_model=SalesDataResponse)
async def create_sales_record(
sales_data: SalesDataCreate,
tenant_id: UUID = Path(..., description="Tenant ID"),
current_user: Dict[str, Any] = Depends(get_current_user_dep),
db: AsyncSession = Depends(get_db)
sales_service: SalesService = Depends(get_sales_service)
):
"""Create a new sales record for tenant"""
"""Create a new sales record using repository pattern"""
try:
logger.debug("Creating sales record",
product=sales_data.product_name,
quantity=sales_data.quantity_sold,
tenant_id=tenant_id,
user_id=current_user["user_id"])
logger.info("Creating sales record with repository pattern",
product=sales_data.product_name,
quantity=sales_data.quantity_sold,
tenant_id=tenant_id,
user_id=current_user["user_id"])
# Override tenant_id from URL path (gateway already verified access)
# Override tenant_id from URL path
sales_data.tenant_id = tenant_id
record = await SalesService.create_sales_record(sales_data, db)
record = await sales_service.create_sales_record(sales_data, str(tenant_id))
# Publish event (non-blocking)
try:
await publish_sales_created({
"tenant_id": tenant_id,
"tenant_id": str(tenant_id),
"product_name": sales_data.product_name,
"quantity_sold": sales_data.quantity_sold,
"revenue": sales_data.revenue,
@@ -73,9 +81,8 @@ async def create_sales_record(
})
except Exception as pub_error:
logger.warning("Failed to publish sales created event", error=str(pub_error))
# Continue - event failure shouldn't break API
logger.info("Successfully created sales record",
logger.info("Successfully created sales record using repository",
record_id=record.id,
tenant_id=tenant_id)
return record
@@ -86,47 +93,6 @@ async def create_sales_record(
tenant_id=tenant_id)
raise HTTPException(status_code=500, detail=f"Failed to create sales record: {str(e)}")
@router.post("/tenants/{tenant_id}/sales/bulk", response_model=List[SalesDataResponse])
async def create_bulk_sales(
sales_data: List[SalesDataCreate],
tenant_id: UUID = Path(..., description="Tenant ID"),
current_user: Dict[str, Any] = Depends(get_current_user_dep),
db: AsyncSession = Depends(get_db)
):
"""Create multiple sales records for tenant"""
try:
logger.debug("Creating bulk sales records",
count=len(sales_data),
tenant_id=tenant_id)
# Override tenant_id for all records
for record in sales_data:
record.tenant_id = tenant_id
records = await SalesService.create_bulk_sales(sales_data, db)
# Publish event
try:
await publish_data_imported({
"tenant_id": tenant_id,
"type": "bulk_create",
"records_created": len(records),
"created_by": current_user["user_id"],
"timestamp": datetime.utcnow().isoformat()
})
except Exception as pub_error:
logger.warning("Failed to publish bulk import event", error=str(pub_error))
logger.info("Successfully created bulk sales records",
count=len(records),
tenant_id=tenant_id)
return records
except Exception as e:
logger.error("Failed to create bulk sales records",
error=str(e),
tenant_id=tenant_id)
raise HTTPException(status_code=500, detail=f"Failed to create bulk sales records: {str(e)}")
@router.get("/tenants/{tenant_id}/sales", response_model=List[SalesDataResponse])
async def get_sales_data(
@@ -134,10 +100,8 @@ async def get_sales_data(
start_date: Optional[datetime] = Query(None, description="Start date filter"),
end_date: Optional[datetime] = Query(None, description="End date filter"),
product_name: Optional[str] = Query(None, description="Product name filter"),
# ✅ FIX: Add missing pagination parameters
limit: Optional[int] = Query(1000, le=5000, description="Maximum number of records to return"),
offset: Optional[int] = Query(0, ge=0, description="Number of records to skip"),
# ✅ FIX: Add additional filtering parameters
product_names: Optional[List[str]] = Query(None, description="Multiple product name filters"),
location_ids: Optional[List[str]] = Query(None, description="Location ID filters"),
sources: Optional[List[str]] = Query(None, description="Source filters"),
@@ -146,19 +110,18 @@ async def get_sales_data(
min_revenue: Optional[float] = Query(None, description="Minimum revenue filter"),
max_revenue: Optional[float] = Query(None, description="Maximum revenue filter"),
current_user: Dict[str, Any] = Depends(get_current_user_dep),
db: AsyncSession = Depends(get_db)
sales_service: SalesService = Depends(get_sales_service)
):
"""Get sales data for tenant with filters and pagination"""
"""Get sales data using repository pattern with enhanced filtering"""
try:
logger.debug("Querying sales data",
logger.debug("Querying sales data with repository pattern",
tenant_id=tenant_id,
start_date=start_date,
end_date=end_date,
product_name=product_name,
limit=limit,
offset=offset)
# ✅ FIX: Create complete SalesDataQuery with all parameters
# Create enhanced query
query = SalesDataQuery(
tenant_id=tenant_id,
start_date=start_date,
@@ -170,17 +133,15 @@ async def get_sales_data(
max_quantity=max_quantity,
min_revenue=min_revenue,
max_revenue=max_revenue,
limit=limit, # ✅ Now properly passed from query params
offset=offset # ✅ Now properly passed from query params
limit=limit,
offset=offset
)
records = await SalesService.get_sales_data(query, db)
records = await sales_service.get_sales_data(query)
logger.debug("Successfully retrieved sales data",
logger.debug("Successfully retrieved sales data using repository",
count=len(records),
tenant_id=tenant_id,
limit=limit,
offset=offset)
tenant_id=tenant_id)
return records
except Exception as e:
@@ -189,17 +150,78 @@ async def get_sales_data(
tenant_id=tenant_id)
raise HTTPException(status_code=500, detail=f"Failed to query sales data: {str(e)}")
@router.get("/tenants/{tenant_id}/sales/analytics")
async def get_sales_analytics(
tenant_id: UUID = Path(..., description="Tenant ID"),
start_date: Optional[datetime] = Query(None, description="Start date"),
end_date: Optional[datetime] = Query(None, description="End date"),
current_user: Dict[str, Any] = Depends(get_current_user_dep),
sales_service: SalesService = Depends(get_sales_service)
):
"""Get sales analytics using repository pattern"""
try:
logger.debug("Getting sales analytics with repository pattern",
tenant_id=tenant_id,
start_date=start_date,
end_date=end_date)
analytics = await sales_service.get_sales_analytics(
str(tenant_id), start_date, end_date
)
logger.debug("Analytics generated successfully using repository", tenant_id=tenant_id)
return analytics
except Exception as e:
logger.error("Failed to generate sales analytics",
error=str(e),
tenant_id=tenant_id)
raise HTTPException(status_code=500, detail=f"Failed to generate analytics: {str(e)}")
@router.get("/tenants/{tenant_id}/sales/aggregation")
async def get_sales_aggregation(
tenant_id: UUID = Path(..., description="Tenant ID"),
start_date: Optional[datetime] = Query(None, description="Start date"),
end_date: Optional[datetime] = Query(None, description="End date"),
group_by: str = Query("daily", description="Aggregation period: daily, weekly, monthly"),
current_user: Dict[str, Any] = Depends(get_current_user_dep),
sales_service: SalesService = Depends(get_sales_service)
):
"""Get sales aggregation data using repository pattern"""
try:
logger.debug("Getting sales aggregation with repository pattern",
tenant_id=tenant_id,
group_by=group_by)
aggregation = await sales_service.get_sales_aggregation(
str(tenant_id), start_date, end_date, group_by
)
logger.debug("Aggregation generated successfully using repository",
tenant_id=tenant_id,
group_by=group_by)
return aggregation
except Exception as e:
logger.error("Failed to get sales aggregation",
error=str(e),
tenant_id=tenant_id)
raise HTTPException(status_code=500, detail=f"Failed to get aggregation: {str(e)}")
@router.post("/tenants/{tenant_id}/sales/import", response_model=SalesImportResult)
async def import_sales_data(
tenant_id: UUID = Path(..., description="Tenant ID"),
file: UploadFile = File(...),
file_format: str = Form(...),
current_user: Dict[str, Any] = Depends(get_current_user_dep),
db: AsyncSession = Depends(get_db)
import_service: EnhancedDataImportService = Depends(get_import_service)
):
"""Import sales data from file for tenant - FIXED VERSION"""
"""Import sales data using enhanced repository pattern"""
try:
logger.info("Importing sales data",
logger.info("Importing sales data with enhanced repository pattern",
tenant_id=tenant_id,
format=file_format,
filename=file.filename,
@@ -209,33 +231,32 @@ async def import_sales_data(
content = await file.read()
file_content = content.decode('utf-8')
# ✅ FIX: tenant_id comes from URL path, not file upload
result = await DataImportService.process_upload(
tenant_id,
# Process using enhanced import service
result = await import_service.process_import(
str(tenant_id),
file_content,
file_format,
db,
filename=file.filename
)
if result["success"]:
if result.success:
# Publish event
try:
await publish_data_imported({
"tenant_id": str(tenant_id), # Ensure string conversion
"tenant_id": str(tenant_id),
"type": "file_import",
"format": file_format,
"filename": file.filename,
"records_created": result["records_created"],
"records_created": result.records_created,
"imported_by": current_user["user_id"],
"timestamp": datetime.utcnow().isoformat()
})
except Exception as pub_error:
logger.warning("Failed to publish import event", error=str(pub_error))
logger.info("Import completed",
success=result["success"],
records_created=result.get("records_created", 0),
logger.info("Import completed with enhanced repository pattern",
success=result.success,
records_created=result.records_created,
tenant_id=tenant_id)
return result
@@ -245,6 +266,7 @@ async def import_sales_data(
tenant_id=tenant_id)
raise HTTPException(status_code=500, detail=f"Failed to import sales data: {str(e)}")
@router.post("/tenants/{tenant_id}/sales/import/validate", response_model=SalesValidationResult)
async def validate_import_data(
tenant_id: UUID = Path(..., description="Tenant ID"),
@@ -252,44 +274,36 @@ async def validate_import_data(
file_format: str = Form(default="csv", description="File format: csv, json, excel"),
validate_only: bool = Form(default=True, description="Only validate, don't import"),
source: str = Form(default="onboarding_upload", description="Source of the upload"),
current_user: Dict[str, Any] = Depends(get_current_user_dep)
current_user: Dict[str, Any] = Depends(get_current_user_dep),
import_service: EnhancedDataImportService = Depends(get_import_service)
):
"""
✅ FIXED: Validate import data using FormData (same as import endpoint)
Now both validation and import endpoints use the same FormData approach
"""
"""Validate import data using enhanced repository pattern"""
try:
logger.info("Validating import data",
logger.info("Validating import data with enhanced repository pattern",
tenant_id=tenant_id,
format=file_format,
filename=file.filename,
user_id=current_user["user_id"])
# ✅ STEP 1: Read file content (same as import endpoint)
# Read file content
content = await file.read()
file_content = content.decode('utf-8')
# ✅ STEP 2: Create validation data structure
# This matches the SalesDataImport schema but gets data from FormData
# Create validation data structure
validation_data = {
"tenant_id": str(tenant_id), # From URL path
"data": file_content, # From uploaded file
"data_format": file_format, # From form field
"source": source, # From form field
"validate_only": validate_only # From form field
"tenant_id": str(tenant_id),
"data": file_content,
"data_format": file_format,
"source": source,
"validate_only": validate_only
}
logger.debug("Validation data prepared",
tenant_id=tenant_id,
data_length=len(file_content),
format=file_format)
# Use enhanced validation service
validation_result = await import_service.validate_import_data(validation_data)
# ✅ STEP 3: Use existing validation service
validation_result = await DataImportService.validate_import_data(validation_data)
logger.info("Validation completed",
is_valid=validation_result.get("is_valid", False),
total_records=validation_result.get("total_records", 0),
logger.info("Validation completed with enhanced repository pattern",
is_valid=validation_result.is_valid,
total_records=validation_result.total_records,
tenant_id=tenant_id)
return validation_result
@@ -300,85 +314,49 @@ async def validate_import_data(
tenant_id=tenant_id)
raise HTTPException(status_code=500, detail=f"Failed to validate import data: {str(e)}")
@router.get("/tenants/{tenant_id}/sales/import/template/{format_type}")
async def get_import_template(
tenant_id: UUID = Path(..., description="Tenant ID"),
format_type: str = Path(..., description="Template format: csv, json, excel"),
current_user: Dict[str, Any] = Depends(get_current_user_dep)
):
"""Get import template for specified format"""
try:
logger.debug("Getting import template",
format=format_type,
tenant_id=tenant_id,
user_id=current_user["user_id"])
template = await DataImportService.get_import_template(format_type)
if "error" in template:
logger.warning("Template generation error", error=template["error"])
raise HTTPException(status_code=400, detail=template["error"])
logger.debug("Template generated successfully",
format=format_type,
tenant_id=tenant_id)
if format_type.lower() == "csv":
return Response(
content=template["template"],
media_type="text/csv",
headers={"Content-Disposition": f"attachment; filename={template['filename']}"}
)
elif format_type.lower() == "json":
return Response(
content=template["template"],
media_type="application/json",
headers={"Content-Disposition": f"attachment; filename={template['filename']}"}
)
elif format_type.lower() in ["excel", "xlsx"]:
return Response(
content=base64.b64decode(template["template"]),
media_type=template["content_type"],
headers={"Content-Disposition": f"attachment; filename={template['filename']}"}
)
else:
return template
except HTTPException:
raise
except Exception as e:
logger.error("Failed to generate import template",
error=str(e),
tenant_id=tenant_id)
raise HTTPException(status_code=500, detail=f"Failed to generate template: {str(e)}")
@router.get("/tenants/{tenant_id}/sales/analytics")
async def get_sales_analytics(
@router.post("/tenants/{tenant_id}/sales/import/validate-json", response_model=SalesValidationResult)
async def validate_import_data_json(
tenant_id: UUID = Path(..., description="Tenant ID"),
start_date: Optional[datetime] = Query(None, description="Start date"),
end_date: Optional[datetime] = Query(None, description="End date"),
request: SalesValidationRequest = ...,
current_user: Dict[str, Any] = Depends(get_current_user_dep),
db: AsyncSession = Depends(get_db)
import_service: EnhancedDataImportService = Depends(get_import_service)
):
"""Get sales analytics for tenant"""
"""Validate import data from JSON request for onboarding flow"""
try:
logger.debug("Getting sales analytics",
tenant_id=tenant_id,
start_date=start_date,
end_date=end_date)
logger.info("Starting JSON-based data validation",
tenant_id=str(tenant_id),
data_format=request.data_format,
data_length=len(request.data),
validate_only=request.validate_only)
analytics = await SalesService.get_sales_analytics(
tenant_id, start_date, end_date, db
)
# Create validation data structure
validation_data = {
"tenant_id": str(tenant_id),
"data": request.data, # Fixed: use 'data' not 'content'
"data_format": request.data_format,
"filename": f"onboarding_data.{request.data_format}",
"source": request.source,
"validate_only": request.validate_only
}
logger.debug("Analytics generated successfully", tenant_id=tenant_id)
return analytics
# Use enhanced validation service
validation_result = await import_service.validate_import_data(validation_data)
logger.info("JSON validation completed",
is_valid=validation_result.is_valid,
total_records=validation_result.total_records,
tenant_id=tenant_id)
return validation_result
except Exception as e:
logger.error("Failed to generate sales analytics",
logger.error("Failed to validate JSON import data",
error=str(e),
tenant_id=tenant_id)
raise HTTPException(status_code=500, detail=f"Failed to generate analytics: {str(e)}")
raise HTTPException(status_code=500, detail=f"Failed to validate import data: {str(e)}")
@router.post("/tenants/{tenant_id}/sales/export")
async def export_sales_data(
@@ -388,17 +366,17 @@ async def export_sales_data(
end_date: Optional[datetime] = Query(None, description="End date"),
products: Optional[List[str]] = Query(None, description="Filter by products"),
current_user: Dict[str, Any] = Depends(get_current_user_dep),
db: AsyncSession = Depends(get_db)
sales_service: SalesService = Depends(get_sales_service)
):
"""Export sales data in specified format for tenant"""
"""Export sales data using repository pattern"""
try:
logger.info("Exporting sales data",
logger.info("Exporting sales data with repository pattern",
tenant_id=tenant_id,
format=export_format,
user_id=current_user["user_id"])
export_result = await SalesService.export_sales_data(
tenant_id, export_format, start_date, end_date, products, db
export_result = await sales_service.export_sales_data(
str(tenant_id), export_format, start_date, end_date, products
)
if not export_result:
@@ -407,7 +385,7 @@ async def export_sales_data(
# Publish export event
try:
await publish_export_completed({
"tenant_id": tenant_id,
"tenant_id": str(tenant_id),
"format": export_format,
"exported_by": current_user["user_id"],
"record_count": export_result.get("record_count", 0),
@@ -416,7 +394,7 @@ async def export_sales_data(
except Exception as pub_error:
logger.warning("Failed to publish export event", error=str(pub_error))
logger.info("Export completed successfully",
logger.info("Export completed successfully using repository",
tenant_id=tenant_id,
format=export_format)
@@ -434,31 +412,27 @@ async def export_sales_data(
tenant_id=tenant_id)
raise HTTPException(status_code=500, detail=f"Failed to export sales data: {str(e)}")
@router.delete("/tenants/{tenant_id}/sales/{record_id}")
async def delete_sales_record(
tenant_id: UUID = Path(..., description="Tenant ID"),
record_id: str = Path(..., description="Sales record ID"),
current_user: Dict[str, Any] = Depends(get_current_user_dep),
db: AsyncSession = Depends(get_db)
sales_service: SalesService = Depends(get_sales_service)
):
"""Delete a sales record for tenant"""
"""Delete a sales record using repository pattern"""
try:
logger.info("Deleting sales record",
logger.info("Deleting sales record with repository pattern",
record_id=record_id,
tenant_id=tenant_id,
user_id=current_user["user_id"])
# Verify record belongs to tenant before deletion
record = await SalesService.get_sales_record(record_id, db)
if not record or record.tenant_id != tenant_id:
raise HTTPException(status_code=404, detail="Sales record not found")
success = await SalesService.delete_sales_record(record_id, db)
success = await sales_service.delete_sales_record(record_id, str(tenant_id))
if not success:
raise HTTPException(status_code=404, detail="Sales record not found")
logger.info("Sales record deleted successfully",
logger.info("Sales record deleted successfully using repository",
record_id=record_id,
tenant_id=tenant_id)
return {"status": "success", "message": "Sales record deleted successfully"}
@@ -471,43 +445,20 @@ async def delete_sales_record(
tenant_id=tenant_id)
raise HTTPException(status_code=500, detail=f"Failed to delete sales record: {str(e)}")
@router.get("/tenants/{tenant_id}/sales/summary")
async def get_sales_summary(
tenant_id: UUID = Path(..., description="Tenant ID"),
period: str = Query("daily", description="Summary period: daily, weekly, monthly"),
current_user: Dict[str, Any] = Depends(get_current_user_dep),
db: AsyncSession = Depends(get_db)
):
"""Get sales summary for specified period for tenant"""
try:
logger.debug("Getting sales summary",
tenant_id=tenant_id,
period=period)
summary = await SalesService.get_sales_summary(tenant_id, period, db)
logger.debug("Summary generated successfully", tenant_id=tenant_id)
return summary
except Exception as e:
logger.error("Failed to generate sales summary",
error=str(e),
tenant_id=tenant_id)
raise HTTPException(status_code=500, detail=f"Failed to generate summary: {str(e)}")
@router.get("/tenants/{tenant_id}/sales/products")
async def get_products_list(
tenant_id: UUID = Path(..., description="Tenant ID"),
current_user: Dict[str, Any] = Depends(get_current_user_dep),
db: AsyncSession = Depends(get_db)
sales_service: SalesService = Depends(get_sales_service)
):
"""Get list of all products with sales data for tenant"""
"""Get list of products using repository pattern"""
try:
logger.debug("Getting products list", tenant_id=tenant_id)
logger.debug("Getting products list with repository pattern", tenant_id=tenant_id)
products = await SalesService.get_products_list(tenant_id, db)
products = await sales_service.get_products_list(str(tenant_id))
logger.debug("Products list retrieved",
logger.debug("Products list retrieved using repository",
count=len(products),
tenant_id=tenant_id)
return products
@@ -518,76 +469,32 @@ async def get_products_list(
tenant_id=tenant_id)
raise HTTPException(status_code=500, detail=f"Failed to get products list: {str(e)}")
@router.get("/tenants/{tenant_id}/sales/{record_id}", response_model=SalesDataResponse)
async def get_sales_record(
tenant_id: UUID = Path(..., description="Tenant ID"),
record_id: str = Path(..., description="Sales record ID"),
current_user: Dict[str, Any] = Depends(get_current_user_dep),
db: AsyncSession = Depends(get_db)
):
"""Get a specific sales record for tenant"""
try:
logger.debug("Getting sales record",
record_id=record_id,
tenant_id=tenant_id)
record = await SalesService.get_sales_record(record_id, db)
if not record or record.tenant_id != tenant_id:
raise HTTPException(status_code=404, detail="Sales record not found")
logger.debug("Sales record retrieved",
record_id=record_id,
tenant_id=tenant_id)
return record
except HTTPException:
raise
except Exception as e:
logger.error("Failed to get sales record",
error=str(e),
tenant_id=tenant_id,
record_id=record_id)
raise HTTPException(status_code=500, detail=f"Failed to get sales record: {str(e)}")
@router.put("/tenants/{tenant_id}/sales/{record_id}", response_model=SalesDataResponse)
async def update_sales_record(
sales_data: SalesDataCreate,
record_id: str = Path(..., description="Sales record ID"),
@router.get("/tenants/{tenant_id}/sales/statistics")
async def get_sales_statistics(
tenant_id: UUID = Path(..., description="Tenant ID"),
current_user: Dict[str, Any] = Depends(get_current_user_dep),
db: AsyncSession = Depends(get_db)
sales_service: SalesService = Depends(get_sales_service)
):
"""Update a sales record for tenant"""
"""Get comprehensive sales statistics using repository pattern"""
try:
logger.info("Updating sales record",
record_id=record_id,
tenant_id=tenant_id,
user_id=current_user["user_id"])
logger.debug("Getting sales statistics with repository pattern", tenant_id=tenant_id)
# Verify record exists and belongs to tenant
existing_record = await SalesService.get_sales_record(record_id, db)
if not existing_record or existing_record.tenant_id != tenant_id:
raise HTTPException(status_code=404, detail="Sales record not found")
# Get analytics which includes comprehensive statistics
analytics = await sales_service.get_sales_analytics(str(tenant_id))
# Override tenant_id from URL path
sales_data.tenant_id = tenant_id
# Create enhanced statistics response
statistics = {
"tenant_id": str(tenant_id),
"analytics": analytics,
"generated_at": datetime.utcnow().isoformat()
}
updated_record = await SalesService.update_sales_record(record_id, sales_data, db)
logger.debug("Sales statistics retrieved using repository", tenant_id=tenant_id)
return statistics
if not updated_record:
raise HTTPException(status_code=404, detail="Sales record not found")
logger.info("Sales record updated successfully",
record_id=record_id,
tenant_id=tenant_id)
return updated_record
except HTTPException:
raise
except Exception as e:
logger.error("Failed to update sales record",
logger.error("Failed to get sales statistics",
error=str(e),
tenant_id=tenant_id,
record_id=record_id)
raise HTTPException(status_code=500, detail=f"Failed to update sales record: {str(e)}")
tenant_id=tenant_id)
raise HTTPException(status_code=500, detail=f"Failed to get statistics: {str(e)}")

212
services/data/app/core/database.py
View File

@@ -1,46 +1,196 @@
"""Database configuration for data service"""
"""
Database configuration for data service
Uses shared database infrastructure for consistency
"""
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession, async_sessionmaker
from sqlalchemy.orm import declarative_base
import structlog
from typing import AsyncGenerator
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import text
from shared.database.base import DatabaseManager, Base
from app.core.config import settings
logger = structlog.get_logger()
# Create async engine
engine = create_async_engine(
settings.DATABASE_URL,
echo=False,
pool_pre_ping=True,
pool_size=10,
max_overflow=20
# Initialize database manager using shared infrastructure
database_manager = DatabaseManager(
database_url=settings.DATABASE_URL,
service_name="data",
pool_size=15,
max_overflow=25,
echo=settings.DEBUG if hasattr(settings, 'DEBUG') else False
)
# Create async session factory
AsyncSessionLocal = async_sessionmaker(
engine,
class_=AsyncSession,
expire_on_commit=False
)
# Alias for convenience - matches the existing interface
get_db = database_manager.get_db
# Base class for models
Base = declarative_base()
# Use the shared background session method
get_background_db_session = database_manager.get_background_session
async def get_db() -> AsyncSession:
"""Get database session"""
async with AsyncSessionLocal() as session:
try:
yield session
finally:
await session.close()
async def get_db_health() -> bool:
"""Health check function for database connectivity"""
try:
async with database_manager.async_engine.begin() as conn:
await conn.execute(text("SELECT 1"))
logger.debug("Database health check passed")
return True
except Exception as e:
logger.error("Database health check failed", error=str(e))
return False
async def init_db():
"""Initialize database tables"""
"""Initialize database tables using shared infrastructure"""
try:
async with engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
logger.info("Database initialized successfully")
logger.info("Initializing data service database")
# Import models to ensure they're registered
from app.models.sales import SalesData
from app.models.traffic import TrafficData
from app.models.weather import WeatherData
# Create tables using shared infrastructure
await database_manager.create_tables()
logger.info("Data service database initialized successfully")
except Exception as e:
logger.error("Failed to initialize database", error=str(e))
raise
logger.error("Failed to initialize data service database", error=str(e))
raise
# Data service specific database utilities
class DataDatabaseUtils:
"""Data service specific database utilities"""
@staticmethod
async def cleanup_old_sales_data(days_old: int = 730):
"""Clean up old sales data (default 2 years)"""
try:
async with database_manager.get_background_session() as session:
if settings.DATABASE_URL.startswith("sqlite"):
query = text(
"DELETE FROM sales_data "
"WHERE created_at < datetime('now', :days_param)"
)
params = {"days_param": f"-{days_old} days"}
else:
query = text(
"DELETE FROM sales_data "
"WHERE created_at < NOW() - INTERVAL :days_param"
)
params = {"days_param": f"{days_old} days"}
result = await session.execute(query, params)
deleted_count = result.rowcount
logger.info("Cleaned up old sales data",
deleted_count=deleted_count,
days_old=days_old)
return deleted_count
except Exception as e:
logger.error("Failed to cleanup old sales data", error=str(e))
return 0
@staticmethod
async def get_data_statistics(tenant_id: str = None) -> dict:
"""Get data service statistics"""
try:
async with database_manager.get_background_session() as session:
# Get sales data statistics
if tenant_id:
sales_query = text(
"SELECT COUNT(*) as count "
"FROM sales_data "
"WHERE tenant_id = :tenant_id"
)
params = {"tenant_id": tenant_id}
else:
sales_query = text("SELECT COUNT(*) as count FROM sales_data")
params = {}
sales_result = await session.execute(sales_query, params)
sales_count = sales_result.scalar() or 0
# Get traffic data statistics (if exists)
try:
traffic_query = text("SELECT COUNT(*) as count FROM traffic_data")
if tenant_id:
# Traffic data might not have tenant_id, check table structure
pass
traffic_result = await session.execute(traffic_query)
traffic_count = traffic_result.scalar() or 0
except:
traffic_count = 0
# Get weather data statistics (if exists)
try:
weather_query = text("SELECT COUNT(*) as count FROM weather_data")
weather_result = await session.execute(weather_query)
weather_count = weather_result.scalar() or 0
except:
weather_count = 0
return {
"tenant_id": tenant_id,
"sales_records": sales_count,
"traffic_records": traffic_count,
"weather_records": weather_count,
"total_records": sales_count + traffic_count + weather_count
}
except Exception as e:
logger.error("Failed to get data statistics", error=str(e))
return {
"tenant_id": tenant_id,
"sales_records": 0,
"traffic_records": 0,
"weather_records": 0,
"total_records": 0
}
# Enhanced database session dependency with better error handling
async def get_db_session() -> AsyncGenerator[AsyncSession, None]:
"""Enhanced database session dependency with better logging and error handling"""
async with database_manager.async_session_local() as session:
try:
logger.debug("Database session created")
yield session
except Exception as e:
logger.error("Database session error", error=str(e), exc_info=True)
await session.rollback()
raise
finally:
await session.close()
logger.debug("Database session closed")
# Database cleanup for data service
async def cleanup_data_database():
"""Cleanup database connections for data service"""
try:
logger.info("Cleaning up data service database connections")
# Close engine connections
if hasattr(database_manager, 'async_engine') and database_manager.async_engine:
await database_manager.async_engine.dispose()
logger.info("Data service database cleanup completed")
except Exception as e:
logger.error("Failed to cleanup data service database", error=str(e))
# Export the commonly used items to maintain compatibility
__all__ = [
'Base',
'database_manager',
'get_db',
'get_background_db_session',
'get_db_session',
'get_db_health',
'DataDatabaseUtils',
'init_db',
'cleanup_data_database'
]

3
services/data/app/main.py
View File

@@ -73,8 +73,9 @@ async def lifespan(app: FastAPI):
async def check_database():
try:
from app.core.database import get_db
from sqlalchemy import text
async for db in get_db():
await db.execute("SELECT 1")
await db.execute(text("SELECT 1"))
return True
except Exception as e:
return f"Database error: {e}"

7
services/data/app/models/traffic.py
View File

@@ -6,9 +6,9 @@
from sqlalchemy import Column, String, DateTime, Float, Integer, Text, Index
from sqlalchemy.dialects.postgresql import UUID
import uuid
from datetime import datetime
from datetime import datetime, timezone
from app.core.database import Base
from shared.database.base import Base
class TrafficData(Base):
__tablename__ = "traffic_data"
@@ -22,7 +22,8 @@ class TrafficData(Base):
average_speed = Column(Float, nullable=True) # km/h
source = Column(String(50), nullable=False, default="madrid_opendata")
raw_data = Column(Text, nullable=True)
created_at = Column(DateTime(timezone=True), default=datetime.utcnow)
created_at = Column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc))
updated_at = Column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), onupdate=lambda: datetime.now(timezone.utc))
__table_args__ = (
Index('idx_traffic_location_date', 'location_id', 'date'),

11
services/data/app/models/weather.py
View File

@@ -6,9 +6,9 @@
from sqlalchemy import Column, String, DateTime, Float, Integer, Text, Index
from sqlalchemy.dialects.postgresql import UUID
import uuid
from datetime import datetime
from datetime import datetime, timezone
from app.core.database import Base
from shared.database.base import Base
class WeatherData(Base):
__tablename__ = "weather_data"
@@ -24,7 +24,8 @@ class WeatherData(Base):
description = Column(String(200), nullable=True)
source = Column(String(50), nullable=False, default="aemet")
raw_data = Column(Text, nullable=True)
created_at = Column(DateTime(timezone=True), default=datetime.utcnow)
created_at = Column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc))
updated_at = Column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), onupdate=lambda: datetime.now(timezone.utc))
__table_args__ = (
Index('idx_weather_location_date', 'location_id', 'date'),
@@ -36,7 +37,7 @@ class WeatherForecast(Base):
id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
location_id = Column(String(100), nullable=False, index=True)
forecast_date = Column(DateTime(timezone=True), nullable=False)
generated_at = Column(DateTime(timezone=True), nullable=False, default=datetime.utcnow)
generated_at = Column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc))
temperature = Column(Float, nullable=True)
precipitation = Column(Float, nullable=True)
humidity = Column(Float, nullable=True)
@@ -44,6 +45,8 @@ class WeatherForecast(Base):
description = Column(String(200), nullable=True)
source = Column(String(50), nullable=False, default="aemet")
raw_data = Column(Text, nullable=True)
created_at = Column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc))
updated_at = Column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), onupdate=lambda: datetime.now(timezone.utc))
__table_args__ = (
Index('idx_forecast_location_date', 'location_id', 'forecast_date'),

12
services/data/app/repositories/__init__.py Normal file
View File

@@ -0,0 +1,12 @@
"""
Data Service Repositories
Repository implementations for data service
"""
from .base import DataBaseRepository
from .sales_repository import SalesRepository
__all__ = [
"DataBaseRepository",
"SalesRepository"
]

167
services/data/app/repositories/base.py Normal file
View File

@@ -0,0 +1,167 @@
"""
Base Repository for Data Service
Service-specific repository base class with data service utilities
"""
from typing import Optional, List, Dict, Any, Type, TypeVar, Generic
from sqlalchemy.ext.asyncio import AsyncSession
from datetime import datetime, timezone
import structlog
from shared.database.repository import BaseRepository
from shared.database.exceptions import DatabaseError, ValidationError
logger = structlog.get_logger()
# Type variables for the data service repository
Model = TypeVar('Model')
CreateSchema = TypeVar('CreateSchema')
UpdateSchema = TypeVar('UpdateSchema')
class DataBaseRepository(BaseRepository[Model, CreateSchema, UpdateSchema], Generic[Model, CreateSchema, UpdateSchema]):
"""Base repository for data service with common data operations"""
def __init__(self, model: Type, session: AsyncSession, cache_ttl: Optional[int] = 300):
super().__init__(model, session, cache_ttl)
async def get_by_tenant_id(
self,
tenant_id: str,
skip: int = 0,
limit: int = 100
) -> List:
"""Get records filtered by tenant_id"""
return await self.get_multi(
skip=skip,
limit=limit,
filters={"tenant_id": tenant_id}
)
async def get_by_date_range(
self,
tenant_id: str,
start_date: Optional[datetime] = None,
end_date: Optional[datetime] = None,
skip: int = 0,
limit: int = 100
) -> List:
"""Get records filtered by tenant and date range"""
try:
filters = {"tenant_id": tenant_id}
# Build date range filter
if start_date or end_date:
if not hasattr(self.model, 'date'):
raise ValidationError("Model does not have 'date' field for date filtering")
# This would need a more complex implementation for date ranges
# For now, we'll use the basic filter
if start_date and end_date:
# Would need custom query building for date ranges
pass
return await self.get_multi(
skip=skip,
limit=limit,
filters=filters,
order_by="date",
order_desc=True
)
except Exception as e:
logger.error(f"Failed to get records by date range",
tenant_id=tenant_id,
start_date=start_date,
end_date=end_date,
error=str(e))
raise DatabaseError(f"Date range query failed: {str(e)}")
async def count_by_tenant(self, tenant_id: str) -> int:
"""Count records for a specific tenant"""
return await self.count(filters={"tenant_id": tenant_id})
async def validate_tenant_access(self, tenant_id: str, record_id: Any) -> bool:
"""Validate that a record belongs to the specified tenant"""
try:
record = await self.get_by_id(record_id)
if not record:
return False
# Check if record has tenant_id field and matches
if hasattr(record, 'tenant_id'):
return str(record.tenant_id) == str(tenant_id)
return True # If no tenant_id field, allow access
except Exception as e:
logger.error("Failed to validate tenant access",
tenant_id=tenant_id,
record_id=record_id,
error=str(e))
return False
async def get_tenant_stats(self, tenant_id: str) -> Dict[str, Any]:
"""Get statistics for a specific tenant"""
try:
total_records = await self.count_by_tenant(tenant_id)
# Get recent activity (if model has created_at)
recent_records = 0
if hasattr(self.model, 'created_at'):
# This would need custom query for date filtering
# For now, return basic stats
pass
return {
"tenant_id": tenant_id,
"total_records": total_records,
"recent_records": recent_records,
"model_type": self.model.__name__
}
except Exception as e:
logger.error("Failed to get tenant statistics",
tenant_id=tenant_id, error=str(e))
return {
"tenant_id": tenant_id,
"total_records": 0,
"recent_records": 0,
"model_type": self.model.__name__,
"error": str(e)
}
async def cleanup_old_records(
self,
tenant_id: str,
days_old: int = 365,
batch_size: int = 1000
) -> int:
"""Clean up old records for a tenant (if model has date/created_at field)"""
try:
if not hasattr(self.model, 'created_at') and not hasattr(self.model, 'date'):
logger.warning(f"Model {self.model.__name__} has no date field for cleanup")
return 0
# This would need custom implementation with raw SQL
# For now, return 0 to indicate no cleanup performed
logger.info(f"Cleanup requested for {self.model.__name__} but not implemented")
return 0
except Exception as e:
logger.error("Failed to cleanup old records",
tenant_id=tenant_id,
days_old=days_old,
error=str(e))
raise DatabaseError(f"Cleanup failed: {str(e)}")
def _ensure_utc_datetime(self, dt: Optional[datetime]) -> Optional[datetime]:
"""Ensure datetime is UTC timezone aware"""
if dt is None:
return None
if dt.tzinfo is None:
# Assume naive datetime is UTC
return dt.replace(tzinfo=timezone.utc)
return dt.astimezone(timezone.utc)

517
services/data/app/repositories/sales_repository.py Normal file
View File

@@ -0,0 +1,517 @@
"""
Sales Repository
Repository for sales data operations with business-specific queries
"""
from typing import Optional, List, Dict, Any, Type
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import select, and_, or_, func, desc, asc, text
from datetime import datetime, timezone
import structlog
from .base import DataBaseRepository
from app.models.sales import SalesData
from app.schemas.sales import SalesDataCreate, SalesDataResponse
from shared.database.exceptions import DatabaseError, ValidationError
logger = structlog.get_logger()
class SalesRepository(DataBaseRepository[SalesData, SalesDataCreate, Dict]):
"""Repository for sales data operations"""
def __init__(self, model_class: Type, session: AsyncSession, cache_ttl: Optional[int] = 300):
super().__init__(model_class, session, cache_ttl)
async def get_by_tenant_and_date_range(
self,
tenant_id: str,
start_date: Optional[datetime] = None,
end_date: Optional[datetime] = None,
product_names: Optional[List[str]] = None,
location_ids: Optional[List[str]] = None,
skip: int = 0,
limit: int = 100
) -> List[SalesData]:
"""Get sales data filtered by tenant, date range, and optional filters"""
try:
query = select(self.model).where(self.model.tenant_id == tenant_id)
# Add date range filter
if start_date:
start_date = self._ensure_utc_datetime(start_date)
query = query.where(self.model.date >= start_date)
if end_date:
end_date = self._ensure_utc_datetime(end_date)
query = query.where(self.model.date <= end_date)
# Add product filter
if product_names:
query = query.where(self.model.product_name.in_(product_names))
# Add location filter
if location_ids:
query = query.where(self.model.location_id.in_(location_ids))
# Order by date descending (most recent first)
query = query.order_by(desc(self.model.date))
# Apply pagination
query = query.offset(skip).limit(limit)
result = await self.session.execute(query)
return result.scalars().all()
except Exception as e:
logger.error("Failed to get sales by tenant and date range",
tenant_id=tenant_id,
start_date=start_date,
end_date=end_date,
error=str(e))
raise DatabaseError(f"Failed to get sales data: {str(e)}")
async def get_sales_aggregation(
self,
tenant_id: str,
start_date: Optional[datetime] = None,
end_date: Optional[datetime] = None,
group_by: str = "daily",
product_name: Optional[str] = None
) -> List[Dict[str, Any]]:
"""Get aggregated sales data for analytics"""
try:
# Determine date truncation based on group_by
if group_by == "daily":
date_trunc = "day"
elif group_by == "weekly":
date_trunc = "week"
elif group_by == "monthly":
date_trunc = "month"
else:
raise ValidationError(f"Invalid group_by value: {group_by}")
# Build base query
if self.session.bind.dialect.name == 'postgresql':
query = text("""
SELECT
DATE_TRUNC(:date_trunc, date) as period,
product_name,
COUNT(*) as record_count,
SUM(quantity_sold) as total_quantity,
SUM(revenue) as total_revenue,
AVG(quantity_sold) as average_quantity,
AVG(revenue) as average_revenue
FROM sales_data
WHERE tenant_id = :tenant_id
""")
else:
# SQLite fallback
query = text("""
SELECT
DATE(date) as period,
product_name,
COUNT(*) as record_count,
SUM(quantity_sold) as total_quantity,
SUM(revenue) as total_revenue,
AVG(quantity_sold) as average_quantity,
AVG(revenue) as average_revenue
FROM sales_data
WHERE tenant_id = :tenant_id
""")
params = {
"tenant_id": tenant_id,
"date_trunc": date_trunc
}
# Add date filters
if start_date:
query = text(str(query) + " AND date >= :start_date")
params["start_date"] = self._ensure_utc_datetime(start_date)
if end_date:
query = text(str(query) + " AND date <= :end_date")
params["end_date"] = self._ensure_utc_datetime(end_date)
# Add product filter
if product_name:
query = text(str(query) + " AND product_name = :product_name")
params["product_name"] = product_name
# Add GROUP BY and ORDER BY
query = text(str(query) + " GROUP BY period, product_name ORDER BY period DESC")
result = await self.session.execute(query, params)
rows = result.fetchall()
# Convert to list of dictionaries
aggregations = []
for row in rows:
aggregations.append({
"period": group_by,
"date": row.period,
"product_name": row.product_name,
"record_count": row.record_count,
"total_quantity": row.total_quantity,
"total_revenue": float(row.total_revenue),
"average_quantity": float(row.average_quantity),
"average_revenue": float(row.average_revenue)
})
return aggregations
except Exception as e:
logger.error("Failed to get sales aggregation",
tenant_id=tenant_id,
group_by=group_by,
error=str(e))
raise DatabaseError(f"Sales aggregation failed: {str(e)}")
async def get_top_products(
self,
tenant_id: str,
start_date: Optional[datetime] = None,
end_date: Optional[datetime] = None,
limit: int = 10,
by_metric: str = "revenue"
) -> List[Dict[str, Any]]:
"""Get top products by quantity or revenue"""
try:
if by_metric not in ["revenue", "quantity"]:
raise ValidationError(f"Invalid metric: {by_metric}")
# Choose the aggregation column
metric_column = "revenue" if by_metric == "revenue" else "quantity_sold"
query = text(f"""
SELECT
product_name,
COUNT(*) as sale_count,
SUM(quantity_sold) as total_quantity,
SUM(revenue) as total_revenue,
AVG(revenue) as avg_revenue_per_sale
FROM sales_data
WHERE tenant_id = :tenant_id
{('AND date >= :start_date' if start_date else '')}
{('AND date <= :end_date' if end_date else '')}
GROUP BY product_name
ORDER BY SUM({metric_column}) DESC
LIMIT :limit
""")
params = {"tenant_id": tenant_id, "limit": limit}
if start_date:
params["start_date"] = self._ensure_utc_datetime(start_date)
if end_date:
params["end_date"] = self._ensure_utc_datetime(end_date)
result = await self.session.execute(query, params)
rows = result.fetchall()
products = []
for row in rows:
products.append({
"product_name": row.product_name,
"sale_count": row.sale_count,
"total_quantity": row.total_quantity,
"total_revenue": float(row.total_revenue),
"avg_revenue_per_sale": float(row.avg_revenue_per_sale),
"metric_used": by_metric
})
return products
except Exception as e:
logger.error("Failed to get top products",
tenant_id=tenant_id,
by_metric=by_metric,
error=str(e))
raise DatabaseError(f"Top products query failed: {str(e)}")
async def get_sales_by_location(
self,
tenant_id: str,
start_date: Optional[datetime] = None,
end_date: Optional[datetime] = None
) -> List[Dict[str, Any]]:
"""Get sales statistics by location"""
try:
query = text("""
SELECT
COALESCE(location_id, 'unknown') as location_id,
COUNT(*) as sale_count,
SUM(quantity_sold) as total_quantity,
SUM(revenue) as total_revenue,
AVG(revenue) as avg_revenue_per_sale
FROM sales_data
WHERE tenant_id = :tenant_id
{date_filters}
GROUP BY location_id
ORDER BY SUM(revenue) DESC
""".format(
date_filters=(
"AND date >= :start_date" if start_date else ""
) + (
" AND date <= :end_date" if end_date else ""
)
))
params = {"tenant_id": tenant_id}
if start_date:
params["start_date"] = self._ensure_utc_datetime(start_date)
if end_date:
params["end_date"] = self._ensure_utc_datetime(end_date)
result = await self.session.execute(query, params)
rows = result.fetchall()
locations = []
for row in rows:
locations.append({
"location_id": row.location_id,
"sale_count": row.sale_count,
"total_quantity": row.total_quantity,
"total_revenue": float(row.total_revenue),
"avg_revenue_per_sale": float(row.avg_revenue_per_sale)
})
return locations
except Exception as e:
logger.error("Failed to get sales by location",
tenant_id=tenant_id,
error=str(e))
raise DatabaseError(f"Sales by location query failed: {str(e)}")
async def create_bulk_sales(
self,
sales_records: List[Dict[str, Any]],
tenant_id: str
) -> List[SalesData]:
"""Create multiple sales records in bulk"""
try:
# Ensure all records have tenant_id
for record in sales_records:
record["tenant_id"] = tenant_id
# Ensure dates are timezone-aware
if "date" in record and record["date"]:
record["date"] = self._ensure_utc_datetime(record["date"])
return await self.bulk_create(sales_records)
except Exception as e:
logger.error("Failed to create bulk sales",
tenant_id=tenant_id,
record_count=len(sales_records),
error=str(e))
raise DatabaseError(f"Bulk sales creation failed: {str(e)}")
async def search_sales(
self,
tenant_id: str,
search_term: str,
skip: int = 0,
limit: int = 100
) -> List[SalesData]:
"""Search sales by product name or notes"""
try:
# Use the parent search method with sales-specific fields
search_fields = ["product_name", "notes", "location_id"]
# Filter by tenant first
query = select(self.model).where(
and_(
self.model.tenant_id == tenant_id,
or_(
self.model.product_name.ilike(f"%{search_term}%"),
self.model.notes.ilike(f"%{search_term}%") if hasattr(self.model, 'notes') else False,
self.model.location_id.ilike(f"%{search_term}%") if hasattr(self.model, 'location_id') else False
)
)
).order_by(desc(self.model.date)).offset(skip).limit(limit)
result = await self.session.execute(query)
return result.scalars().all()
except Exception as e:
logger.error("Failed to search sales",
tenant_id=tenant_id,
search_term=search_term,
error=str(e))
raise DatabaseError(f"Sales search failed: {str(e)}")
async def get_sales_summary(
self,
tenant_id: str,
start_date: Optional[datetime] = None,
end_date: Optional[datetime] = None
) -> Dict[str, Any]:
"""Get comprehensive sales summary for a tenant"""
try:
base_filters = {"tenant_id": tenant_id}
# Build date filter for count
date_query = select(func.count(self.model.id)).where(self.model.tenant_id == tenant_id)
if start_date:
date_query = date_query.where(self.model.date >= self._ensure_utc_datetime(start_date))
if end_date:
date_query = date_query.where(self.model.date <= self._ensure_utc_datetime(end_date))
# Get basic counts
total_result = await self.session.execute(date_query)
total_sales = total_result.scalar() or 0
# Get revenue and quantity totals
summary_query = text("""
SELECT
COUNT(*) as total_records,
SUM(quantity_sold) as total_quantity,
SUM(revenue) as total_revenue,
AVG(revenue) as avg_revenue,
MIN(date) as earliest_sale,
MAX(date) as latest_sale,
COUNT(DISTINCT product_name) as unique_products,
COUNT(DISTINCT location_id) as unique_locations
FROM sales_data
WHERE tenant_id = :tenant_id
{date_filters}
""".format(
date_filters=(
"AND date >= :start_date" if start_date else ""
) + (
" AND date <= :end_date" if end_date else ""
)
))
params = {"tenant_id": tenant_id}
if start_date:
params["start_date"] = self._ensure_utc_datetime(start_date)
if end_date:
params["end_date"] = self._ensure_utc_datetime(end_date)
result = await self.session.execute(summary_query, params)
row = result.fetchone()
if row:
return {
"tenant_id": tenant_id,
"period_start": start_date,
"period_end": end_date,
"total_sales": row.total_records or 0,
"total_quantity": row.total_quantity or 0,
"total_revenue": float(row.total_revenue or 0),
"average_revenue": float(row.avg_revenue or 0),
"earliest_sale": row.earliest_sale,
"latest_sale": row.latest_sale,
"unique_products": row.unique_products or 0,
"unique_locations": row.unique_locations or 0
}
else:
return {
"tenant_id": tenant_id,
"period_start": start_date,
"period_end": end_date,
"total_sales": 0,
"total_quantity": 0,
"total_revenue": 0.0,
"average_revenue": 0.0,
"earliest_sale": None,
"latest_sale": None,
"unique_products": 0,
"unique_locations": 0
}
except Exception as e:
logger.error("Failed to get sales summary",
tenant_id=tenant_id,
error=str(e))
raise DatabaseError(f"Sales summary failed: {str(e)}")
async def validate_sales_data(self, sales_data: Dict[str, Any]) -> Dict[str, Any]:
"""Validate sales data before insertion"""
errors = []
warnings = []
try:
# Check required fields
required_fields = ["date", "product_name", "quantity_sold", "revenue"]
for field in required_fields:
if field not in sales_data or sales_data[field] is None:
errors.append(f"Missing required field: {field}")
# Validate data types and ranges
if "quantity_sold" in sales_data:
if not isinstance(sales_data["quantity_sold"], (int, float)) or sales_data["quantity_sold"] <= 0:
errors.append("quantity_sold must be a positive number")
if "revenue" in sales_data:
if not isinstance(sales_data["revenue"], (int, float)) or sales_data["revenue"] <= 0:
errors.append("revenue must be a positive number")
# Validate string lengths
if "product_name" in sales_data and len(str(sales_data["product_name"])) > 255:
errors.append("product_name exceeds maximum length of 255 characters")
# Check for suspicious data
if "quantity_sold" in sales_data and "revenue" in sales_data:
unit_price = sales_data["revenue"] / sales_data["quantity_sold"]
if unit_price > 10000: # Arbitrary high price threshold
warnings.append(f"Unusually high unit price: {unit_price:.2f}")
elif unit_price < 0.01: # Very low price
warnings.append(f"Unusually low unit price: {unit_price:.2f}")
return {
"is_valid": len(errors) == 0,
"errors": errors,
"warnings": warnings
}
except Exception as e:
logger.error("Failed to validate sales data", error=str(e))
return {
"is_valid": False,
"errors": [f"Validation error: {str(e)}"],
"warnings": []
}
async def get_product_statistics(self, tenant_id: str) -> List[Dict[str, Any]]:
"""Get product statistics for tenant"""
try:
query = text("""
SELECT
product_name,
COUNT(*) as total_sales,
SUM(quantity_sold) as total_quantity,
SUM(revenue) as total_revenue,
AVG(revenue) as avg_revenue,
MIN(date) as first_sale,
MAX(date) as last_sale
FROM sales_data
WHERE tenant_id = :tenant_id
GROUP BY product_name
ORDER BY SUM(revenue) DESC
""")
result = await self.session.execute(query, {"tenant_id": tenant_id})
rows = result.fetchall()
products = []
for row in rows:
products.append({
"product_name": row.product_name,
"total_sales": int(row.total_sales or 0),
"total_quantity": int(row.total_quantity or 0),
"total_revenue": float(row.total_revenue or 0),
"avg_revenue": float(row.avg_revenue or 0),
"first_sale": row.first_sale.isoformat() if row.first_sale else None,
"last_sale": row.last_sale.isoformat() if row.last_sale else None
})
logger.debug(f"Found {len(products)} products for tenant {tenant_id}")
return products
except Exception as e:
logger.error(f"Error getting product statistics: {str(e)}", tenant_id=tenant_id)
return []

10
services/data/app/schemas/sales.py
View File

@@ -156,5 +156,15 @@ class SalesExportRequest(BaseModel):
location_ids: Optional[List[str]] = None
include_metadata: bool = Field(default=True)
class Config:
from_attributes = True
class SalesValidationRequest(BaseModel):
"""Schema for JSON-based sales data validation request"""
data: str = Field(..., description="Raw data content (CSV, JSON, etc.)")
data_format: str = Field(..., pattern="^(csv|json|excel)$", description="Format of the data")
validate_only: bool = Field(default=True, description="Only validate, don't import")
source: str = Field(default="onboarding_upload", description="Source of the data")
class Config:
from_attributes = True

71
services/data/app/schemas/traffic.py Normal file
View File

@@ -0,0 +1,71 @@
# ================================================================
# services/data/app/schemas/traffic.py
# ================================================================
"""Traffic data schemas"""
from pydantic import BaseModel, Field, validator
from datetime import datetime
from typing import Optional, List
from uuid import UUID
class TrafficDataBase(BaseModel):
"""Base traffic data schema"""
location_id: str = Field(..., max_length=100, description="Traffic monitoring location ID")
date: datetime = Field(..., description="Date and time of traffic measurement")
traffic_volume: Optional[int] = Field(None, ge=0, description="Vehicles per hour")
pedestrian_count: Optional[int] = Field(None, ge=0, description="Pedestrians per hour")
congestion_level: Optional[str] = Field(None, regex="^(low|medium|high)$", description="Traffic congestion level")
average_speed: Optional[float] = Field(None, ge=0, le=200, description="Average speed in km/h")
source: str = Field("madrid_opendata", max_length=50, description="Data source")
raw_data: Optional[str] = Field(None, description="Raw data from source")
class TrafficDataCreate(TrafficDataBase):
"""Schema for creating traffic data"""
pass
class TrafficDataUpdate(BaseModel):
"""Schema for updating traffic data"""
traffic_volume: Optional[int] = Field(None, ge=0)
pedestrian_count: Optional[int] = Field(None, ge=0)
congestion_level: Optional[str] = Field(None, regex="^(low|medium|high)$")
average_speed: Optional[float] = Field(None, ge=0, le=200)
raw_data: Optional[str] = None
class TrafficDataResponse(TrafficDataBase):
"""Schema for traffic data responses"""
id: str = Field(..., description="Unique identifier")
created_at: datetime = Field(..., description="Creation timestamp")
updated_at: datetime = Field(..., description="Last update timestamp")
@validator('id', pre=True)
def convert_uuid_to_string(cls, v):
if isinstance(v, UUID):
return str(v)
return v
class Config:
from_attributes = True
json_encoders = {
datetime: lambda v: v.isoformat()
}
class TrafficDataList(BaseModel):
"""Schema for paginated traffic data responses"""
data: List[TrafficDataResponse]
total: int = Field(..., description="Total number of records")
page: int = Field(..., description="Current page number")
per_page: int = Field(..., description="Records per page")
has_next: bool = Field(..., description="Whether there are more pages")
has_prev: bool = Field(..., description="Whether there are previous pages")
class TrafficAnalytics(BaseModel):
"""Schema for traffic analytics"""
location_id: str
period_start: datetime
period_end: datetime
avg_traffic_volume: Optional[float] = None
avg_pedestrian_count: Optional[float] = None
peak_traffic_hour: Optional[int] = None
peak_pedestrian_hour: Optional[int] = None
congestion_distribution: dict = Field(default_factory=dict)
avg_speed: Optional[float] = None

121
services/data/app/schemas/weather.py Normal file
View File

@@ -0,0 +1,121 @@
# ================================================================
# services/data/app/schemas/weather.py
# ================================================================
"""Weather data schemas"""
from pydantic import BaseModel, Field, validator
from datetime import datetime
from typing import Optional, List
from uuid import UUID
class WeatherDataBase(BaseModel):
"""Base weather data schema"""
location_id: str = Field(..., max_length=100, description="Weather monitoring location ID")
date: datetime = Field(..., description="Date and time of weather measurement")
temperature: Optional[float] = Field(None, ge=-50, le=60, description="Temperature in Celsius")
precipitation: Optional[float] = Field(None, ge=0, description="Precipitation in mm")
humidity: Optional[float] = Field(None, ge=0, le=100, description="Humidity percentage")
wind_speed: Optional[float] = Field(None, ge=0, le=200, description="Wind speed in km/h")
pressure: Optional[float] = Field(None, ge=800, le=1200, description="Atmospheric pressure in hPa")
description: Optional[str] = Field(None, max_length=200, description="Weather description")
source: str = Field("aemet", max_length=50, description="Data source")
raw_data: Optional[str] = Field(None, description="Raw data from source")
class WeatherDataCreate(WeatherDataBase):
"""Schema for creating weather data"""
pass
class WeatherDataUpdate(BaseModel):
"""Schema for updating weather data"""
temperature: Optional[float] = Field(None, ge=-50, le=60)
precipitation: Optional[float] = Field(None, ge=0)
humidity: Optional[float] = Field(None, ge=0, le=100)
wind_speed: Optional[float] = Field(None, ge=0, le=200)
pressure: Optional[float] = Field(None, ge=800, le=1200)
description: Optional[str] = Field(None, max_length=200)
raw_data: Optional[str] = None
class WeatherDataResponse(WeatherDataBase):
"""Schema for weather data responses"""
id: str = Field(..., description="Unique identifier")
created_at: datetime = Field(..., description="Creation timestamp")
updated_at: datetime = Field(..., description="Last update timestamp")
@validator('id', pre=True)
def convert_uuid_to_string(cls, v):
if isinstance(v, UUID):
return str(v)
return v
class Config:
from_attributes = True
json_encoders = {
datetime: lambda v: v.isoformat()
}
class WeatherForecastBase(BaseModel):
"""Base weather forecast schema"""
location_id: str = Field(..., max_length=100, description="Location ID")
forecast_date: datetime = Field(..., description="Date for forecast")
temperature: Optional[float] = Field(None, ge=-50, le=60, description="Forecasted temperature")
precipitation: Optional[float] = Field(None, ge=0, description="Forecasted precipitation")
humidity: Optional[float] = Field(None, ge=0, le=100, description="Forecasted humidity")
wind_speed: Optional[float] = Field(None, ge=0, le=200, description="Forecasted wind speed")
description: Optional[str] = Field(None, max_length=200, description="Forecast description")
source: str = Field("aemet", max_length=50, description="Data source")
raw_data: Optional[str] = Field(None, description="Raw forecast data")
class WeatherForecastCreate(WeatherForecastBase):
"""Schema for creating weather forecasts"""
pass
class WeatherForecastResponse(WeatherForecastBase):
"""Schema for weather forecast responses"""
id: str = Field(..., description="Unique identifier")
generated_at: datetime = Field(..., description="When forecast was generated")
created_at: datetime = Field(..., description="Creation timestamp")
updated_at: datetime = Field(..., description="Last update timestamp")
@validator('id', pre=True)
def convert_uuid_to_string(cls, v):
if isinstance(v, UUID):
return str(v)
return v
class Config:
from_attributes = True
json_encoders = {
datetime: lambda v: v.isoformat()
}
class WeatherDataList(BaseModel):
"""Schema for paginated weather data responses"""
data: List[WeatherDataResponse]
total: int = Field(..., description="Total number of records")
page: int = Field(..., description="Current page number")
per_page: int = Field(..., description="Records per page")
has_next: bool = Field(..., description="Whether there are more pages")
has_prev: bool = Field(..., description="Whether there are previous pages")
class WeatherForecastList(BaseModel):
"""Schema for paginated weather forecast responses"""
forecasts: List[WeatherForecastResponse]
total: int = Field(..., description="Total number of forecasts")
page: int = Field(..., description="Current page number")
per_page: int = Field(..., description="Forecasts per page")
class WeatherAnalytics(BaseModel):
"""Schema for weather analytics"""
location_id: str
period_start: datetime
period_end: datetime
avg_temperature: Optional[float] = None
min_temperature: Optional[float] = None
max_temperature: Optional[float] = None
total_precipitation: Optional[float] = None
avg_humidity: Optional[float] = None
avg_wind_speed: Optional[float] = None
avg_pressure: Optional[float] = None
weather_conditions: dict = Field(default_factory=dict)
rainy_days: int = 0
sunny_days: int = 0

20
services/data/app/services/__init__.py
View File

@@ -0,0 +1,20 @@
"""
Data Service Layer
Business logic services for data operations
"""
from .sales_service import SalesService
from .data_import_service import DataImportService, EnhancedDataImportService
from .traffic_service import TrafficService
from .weather_service import WeatherService
from .messaging import publish_sales_data_imported, publish_data_updated
__all__ = [
"SalesService",
"DataImportService",
"EnhancedDataImportService",
"TrafficService",
"WeatherService",
"publish_sales_data_imported",
"publish_data_updated"
]

1274
services/data/app/services/data_import_service.py
View File

File diff suppressed because it is too large Load Diff

16
services/data/app/services/messaging.py
View File

@@ -106,6 +106,22 @@ async def publish_import_failed(data: dict) -> bool:
logger.warning("Failed to publish import failed event", error=str(e))
return False
async def publish_sales_data_imported(data: dict) -> bool:
"""Publish sales data imported event"""
try:
return await data_publisher.publish_data_event("sales.imported", data)
except Exception as e:
logger.warning("Failed to publish sales data imported event", error=str(e))
return False
async def publish_data_updated(data: dict) -> bool:
"""Publish data updated event"""
try:
return await data_publisher.publish_data_event("data.updated", data)
except Exception as e:
logger.warning("Failed to publish data updated event", error=str(e))
return False
# Health check for messaging
async def check_messaging_health() -> dict:
"""Check messaging system health"""

520
services/data/app/services/sales_service.py
View File

@@ -1,278 +1,292 @@
# ================================================================
# services/data/app/services/sales_service.py - SIMPLIFIED VERSION
# ================================================================
"""Sales service without notes column for now"""
"""
Sales Service with Repository Pattern
Enhanced service using the new repository architecture for better separation of concerns
"""
from typing import List, Dict, Any, Optional
from datetime import datetime
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import select, and_, func, desc
import structlog
import uuid
from app.repositories.sales_repository import SalesRepository
from app.models.sales import SalesData
from app.schemas.sales import (
SalesDataCreate,
SalesDataResponse,
SalesDataQuery
SalesDataQuery,
SalesAggregation,
SalesImportResult,
SalesValidationResult
)
from shared.database.unit_of_work import UnitOfWork
from shared.database.transactions import transactional
from shared.database.exceptions import DatabaseError, ValidationError
logger = structlog.get_logger()
class SalesService:
"""Enhanced Sales Service using Repository Pattern and Unit of Work"""
@staticmethod
async def create_sales_record(sales_data: SalesDataCreate, db: AsyncSession) -> SalesDataResponse:
"""Create a new sales record"""
try:
# Create new sales record without notes and updated_at for now
db_record = SalesData(
id=uuid.uuid4(),
tenant_id=sales_data.tenant_id,
date=sales_data.date,
product_name=sales_data.product_name,
quantity_sold=sales_data.quantity_sold,
revenue=sales_data.revenue,
location_id=sales_data.location_id,
source=sales_data.source,
created_at=datetime.utcnow()
# Skip notes and updated_at until database is migrated
)
db.add(db_record)
await db.commit()
await db.refresh(db_record)
logger.debug("Sales record created", record_id=db_record.id, product=db_record.product_name)
return SalesDataResponse(
id=db_record.id,
tenant_id=db_record.tenant_id,
date=db_record.date,
product_name=db_record.product_name,
quantity_sold=db_record.quantity_sold,
revenue=db_record.revenue,
location_id=db_record.location_id,
source=db_record.source,
notes=None, # Always None for now
created_at=db_record.created_at,
updated_at=None # Always None for now
)
except Exception as e:
await db.rollback()
logger.error("Failed to create sales record", error=str(e))
raise
def __init__(self, database_manager):
"""Initialize service with database manager for dependency injection"""
self.database_manager = database_manager
@staticmethod
async def get_sales_data(query: SalesDataQuery, db: AsyncSession) -> List[SalesDataResponse]:
"""Get sales data based on query parameters"""
async def create_sales_record(self, sales_data: SalesDataCreate, tenant_id: str) -> SalesDataResponse:
"""Create a new sales record using repository pattern"""
try:
# Build query conditions
conditions = [SalesData.tenant_id == query.tenant_id]
async with self.database_manager.get_session() as session:
async with UnitOfWork(session) as uow:
# Register sales repository
sales_repo = uow.register_repository("sales", SalesRepository, SalesData)
# Ensure tenant_id is set
record_data = sales_data.model_dump()
record_data["tenant_id"] = tenant_id
# Validate the data first
validation_result = await sales_repo.validate_sales_data(record_data)
if not validation_result["is_valid"]:
raise ValidationError(f"Invalid sales data: {validation_result['errors']}")
# Create the record
db_record = await sales_repo.create(record_data)
# Commit transaction
await uow.commit()
logger.debug("Sales record created",
record_id=db_record.id,
product=db_record.product_name,
tenant_id=tenant_id)
return SalesDataResponse.model_validate(db_record)
if query.start_date:
conditions.append(SalesData.date >= query.start_date)
if query.end_date:
conditions.append(SalesData.date <= query.end_date)
if query.product_names:
conditions.append(SalesData.product_name.in_(query.product_names))
if query.location_ids:
conditions.append(SalesData.location_id.in_(query.location_ids))
if query.sources:
conditions.append(SalesData.source.in_(query.sources))
if query.min_quantity:
conditions.append(SalesData.quantity_sold >= query.min_quantity)
if query.max_quantity:
conditions.append(SalesData.quantity_sold <= query.max_quantity)
if query.min_revenue:
conditions.append(SalesData.revenue >= query.min_revenue)
if query.max_revenue:
conditions.append(SalesData.revenue <= query.max_revenue)
# Execute query
stmt = select(SalesData).where(and_(*conditions)).order_by(desc(SalesData.date))
if query.limit:
stmt = stmt.limit(query.limit)
if query.offset:
stmt = stmt.offset(query.offset)
result = await db.execute(stmt)
records = result.scalars().all()
logger.debug("Sales data retrieved", count=len(records), tenant_id=query.tenant_id)
return [SalesDataResponse(
id=record.id,
tenant_id=record.tenant_id,
date=record.date,
product_name=record.product_name,
quantity_sold=record.quantity_sold,
revenue=record.revenue,
location_id=record.location_id,
source=record.source,
notes=None, # Always None for now
created_at=record.created_at,
updated_at=None # Always None for now
) for record in records]
except Exception as e:
logger.error("Failed to retrieve sales data", error=str(e))
except ValidationError:
raise
except Exception as e:
logger.error("Failed to create sales record",
tenant_id=tenant_id,
product=sales_data.product_name,
error=str(e))
raise DatabaseError(f"Failed to create sales record: {str(e)}")
@staticmethod
async def get_sales_analytics(tenant_id: str, start_date: Optional[datetime],
end_date: Optional[datetime], db: AsyncSession) -> Dict[str, Any]:
"""Get basic sales analytics"""
async def get_sales_data(self, query: SalesDataQuery) -> List[SalesDataResponse]:
"""Get sales data based on query parameters using repository pattern"""
try:
conditions = [SalesData.tenant_id == tenant_id]
if start_date:
conditions.append(SalesData.date >= start_date)
if end_date:
conditions.append(SalesData.date <= end_date)
# Total sales
total_stmt = select(
func.sum(SalesData.quantity_sold).label('total_quantity'),
func.sum(SalesData.revenue).label('total_revenue'),
func.count(SalesData.id).label('total_records')
).where(and_(*conditions))
total_result = await db.execute(total_stmt)
totals = total_result.first()
analytics = {
"total_quantity": int(totals.total_quantity or 0),
"total_revenue": float(totals.total_revenue or 0.0),
"total_records": int(totals.total_records or 0),
"average_order_value": float(totals.total_revenue or 0.0) / max(totals.total_records or 1, 1),
"date_range": {
"start": start_date.isoformat() if start_date else None,
"end": end_date.isoformat() if end_date else None
}
}
logger.debug("Sales analytics generated", tenant_id=tenant_id, total_records=analytics["total_records"])
return analytics
except Exception as e:
logger.error("Failed to generate sales analytics", error=str(e))
raise
@staticmethod
async def export_sales_data(tenant_id: str, export_format: str, start_date: Optional[datetime],
end_date: Optional[datetime], products: Optional[List[str]],
db: AsyncSession) -> Optional[Dict[str, Any]]:
"""Export sales data in specified format"""
try:
# Build query conditions
conditions = [SalesData.tenant_id == tenant_id]
if start_date:
conditions.append(SalesData.date >= start_date)
if end_date:
conditions.append(SalesData.date <= end_date)
if products:
conditions.append(SalesData.product_name.in_(products))
stmt = select(SalesData).where(and_(*conditions)).order_by(desc(SalesData.date))
result = await db.execute(stmt)
records = result.scalars().all()
if not records:
return None
# Simple CSV export
if export_format.lower() == "csv":
import io
output = io.StringIO()
output.write("date,product_name,quantity_sold,revenue,location_id,source\n")
for record in records:
output.write(f"{record.date},{record.product_name},{record.quantity_sold},{record.revenue},{record.location_id or ''},{record.source}\n")
return {
"content": output.getvalue(),
"media_type": "text/csv",
"filename": f"sales_export_{datetime.now().strftime('%Y%m%d_%H%M%S')}.csv"
}
return None
except Exception as e:
logger.error("Failed to export sales data", error=str(e))
raise
@staticmethod
async def delete_sales_record(record_id: str, db: AsyncSession) -> bool:
"""Delete a sales record"""
try:
stmt = select(SalesData).where(SalesData.id == record_id)
result = await db.execute(stmt)
record = result.scalar_one_or_none()
if not record:
return False
await db.delete(record)
await db.commit()
logger.debug("Sales record deleted", record_id=record_id)
return True
except Exception as e:
await db.rollback()
logger.error("Failed to delete sales record", error=str(e))
raise
@staticmethod
async def get_products_list(tenant_id: str, db: AsyncSession) -> List[Dict[str, Any]]:
"""Get list of all products with sales data for tenant"""
try:
# Query to get unique products with aggregated sales data
query = (
select(
SalesData.product_name,
func.count(SalesData.id).label('total_sales'),
func.sum(SalesData.quantity_sold).label('total_quantity'),
func.sum(SalesData.revenue).label('total_revenue'),
func.min(SalesData.date).label('first_sale_date'),
func.max(SalesData.date).label('last_sale_date'),
func.avg(SalesData.quantity_sold).label('avg_quantity'),
func.avg(SalesData.revenue).label('avg_revenue')
async with self.database_manager.get_session() as session:
async with UnitOfWork(session) as uow:
sales_repo = uow.register_repository("sales", SalesRepository, SalesData)
# Use repository's advanced query method
records = await sales_repo.get_by_tenant_and_date_range(
tenant_id=str(query.tenant_id),
start_date=query.start_date,
end_date=query.end_date,
product_names=query.product_names,
location_ids=query.location_ids,
skip=query.offset or 0,
limit=query.limit or 100
)
.where(SalesData.tenant_id == tenant_id)
.group_by(SalesData.product_name)
.order_by(desc(func.sum(SalesData.revenue)))
)
result = await db.execute(query)
products_data = result.all()
# Format the response
products = []
for row in products_data:
products.append({
'product_name': row.product_name,
'total_sales': row.total_sales,
'total_quantity': int(row.total_quantity) if row.total_quantity else 0,
'total_revenue': float(row.total_revenue) if row.total_revenue else 0.0,
'first_sale_date': row.first_sale_date.isoformat() if row.first_sale_date else None,
'last_sale_date': row.last_sale_date.isoformat() if row.last_sale_date else None,
'avg_quantity': float(row.avg_quantity) if row.avg_quantity else 0.0,
'avg_revenue': float(row.avg_revenue) if row.avg_revenue else 0.0
})
logger.debug("Products list retrieved successfully",
tenant_id=tenant_id,
product_count=len(products))
return products
logger.debug("Sales data retrieved",
count=len(records),
tenant_id=query.tenant_id)
return [SalesDataResponse.model_validate(record) for record in records]
except Exception as e:
logger.error("Failed to get products list from database",
logger.error("Failed to retrieve sales data",
tenant_id=query.tenant_id,
error=str(e))
raise DatabaseError(f"Failed to retrieve sales data: {str(e)}")
async def get_sales_analytics(self, tenant_id: str, start_date: Optional[datetime] = None,
end_date: Optional[datetime] = None) -> Dict[str, Any]:
"""Get comprehensive sales analytics using repository pattern"""
try:
async with self.database_manager.get_session() as session:
async with UnitOfWork(session) as uow:
sales_repo = uow.register_repository("sales", SalesRepository, SalesData)
# Get summary data
summary = await sales_repo.get_sales_summary(
tenant_id=tenant_id,
start_date=start_date,
end_date=end_date
)
# Get top products
top_products = await sales_repo.get_top_products(
tenant_id=tenant_id,
start_date=start_date,
end_date=end_date,
limit=5
)
# Get aggregated data by day
daily_aggregation = await sales_repo.get_sales_aggregation(
tenant_id=tenant_id,
start_date=start_date,
end_date=end_date,
group_by="daily"
)
analytics = {
**summary,
"top_products": top_products,
"daily_sales": daily_aggregation[:30], # Last 30 days
"average_order_value": (
summary["total_revenue"] / max(summary["total_sales"], 1)
if summary["total_sales"] > 0 else 0.0
)
}
logger.debug("Sales analytics generated",
tenant_id=tenant_id,
total_records=analytics["total_sales"])
return analytics
except Exception as e:
logger.error("Failed to generate sales analytics",
tenant_id=tenant_id,
error=str(e))
raise DatabaseError(f"Failed to generate analytics: {str(e)}")
async def get_sales_aggregation(self, tenant_id: str, start_date: Optional[datetime] = None,
end_date: Optional[datetime] = None, group_by: str = "daily") -> List[SalesAggregation]:
"""Get sales aggregation data"""
try:
async with self.database_manager.get_session() as session:
async with UnitOfWork(session) as uow:
sales_repo = uow.register_repository("sales", SalesRepository, SalesData)
aggregations = await sales_repo.get_sales_aggregation(
tenant_id=tenant_id,
start_date=start_date,
end_date=end_date,
group_by=group_by
)
return [
SalesAggregation(
period=agg["period"],
date=agg["date"],
product_name=agg["product_name"],
total_quantity=agg["total_quantity"],
total_revenue=agg["total_revenue"],
average_quantity=agg["average_quantity"],
average_revenue=agg["average_revenue"],
record_count=agg["record_count"]
)
for agg in aggregations
]
except Exception as e:
logger.error("Failed to get sales aggregation",
tenant_id=tenant_id,
error=str(e))
raise DatabaseError(f"Failed to get aggregation: {str(e)}")
async def export_sales_data(self, tenant_id: str, export_format: str, start_date: Optional[datetime] = None,
end_date: Optional[datetime] = None, products: Optional[List[str]] = None) -> Optional[Dict[str, Any]]:
"""Export sales data in specified format using repository pattern"""
try:
async with self.database_manager.get_session() as session:
async with UnitOfWork(session) as uow:
sales_repo = uow.register_repository("sales", SalesRepository, SalesData)
# Get sales data based on filters
records = await sales_repo.get_by_tenant_and_date_range(
tenant_id=tenant_id,
start_date=start_date,
end_date=end_date,
product_names=products,
skip=0,
limit=10000 # Large limit for export
)
if not records:
return None
# Simple CSV export
if export_format.lower() == "csv":
import io
output = io.StringIO()
output.write("date,product_name,quantity_sold,revenue,location_id,source\n")
for record in records:
output.write(f"{record.date},{record.product_name},{record.quantity_sold},{record.revenue},{record.location_id or ''},{record.source}\n")
logger.info("Sales data exported",
tenant_id=tenant_id,
format=export_format,
record_count=len(records))
return {
"content": output.getvalue(),
"media_type": "text/csv",
"filename": f"sales_export_{datetime.now().strftime('%Y%m%d_%H%M%S')}.csv"
}
return None
except Exception as e:
logger.error("Failed to export sales data",
tenant_id=tenant_id,
error=str(e))
raise DatabaseError(f"Failed to export sales data: {str(e)}")
async def delete_sales_record(self, record_id: str, tenant_id: str) -> bool:
"""Delete a sales record using repository pattern"""
try:
async with self.database_manager.get_session() as session:
async with UnitOfWork(session) as uow:
sales_repo = uow.register_repository("sales", SalesRepository, SalesData)
# First verify the record exists and belongs to the tenant
record = await sales_repo.get_by_id(record_id)
if not record:
return False
if str(record.tenant_id) != tenant_id:
raise ValidationError("Record does not belong to the specified tenant")
# Delete the record
success = await sales_repo.delete(record_id)
if success:
logger.info("Sales record deleted",
record_id=record_id,
tenant_id=tenant_id)
return success
except ValidationError:
raise
except Exception as e:
logger.error("Failed to delete sales record",
record_id=record_id,
error=str(e))
raise DatabaseError(f"Failed to delete sales record: {str(e)}")
async def get_products_list(self, tenant_id: str) -> List[Dict[str, Any]]:
"""Get list of all products with sales data for tenant using repository pattern"""
try:
async with self.database_manager.get_session() as session:
async with UnitOfWork(session) as uow:
sales_repo = uow.register_repository("sales", SalesRepository, SalesData)
# Use repository method for product statistics
products = await sales_repo.get_product_statistics(tenant_id)
logger.debug("Products list retrieved successfully",
tenant_id=tenant_id,
product_count=len(products))
return products
except Exception as e:
logger.error("Failed to get products list",
error=str(e),
tenant_id=tenant_id)
raise
raise DatabaseError(f"Failed to get products list: {str(e)}")