REFACTOR data service
This commit is contained in:
Urtzi Alfaro
397
services/sales/app/api/import_data.py
Normal file
397
services/sales/app/api/import_data.py
Normal file
@@ -0,0 +1,397 @@
|
||||
# services/sales/app/api/import_data.py
|
||||
"""
|
||||
Sales Data Import API Endpoints
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, UploadFile, File, Form, Path
|
||||
from typing import Dict, Any, Optional
|
||||
from uuid import UUID
|
||||
import structlog
|
||||
import json
|
||||
|
||||
from app.services.data_import_service import DataImportService
|
||||
from shared.auth.decorators import get_current_user_dep, get_current_tenant_id_dep
|
||||
|
||||
router = APIRouter(tags=["data-import"])
|
||||
logger = structlog.get_logger()
|
||||
|
||||
|
||||
def get_import_service():
|
||||
"""Dependency injection for DataImportService"""
|
||||
return DataImportService()
|
||||
|
||||
|
||||
@router.post("/tenants/{tenant_id}/sales/import/validate-json")
|
||||
async def validate_json_data(
|
||||
tenant_id: UUID = Path(..., description="Tenant ID"),
|
||||
data: Dict[str, Any] = None,
|
||||
current_user: Dict[str, Any] = Depends(get_current_user_dep),
|
||||
current_tenant: str = Depends(get_current_tenant_id_dep),
|
||||
import_service: DataImportService = Depends(get_import_service)
|
||||
):
|
||||
"""Validate JSON sales data"""
|
||||
try:
|
||||
# Verify tenant access
|
||||
if str(tenant_id) != current_tenant:
|
||||
raise HTTPException(status_code=403, detail="Access denied to this tenant")
|
||||
|
||||
if not data:
|
||||
raise HTTPException(status_code=400, detail="No data provided")
|
||||
|
||||
logger.info("Validating JSON data", tenant_id=tenant_id, record_count=len(data.get("records", [])))
|
||||
|
||||
# Validate the data - handle different input formats
|
||||
if "records" in data:
|
||||
# New format with records array
|
||||
validation_data = {
|
||||
"tenant_id": str(tenant_id),
|
||||
"data": json.dumps(data.get("records", [])),
|
||||
"data_format": "json"
|
||||
}
|
||||
else:
|
||||
# Legacy format where the entire payload is the validation data
|
||||
validation_data = data.copy()
|
||||
validation_data["tenant_id"] = str(tenant_id)
|
||||
if "data_format" not in validation_data:
|
||||
validation_data["data_format"] = "json"
|
||||
|
||||
validation_result = await import_service.validate_import_data(validation_data)
|
||||
|
||||
logger.info("JSON validation completed", tenant_id=tenant_id, valid=validation_result.is_valid)
|
||||
|
||||
return {
|
||||
"is_valid": validation_result.is_valid,
|
||||
"total_records": validation_result.total_records,
|
||||
"valid_records": validation_result.valid_records,
|
||||
"invalid_records": validation_result.invalid_records,
|
||||
"errors": validation_result.errors,
|
||||
"warnings": validation_result.warnings,
|
||||
"summary": validation_result.summary
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Failed to validate JSON data", error=str(e), tenant_id=tenant_id)
|
||||
raise HTTPException(status_code=500, detail=f"Failed to validate data: {str(e)}")
|
||||
|
||||
|
||||
@router.post("/tenants/{tenant_id}/sales/import/validate")
|
||||
async def validate_sales_data_universal(
|
||||
tenant_id: UUID = Path(..., description="Tenant ID"),
|
||||
file: Optional[UploadFile] = File(None),
|
||||
data: Optional[Dict[str, Any]] = None,
|
||||
file_format: Optional[str] = Form(None),
|
||||
current_user: Dict[str, Any] = Depends(get_current_user_dep),
|
||||
current_tenant: str = Depends(get_current_tenant_id_dep),
|
||||
import_service: DataImportService = Depends(get_import_service)
|
||||
):
|
||||
"""Universal validation endpoint for sales data - supports files and JSON"""
|
||||
try:
|
||||
# Verify tenant access
|
||||
if str(tenant_id) != current_tenant:
|
||||
raise HTTPException(status_code=403, detail="Access denied to this tenant")
|
||||
|
||||
# Handle file upload validation
|
||||
if file:
|
||||
logger.info("Validating uploaded file", tenant_id=tenant_id, filename=file.filename)
|
||||
|
||||
# Auto-detect format from filename
|
||||
filename = file.filename.lower()
|
||||
if filename.endswith('.csv'):
|
||||
detected_format = 'csv'
|
||||
elif filename.endswith('.xlsx') or filename.endswith('.xls'):
|
||||
detected_format = 'excel'
|
||||
elif filename.endswith('.json'):
|
||||
detected_format = 'json'
|
||||
else:
|
||||
detected_format = file_format or 'csv' # Default to CSV
|
||||
|
||||
# Read file content
|
||||
content = await file.read()
|
||||
|
||||
if detected_format in ['xlsx', 'xls', 'excel']:
|
||||
# For Excel files, encode as base64
|
||||
import base64
|
||||
file_content = base64.b64encode(content).decode('utf-8')
|
||||
else:
|
||||
# For CSV/JSON, decode as text
|
||||
file_content = content.decode('utf-8')
|
||||
|
||||
validation_data = {
|
||||
"tenant_id": str(tenant_id),
|
||||
"data": file_content,
|
||||
"data_format": detected_format,
|
||||
"filename": file.filename
|
||||
}
|
||||
|
||||
# Handle JSON data validation
|
||||
elif data:
|
||||
logger.info("Validating JSON data", tenant_id=tenant_id)
|
||||
|
||||
validation_data = data.copy()
|
||||
validation_data["tenant_id"] = str(tenant_id)
|
||||
if "data_format" not in validation_data:
|
||||
validation_data["data_format"] = "json"
|
||||
|
||||
else:
|
||||
raise HTTPException(status_code=400, detail="No file or data provided for validation")
|
||||
|
||||
# Perform validation
|
||||
validation_result = await import_service.validate_import_data(validation_data)
|
||||
|
||||
logger.info("Validation completed",
|
||||
tenant_id=tenant_id,
|
||||
valid=validation_result.is_valid,
|
||||
total_records=validation_result.total_records)
|
||||
|
||||
return {
|
||||
"is_valid": validation_result.is_valid,
|
||||
"total_records": validation_result.total_records,
|
||||
"valid_records": validation_result.valid_records,
|
||||
"invalid_records": validation_result.invalid_records,
|
||||
"errors": validation_result.errors,
|
||||
"warnings": validation_result.warnings,
|
||||
"summary": validation_result.summary,
|
||||
"message": "Validation completed successfully" if validation_result.is_valid else "Validation found errors",
|
||||
"details": {
|
||||
"total_records": validation_result.total_records,
|
||||
"format": validation_data.get("data_format", "unknown")
|
||||
}
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Failed to validate sales data", error=str(e), tenant_id=tenant_id)
|
||||
raise HTTPException(status_code=500, detail=f"Failed to validate data: {str(e)}")
|
||||
|
||||
|
||||
@router.post("/tenants/{tenant_id}/sales/import/validate-csv")
|
||||
async def validate_csv_data_legacy(
|
||||
tenant_id: UUID = Path(..., description="Tenant ID"),
|
||||
file: UploadFile = File(...),
|
||||
current_user: Dict[str, Any] = Depends(get_current_user_dep),
|
||||
current_tenant: str = Depends(get_current_tenant_id_dep),
|
||||
import_service: DataImportService = Depends(get_import_service)
|
||||
):
|
||||
"""Legacy CSV validation endpoint - redirects to universal validator"""
|
||||
return await validate_sales_data_universal(
|
||||
tenant_id=tenant_id,
|
||||
file=file,
|
||||
current_user=current_user,
|
||||
current_tenant=current_tenant,
|
||||
import_service=import_service
|
||||
)
|
||||
|
||||
|
||||
@router.post("/tenants/{tenant_id}/sales/import")
|
||||
async def import_sales_data(
|
||||
tenant_id: UUID = Path(..., description="Tenant ID"),
|
||||
data: Optional[Dict[str, Any]] = None,
|
||||
file: Optional[UploadFile] = File(None),
|
||||
file_format: Optional[str] = Form(None),
|
||||
update_existing: bool = Form(False, description="Whether to update existing records"),
|
||||
current_user: Dict[str, Any] = Depends(get_current_user_dep),
|
||||
current_tenant: str = Depends(get_current_tenant_id_dep),
|
||||
import_service: DataImportService = Depends(get_import_service)
|
||||
):
|
||||
"""Enhanced import sales data - supports multiple file formats and JSON"""
|
||||
try:
|
||||
# Verify tenant access
|
||||
if str(tenant_id) != current_tenant:
|
||||
raise HTTPException(status_code=403, detail="Access denied to this tenant")
|
||||
|
||||
# Handle file upload (form data)
|
||||
if file:
|
||||
if not file.filename:
|
||||
raise HTTPException(status_code=400, detail="No file provided")
|
||||
|
||||
logger.info("Starting enhanced file import", tenant_id=tenant_id, filename=file.filename)
|
||||
|
||||
# Auto-detect format from filename
|
||||
filename = file.filename.lower()
|
||||
if filename.endswith('.csv'):
|
||||
detected_format = 'csv'
|
||||
elif filename.endswith('.xlsx') or filename.endswith('.xls'):
|
||||
detected_format = 'excel'
|
||||
elif filename.endswith('.json'):
|
||||
detected_format = 'json'
|
||||
else:
|
||||
detected_format = file_format or 'csv' # Default to CSV
|
||||
|
||||
# Read file content
|
||||
content = await file.read()
|
||||
|
||||
if detected_format in ['xlsx', 'xls', 'excel']:
|
||||
# For Excel files, encode as base64
|
||||
import base64
|
||||
file_content = base64.b64encode(content).decode('utf-8')
|
||||
else:
|
||||
# For CSV/JSON, decode as text
|
||||
file_content = content.decode('utf-8')
|
||||
|
||||
# Import the file using enhanced service
|
||||
import_result = await import_service.process_import(
|
||||
str(tenant_id), # Ensure string type
|
||||
file_content,
|
||||
detected_format,
|
||||
filename=file.filename
|
||||
)
|
||||
|
||||
# Handle JSON data
|
||||
elif data:
|
||||
logger.info("Starting enhanced JSON data import", tenant_id=tenant_id, record_count=len(data.get("records", [])))
|
||||
|
||||
# Import the data - handle different input formats
|
||||
if "records" in data:
|
||||
# New format with records array
|
||||
records_json = json.dumps(data.get("records", []))
|
||||
import_result = await import_service.process_import(
|
||||
str(tenant_id),
|
||||
records_json,
|
||||
"json"
|
||||
)
|
||||
else:
|
||||
# Legacy format - data field contains the data directly
|
||||
import_result = await import_service.process_import(
|
||||
str(tenant_id),
|
||||
data.get("data", ""),
|
||||
data.get("data_format", "json")
|
||||
)
|
||||
else:
|
||||
raise HTTPException(status_code=400, detail="No data or file provided")
|
||||
|
||||
logger.info("Enhanced import completed",
|
||||
tenant_id=tenant_id,
|
||||
created=import_result.records_created,
|
||||
updated=import_result.records_updated,
|
||||
failed=import_result.records_failed,
|
||||
processing_time=import_result.processing_time_seconds)
|
||||
|
||||
# Return enhanced response matching frontend expectations
|
||||
response = {
|
||||
"success": import_result.success,
|
||||
"records_processed": import_result.records_processed,
|
||||
"records_created": import_result.records_created,
|
||||
"records_updated": import_result.records_updated,
|
||||
"records_failed": import_result.records_failed,
|
||||
"errors": import_result.errors,
|
||||
"warnings": import_result.warnings,
|
||||
"processing_time_seconds": import_result.processing_time_seconds,
|
||||
"records_imported": import_result.records_created, # Frontend compatibility
|
||||
"message": f"Successfully imported {import_result.records_created} records" if import_result.success else "Import completed with errors"
|
||||
}
|
||||
|
||||
# Add file-specific information if available
|
||||
if file:
|
||||
response["file_info"] = {
|
||||
"name": file.filename,
|
||||
"format": detected_format,
|
||||
"size_bytes": len(content) if 'content' in locals() else 0
|
||||
}
|
||||
|
||||
return response
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Failed to import sales data", error=str(e), tenant_id=tenant_id, exc_info=True)
|
||||
raise HTTPException(status_code=500, detail=f"Failed to import data: {str(e)}")
|
||||
|
||||
|
||||
@router.post("/tenants/{tenant_id}/sales/import/csv")
|
||||
async def import_csv_data(
|
||||
tenant_id: UUID = Path(..., description="Tenant ID"),
|
||||
file: UploadFile = File(...),
|
||||
update_existing: bool = Form(False, description="Whether to update existing records"),
|
||||
current_user: Dict[str, Any] = Depends(get_current_user_dep),
|
||||
current_tenant: str = Depends(get_current_tenant_id_dep),
|
||||
import_service: DataImportService = Depends(get_import_service)
|
||||
):
|
||||
"""Import CSV sales data file"""
|
||||
try:
|
||||
# Verify tenant access
|
||||
if str(tenant_id) != current_tenant:
|
||||
raise HTTPException(status_code=403, detail="Access denied to this tenant")
|
||||
|
||||
if not file.filename.endswith('.csv'):
|
||||
raise HTTPException(status_code=400, detail="File must be a CSV file")
|
||||
|
||||
logger.info("Starting CSV data import", tenant_id=tenant_id, filename=file.filename)
|
||||
|
||||
# Read file content
|
||||
content = await file.read()
|
||||
file_content = content.decode('utf-8')
|
||||
|
||||
# Import the data
|
||||
import_result = await import_service.process_import(
|
||||
tenant_id,
|
||||
file_content,
|
||||
"csv",
|
||||
filename=file.filename
|
||||
)
|
||||
|
||||
logger.info("CSV import completed",
|
||||
tenant_id=tenant_id,
|
||||
filename=file.filename,
|
||||
created=import_result.records_created,
|
||||
updated=import_result.records_updated,
|
||||
failed=import_result.records_failed)
|
||||
|
||||
return {
|
||||
"success": import_result.success,
|
||||
"records_processed": import_result.records_processed,
|
||||
"records_created": import_result.records_created,
|
||||
"records_updated": import_result.records_updated,
|
||||
"records_failed": import_result.records_failed,
|
||||
"errors": import_result.errors,
|
||||
"warnings": import_result.warnings,
|
||||
"processing_time_seconds": import_result.processing_time_seconds
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Failed to import CSV data", error=str(e), tenant_id=tenant_id)
|
||||
raise HTTPException(status_code=500, detail=f"Failed to import CSV data: {str(e)}")
|
||||
|
||||
|
||||
@router.get("/tenants/{tenant_id}/sales/import/template")
|
||||
async def get_import_template(
|
||||
tenant_id: UUID = Path(..., description="Tenant ID"),
|
||||
format: str = "csv",
|
||||
current_tenant: str = Depends(get_current_tenant_id_dep)
|
||||
):
|
||||
"""Get sales data import template"""
|
||||
try:
|
||||
# Verify tenant access
|
||||
if str(tenant_id) != current_tenant:
|
||||
raise HTTPException(status_code=403, detail="Access denied to this tenant")
|
||||
|
||||
if format not in ["csv", "json"]:
|
||||
raise HTTPException(status_code=400, detail="Format must be 'csv' or 'json'")
|
||||
|
||||
if format == "csv":
|
||||
template = "date,product_name,product_category,product_sku,quantity_sold,unit_price,revenue,cost_of_goods,discount_applied,location_id,sales_channel,source,notes,weather_condition,is_holiday,is_weekend"
|
||||
else:
|
||||
template = {
|
||||
"records": [
|
||||
{
|
||||
"date": "2024-01-01T10:00:00Z",
|
||||
"product_name": "Sample Product",
|
||||
"product_category": "Sample Category",
|
||||
"product_sku": "SAMPLE001",
|
||||
"quantity_sold": 1,
|
||||
"unit_price": 10.50,
|
||||
"revenue": 10.50,
|
||||
"cost_of_goods": 5.25,
|
||||
"discount_applied": 0.0,
|
||||
"location_id": "LOC001",
|
||||
"sales_channel": "in_store",
|
||||
"source": "manual",
|
||||
"notes": "Sample sales record",
|
||||
"weather_condition": "sunny",
|
||||
"is_holiday": False,
|
||||
"is_weekend": False
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
return {"template": template, "format": format}
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Failed to get import template", error=str(e), tenant_id=tenant_id)
|
||||
raise HTTPException(status_code=500, detail=f"Failed to get import template: {str(e)}")
|
||||
Reference in New Issue
Block a user