399 lines
16 KiB
Python
399 lines
16 KiB
Python
# services/sales/app/api/import_data.py
|
|
"""
|
|
Sales Data Import API Endpoints
|
|
"""
|
|
|
|
from fastapi import APIRouter, Depends, HTTPException, UploadFile, File, Form, Path
|
|
from typing import Dict, Any, Optional
|
|
from uuid import UUID
|
|
import structlog
|
|
import json
|
|
|
|
from app.services.data_import_service import DataImportService
|
|
from shared.auth.decorators import get_current_user_dep, get_current_tenant_id_dep
|
|
|
|
router = APIRouter(tags=["data-import"])
|
|
logger = structlog.get_logger()
|
|
|
|
|
|
def get_import_service():
|
|
"""Dependency injection for DataImportService"""
|
|
return DataImportService()
|
|
|
|
|
|
@router.post("/tenants/{tenant_id}/sales/import/validate-json")
|
|
async def validate_json_data(
|
|
tenant_id: UUID = Path(..., description="Tenant ID"),
|
|
data: Dict[str, Any] = None,
|
|
current_user: Dict[str, Any] = Depends(get_current_user_dep),
|
|
current_tenant: str = Depends(get_current_tenant_id_dep),
|
|
import_service: DataImportService = Depends(get_import_service)
|
|
):
|
|
"""Validate JSON sales data"""
|
|
try:
|
|
# Verify tenant access
|
|
if str(tenant_id) != current_tenant:
|
|
raise HTTPException(status_code=403, detail="Access denied to this tenant")
|
|
|
|
if not data:
|
|
raise HTTPException(status_code=400, detail="No data provided")
|
|
|
|
logger.info("Validating JSON data", tenant_id=tenant_id, record_count=len(data.get("records", [])))
|
|
|
|
# Validate the data - handle different input formats
|
|
if "records" in data:
|
|
# New format with records array
|
|
validation_data = {
|
|
"tenant_id": str(tenant_id),
|
|
"data": json.dumps(data.get("records", [])),
|
|
"data_format": "json"
|
|
}
|
|
else:
|
|
# Legacy format where the entire payload is the validation data
|
|
validation_data = data.copy()
|
|
validation_data["tenant_id"] = str(tenant_id)
|
|
if "data_format" not in validation_data:
|
|
validation_data["data_format"] = "json"
|
|
|
|
validation_result = await import_service.validate_import_data(validation_data)
|
|
|
|
logger.info("JSON validation completed", tenant_id=tenant_id, valid=validation_result.is_valid)
|
|
|
|
return {
|
|
"is_valid": validation_result.is_valid,
|
|
"total_records": validation_result.total_records,
|
|
"valid_records": validation_result.valid_records,
|
|
"invalid_records": validation_result.invalid_records,
|
|
"errors": validation_result.errors,
|
|
"warnings": validation_result.warnings,
|
|
"summary": validation_result.summary
|
|
}
|
|
|
|
except Exception as e:
|
|
logger.error("Failed to validate JSON data", error=str(e), tenant_id=tenant_id)
|
|
raise HTTPException(status_code=500, detail=f"Failed to validate data: {str(e)}")
|
|
|
|
|
|
@router.post("/tenants/{tenant_id}/sales/import/validate")
|
|
async def validate_sales_data_universal(
|
|
tenant_id: UUID = Path(..., description="Tenant ID"),
|
|
file: Optional[UploadFile] = File(None),
|
|
data: Optional[Dict[str, Any]] = None,
|
|
file_format: Optional[str] = Form(None),
|
|
current_user: Dict[str, Any] = Depends(get_current_user_dep),
|
|
current_tenant: str = Depends(get_current_tenant_id_dep),
|
|
import_service: DataImportService = Depends(get_import_service)
|
|
):
|
|
"""Universal validation endpoint for sales data - supports files and JSON"""
|
|
try:
|
|
# Verify tenant access
|
|
if str(tenant_id) != current_tenant:
|
|
raise HTTPException(status_code=403, detail="Access denied to this tenant")
|
|
|
|
# Handle file upload validation
|
|
if file:
|
|
logger.info("Validating uploaded file", tenant_id=tenant_id, filename=file.filename)
|
|
|
|
# Auto-detect format from filename
|
|
filename = file.filename.lower()
|
|
if filename.endswith('.csv'):
|
|
detected_format = 'csv'
|
|
elif filename.endswith('.xlsx') or filename.endswith('.xls'):
|
|
detected_format = 'excel'
|
|
elif filename.endswith('.json'):
|
|
detected_format = 'json'
|
|
else:
|
|
detected_format = file_format or 'csv' # Default to CSV
|
|
|
|
# Read file content
|
|
content = await file.read()
|
|
|
|
if detected_format in ['xlsx', 'xls', 'excel']:
|
|
# For Excel files, encode as base64
|
|
import base64
|
|
file_content = base64.b64encode(content).decode('utf-8')
|
|
else:
|
|
# For CSV/JSON, decode as text
|
|
file_content = content.decode('utf-8')
|
|
|
|
validation_data = {
|
|
"tenant_id": str(tenant_id),
|
|
"data": file_content,
|
|
"data_format": detected_format,
|
|
"filename": file.filename
|
|
}
|
|
|
|
# Handle JSON data validation
|
|
elif data:
|
|
logger.info("Validating JSON data", tenant_id=tenant_id)
|
|
|
|
validation_data = data.copy()
|
|
validation_data["tenant_id"] = str(tenant_id)
|
|
if "data_format" not in validation_data:
|
|
validation_data["data_format"] = "json"
|
|
|
|
else:
|
|
raise HTTPException(status_code=400, detail="No file or data provided for validation")
|
|
|
|
# Perform validation
|
|
validation_result = await import_service.validate_import_data(validation_data)
|
|
|
|
logger.info("Validation completed",
|
|
tenant_id=tenant_id,
|
|
valid=validation_result.is_valid,
|
|
total_records=validation_result.total_records)
|
|
|
|
return {
|
|
"is_valid": validation_result.is_valid,
|
|
"total_records": validation_result.total_records,
|
|
"valid_records": validation_result.valid_records,
|
|
"invalid_records": validation_result.invalid_records,
|
|
"errors": validation_result.errors,
|
|
"warnings": validation_result.warnings,
|
|
"summary": validation_result.summary,
|
|
"unique_products": validation_result.unique_products,
|
|
"product_list": validation_result.product_list,
|
|
"message": "Validation completed successfully" if validation_result.is_valid else "Validation found errors",
|
|
"details": {
|
|
"total_records": validation_result.total_records,
|
|
"format": validation_data.get("data_format", "unknown")
|
|
}
|
|
}
|
|
|
|
except Exception as e:
|
|
logger.error("Failed to validate sales data", error=str(e), tenant_id=tenant_id)
|
|
raise HTTPException(status_code=500, detail=f"Failed to validate data: {str(e)}")
|
|
|
|
|
|
@router.post("/tenants/{tenant_id}/sales/import/validate-csv")
|
|
async def validate_csv_data_legacy(
|
|
tenant_id: UUID = Path(..., description="Tenant ID"),
|
|
file: UploadFile = File(...),
|
|
current_user: Dict[str, Any] = Depends(get_current_user_dep),
|
|
current_tenant: str = Depends(get_current_tenant_id_dep),
|
|
import_service: DataImportService = Depends(get_import_service)
|
|
):
|
|
"""Legacy CSV validation endpoint - redirects to universal validator"""
|
|
return await validate_sales_data_universal(
|
|
tenant_id=tenant_id,
|
|
file=file,
|
|
current_user=current_user,
|
|
current_tenant=current_tenant,
|
|
import_service=import_service
|
|
)
|
|
|
|
|
|
@router.post("/tenants/{tenant_id}/sales/import")
|
|
async def import_sales_data(
|
|
tenant_id: UUID = Path(..., description="Tenant ID"),
|
|
data: Optional[Dict[str, Any]] = None,
|
|
file: Optional[UploadFile] = File(None),
|
|
file_format: Optional[str] = Form(None),
|
|
update_existing: bool = Form(False, description="Whether to update existing records"),
|
|
current_user: Dict[str, Any] = Depends(get_current_user_dep),
|
|
current_tenant: str = Depends(get_current_tenant_id_dep),
|
|
import_service: DataImportService = Depends(get_import_service)
|
|
):
|
|
"""Enhanced import sales data - supports multiple file formats and JSON"""
|
|
try:
|
|
# Verify tenant access
|
|
if str(tenant_id) != current_tenant:
|
|
raise HTTPException(status_code=403, detail="Access denied to this tenant")
|
|
|
|
# Handle file upload (form data)
|
|
if file:
|
|
if not file.filename:
|
|
raise HTTPException(status_code=400, detail="No file provided")
|
|
|
|
logger.info("Starting enhanced file import", tenant_id=tenant_id, filename=file.filename)
|
|
|
|
# Auto-detect format from filename
|
|
filename = file.filename.lower()
|
|
if filename.endswith('.csv'):
|
|
detected_format = 'csv'
|
|
elif filename.endswith('.xlsx') or filename.endswith('.xls'):
|
|
detected_format = 'excel'
|
|
elif filename.endswith('.json'):
|
|
detected_format = 'json'
|
|
else:
|
|
detected_format = file_format or 'csv' # Default to CSV
|
|
|
|
# Read file content
|
|
content = await file.read()
|
|
|
|
if detected_format in ['xlsx', 'xls', 'excel']:
|
|
# For Excel files, encode as base64
|
|
import base64
|
|
file_content = base64.b64encode(content).decode('utf-8')
|
|
else:
|
|
# For CSV/JSON, decode as text
|
|
file_content = content.decode('utf-8')
|
|
|
|
# Import the file using enhanced service
|
|
import_result = await import_service.process_import(
|
|
str(tenant_id), # Ensure string type
|
|
file_content,
|
|
detected_format,
|
|
filename=file.filename
|
|
)
|
|
|
|
# Handle JSON data
|
|
elif data:
|
|
logger.info("Starting enhanced JSON data import", tenant_id=tenant_id, record_count=len(data.get("records", [])))
|
|
|
|
# Import the data - handle different input formats
|
|
if "records" in data:
|
|
# New format with records array
|
|
records_json = json.dumps(data.get("records", []))
|
|
import_result = await import_service.process_import(
|
|
str(tenant_id),
|
|
records_json,
|
|
"json"
|
|
)
|
|
else:
|
|
# Legacy format - data field contains the data directly
|
|
import_result = await import_service.process_import(
|
|
str(tenant_id),
|
|
data.get("data", ""),
|
|
data.get("data_format", "json")
|
|
)
|
|
else:
|
|
raise HTTPException(status_code=400, detail="No data or file provided")
|
|
|
|
logger.info("Enhanced import completed",
|
|
tenant_id=tenant_id,
|
|
created=import_result.records_created,
|
|
updated=import_result.records_updated,
|
|
failed=import_result.records_failed,
|
|
processing_time=import_result.processing_time_seconds)
|
|
|
|
# Return enhanced response matching frontend expectations
|
|
response = {
|
|
"success": import_result.success,
|
|
"records_processed": import_result.records_processed,
|
|
"records_created": import_result.records_created,
|
|
"records_updated": import_result.records_updated,
|
|
"records_failed": import_result.records_failed,
|
|
"errors": import_result.errors,
|
|
"warnings": import_result.warnings,
|
|
"processing_time_seconds": import_result.processing_time_seconds,
|
|
"records_imported": import_result.records_created, # Frontend compatibility
|
|
"message": f"Successfully imported {import_result.records_created} records" if import_result.success else "Import completed with errors"
|
|
}
|
|
|
|
# Add file-specific information if available
|
|
if file:
|
|
response["file_info"] = {
|
|
"name": file.filename,
|
|
"format": detected_format,
|
|
"size_bytes": len(content) if 'content' in locals() else 0
|
|
}
|
|
|
|
return response
|
|
|
|
except Exception as e:
|
|
logger.error("Failed to import sales data", error=str(e), tenant_id=tenant_id, exc_info=True)
|
|
raise HTTPException(status_code=500, detail=f"Failed to import data: {str(e)}")
|
|
|
|
|
|
@router.post("/tenants/{tenant_id}/sales/import/csv")
|
|
async def import_csv_data(
|
|
tenant_id: UUID = Path(..., description="Tenant ID"),
|
|
file: UploadFile = File(...),
|
|
update_existing: bool = Form(False, description="Whether to update existing records"),
|
|
current_user: Dict[str, Any] = Depends(get_current_user_dep),
|
|
current_tenant: str = Depends(get_current_tenant_id_dep),
|
|
import_service: DataImportService = Depends(get_import_service)
|
|
):
|
|
"""Import CSV sales data file"""
|
|
try:
|
|
# Verify tenant access
|
|
if str(tenant_id) != current_tenant:
|
|
raise HTTPException(status_code=403, detail="Access denied to this tenant")
|
|
|
|
if not file.filename.endswith('.csv'):
|
|
raise HTTPException(status_code=400, detail="File must be a CSV file")
|
|
|
|
logger.info("Starting CSV data import", tenant_id=tenant_id, filename=file.filename)
|
|
|
|
# Read file content
|
|
content = await file.read()
|
|
file_content = content.decode('utf-8')
|
|
|
|
# Import the data
|
|
import_result = await import_service.process_import(
|
|
tenant_id,
|
|
file_content,
|
|
"csv",
|
|
filename=file.filename
|
|
)
|
|
|
|
logger.info("CSV import completed",
|
|
tenant_id=tenant_id,
|
|
filename=file.filename,
|
|
created=import_result.records_created,
|
|
updated=import_result.records_updated,
|
|
failed=import_result.records_failed)
|
|
|
|
return {
|
|
"success": import_result.success,
|
|
"records_processed": import_result.records_processed,
|
|
"records_created": import_result.records_created,
|
|
"records_updated": import_result.records_updated,
|
|
"records_failed": import_result.records_failed,
|
|
"errors": import_result.errors,
|
|
"warnings": import_result.warnings,
|
|
"processing_time_seconds": import_result.processing_time_seconds
|
|
}
|
|
|
|
except Exception as e:
|
|
logger.error("Failed to import CSV data", error=str(e), tenant_id=tenant_id)
|
|
raise HTTPException(status_code=500, detail=f"Failed to import CSV data: {str(e)}")
|
|
|
|
|
|
@router.get("/tenants/{tenant_id}/sales/import/template")
|
|
async def get_import_template(
|
|
tenant_id: UUID = Path(..., description="Tenant ID"),
|
|
format: str = "csv",
|
|
current_tenant: str = Depends(get_current_tenant_id_dep)
|
|
):
|
|
"""Get sales data import template"""
|
|
try:
|
|
# Verify tenant access
|
|
if str(tenant_id) != current_tenant:
|
|
raise HTTPException(status_code=403, detail="Access denied to this tenant")
|
|
|
|
if format not in ["csv", "json"]:
|
|
raise HTTPException(status_code=400, detail="Format must be 'csv' or 'json'")
|
|
|
|
if format == "csv":
|
|
template = "date,product_name,product_category,product_sku,quantity_sold,unit_price,revenue,cost_of_goods,discount_applied,location_id,sales_channel,source,notes,weather_condition,is_holiday,is_weekend"
|
|
else:
|
|
template = {
|
|
"records": [
|
|
{
|
|
"date": "2024-01-01T10:00:00Z",
|
|
"product_name": "Sample Product",
|
|
"product_category": "Sample Category",
|
|
"product_sku": "SAMPLE001",
|
|
"quantity_sold": 1,
|
|
"unit_price": 10.50,
|
|
"revenue": 10.50,
|
|
"cost_of_goods": 5.25,
|
|
"discount_applied": 0.0,
|
|
"location_id": "LOC001",
|
|
"sales_channel": "in_store",
|
|
"source": "manual",
|
|
"notes": "Sample sales record",
|
|
"weather_condition": "sunny",
|
|
"is_holiday": False,
|
|
"is_weekend": False
|
|
}
|
|
]
|
|
}
|
|
|
|
return {"template": template, "format": format}
|
|
|
|
except Exception as e:
|
|
logger.error("Failed to get import template", error=str(e), tenant_id=tenant_id)
|
|
raise HTTPException(status_code=500, detail=f"Failed to get import template: {str(e)}") |