# services/sales/app/services/data_import_service.py
"""
Data Import Service
Service for importing sales data using repository pattern and enhanced error handling
"""

import csv
import io
import json
import base64
import pandas as pd
from typing import Dict, Any, List, Optional, Union
from datetime import datetime, timezone
from uuid import UUID
import structlog
import re
import asyncio

from app.repositories.sales_repository import SalesRepository
from app.models.sales import SalesData
from app.schemas.sales import SalesDataCreate
from app.core.database import get_db_transaction
from app.services.inventory_client import InventoryServiceClient

logger = structlog.get_logger()


# Import result schemas (dataclass definitions)
from dataclasses import dataclass, field
from typing import List, Dict, Any

@dataclass
class SalesValidationResult:
    is_valid: bool
    total_records: int
    valid_records: int
    invalid_records: int
    errors: List[Dict[str, Any]]
    warnings: List[Dict[str, Any]]
    summary: Dict[str, Any]
    unique_products: int = 0
    product_list: List[str] = field(default_factory=list)

@dataclass
class SalesImportResult:
    success: bool
    records_processed: int
    records_created: int
    records_updated: int
    records_failed: int
    errors: List[Dict[str, Any]]
    warnings: List[Dict[str, Any]]
    processing_time_seconds: float


class DataImportService:
    """Enhanced data import service using repository pattern with STRICT validation for production"""
    
    # PRODUCTION VALIDATION CONFIGURATION
    STRICT_VALIDATION = False  # Set to False for lenient validation, True for production quality
    MAX_QUANTITY_PER_DAY = 10000  # Maximum reasonable quantity per product per day
    MAX_REVENUE_PER_ITEM = 100000  # Maximum reasonable revenue per line item  
    MAX_UNIT_PRICE = 10000  # Maximum reasonable price per unit for bakery items
    
    # Common column mappings for different languages/formats
    COLUMN_MAPPINGS = {
        'date': ['date', 'fecha', 'datum', 'data', 'dia'],
        'datetime': ['datetime', 'fecha_hora', 'timestamp'],
        'product': ['product', 'producto', 'item', 'articulo', 'nombre', 'name'],
        'product_name': ['product_name', 'nombre_producto', 'item_name'],
        'quantity': ['quantity', 'cantidad', 'qty', 'units', 'unidades'],
        'quantity_sold': ['quantity_sold', 'cantidad_vendida', 'sold'],
        'revenue': ['revenue', 'ingresos', 'sales', 'ventas', 'total', 'importe'],
        'price': ['price', 'precio', 'cost', 'coste'],
        'location': ['location', 'ubicacion', 'tienda', 'store', 'punto_venta'],
        'location_id': ['location_id', 'store_id', 'tienda_id'],
    }
    
    DATE_FORMATS = [
        '%Y-%m-%d', '%d/%m/%Y', '%m/%d/%Y', '%d-%m-%Y', '%m-%d-%Y',
        '%d.%m.%Y', '%Y/%m/%d', '%d/%m/%y', '%m/%d/%y',
        '%Y-%m-%d %H:%M:%S', '%d/%m/%Y %H:%M',
    ]
    
    def __init__(self):
        """Initialize enhanced import service"""
        self.inventory_client = InventoryServiceClient()
        # Product resolution cache for the import session
        self.product_cache = {}  # product_name -> inventory_product_id
        self.failed_products = set()  # Track products that failed to resolve
    
    async def validate_import_data(self, data: Dict[str, Any]) -> SalesValidationResult:
        """Enhanced validation with better error handling and suggestions"""
        try:
            logger.info("Starting enhanced import data validation", tenant_id=data.get("tenant_id"))
            
            validation_result = SalesValidationResult(
                is_valid=True,
                total_records=0,
                valid_records=0,
                invalid_records=0,
                errors=[],
                warnings=[],
                summary={},
                unique_products=0,
                product_list=[]
            )
            
            errors = []
            warnings = []
            
            # Basic validation checks
            if not data.get("tenant_id"):
                errors.append({
                    "type": "missing_field",
                    "message": "tenant_id es requerido",
                    "field": "tenant_id",
                    "row": None,
                    "code": "MISSING_TENANT_ID"
                })
            
            if not data.get("data"):
                errors.append({
                    "type": "missing_data",
                    "message": "Datos de archivo faltantes",
                    "field": "data",
                    "row": None,
                    "code": "NO_DATA_PROVIDED"
                })
                
                validation_result.is_valid = False
                validation_result.errors = errors
                validation_result.summary = {
                    "status": "failed",
                    "reason": "no_data_provided",
                    "file_format": data.get("data_format", "unknown"),
                    "suggestions": ["Selecciona un archivo válido para importar"]
                }
                return validation_result
            
            # Validate file format
            format_type = data.get("data_format", "").lower()
            supported_formats = ["csv", "excel", "xlsx", "xls", "json", "pos"]
            
            if format_type not in supported_formats:
                errors.append({
                    "type": "unsupported_format",
                    "message": f"Formato no soportado: {format_type}",
                    "field": "data_format",
                    "row": None,
                    "code": "UNSUPPORTED_FORMAT"
                })
            
            # Validate data size
            data_content = data.get("data", "")
            data_size = len(data_content)
            
            if data_size == 0:
                errors.append({
                    "type": "empty_file",
                    "message": "El archivo está vacío",
                    "field": "data",
                    "row": None,
                    "code": "EMPTY_FILE"
                })
            elif data_size > 10 * 1024 * 1024:  # 10MB limit
                errors.append({
                    "type": "file_too_large",
                    "message": "Archivo demasiado grande (máximo 10MB)",
                    "field": "data",
                    "row": None,
                    "code": "FILE_TOO_LARGE"
                })
            elif data_size > 1024 * 1024:  # 1MB warning
                warnings.append({
                    "type": "large_file",
                    "message": "Archivo grande detectado. El procesamiento puede tomar más tiempo.",
                    "field": "data",
                    "row": None,
                    "code": "LARGE_FILE_WARNING"
                })
            
            # Analyze CSV content if format is CSV
            if format_type == "csv" and data_content and not errors:
                try:
                    reader = csv.DictReader(io.StringIO(data_content))
                    rows = list(reader)
                    
                    validation_result.total_records = len(rows)
                    
                    if not rows:
                        errors.append({
                            "type": "empty_content",
                            "message": "El archivo CSV no contiene datos",
                            "field": "data",
                            "row": None,
                            "code": "NO_CONTENT"
                        })
                    else:
                        # Enhanced column analysis
                        headers = list(rows[0].keys()) if rows else []
                        column_mapping = self._detect_columns(headers)
                        
                        # Check for required columns
                        if not column_mapping.get('date'):
                            errors.append({
                                "type": "missing_column",
                                "message": "Columna de fecha no encontrada",
                                "field": "date",
                                "row": None,
                                "code": "MISSING_DATE_COLUMN"
                            })
                        
                        if not column_mapping.get('product'):
                            errors.append({
                                "type": "missing_column",
                                "message": "Columna de producto no encontrada",
                                "field": "product",
                                "row": None,
                                "code": "MISSING_PRODUCT_COLUMN"
                            })
                        
                        # Extract unique products for AI suggestions
                        if column_mapping.get('product') and not errors:
                            product_column = column_mapping['product']
                            unique_products_set = set()
                            
                            for row in rows:
                                product_name = row.get(product_column, '').strip()
                                if product_name and len(product_name) > 0:
                                    unique_products_set.add(product_name)
                            
                            validation_result.product_list = list(unique_products_set)
                            validation_result.unique_products = len(unique_products_set)
                            
                            logger.info(f"Extracted {validation_result.unique_products} unique products from CSV", 
                                      tenant_id=data.get("tenant_id"))
                        
                        if not column_mapping.get('quantity'):
                            warnings.append({
                                "type": "missing_column",
                                "message": "Columna de cantidad no encontrada, se usará 1 por defecto",
                                "field": "quantity",
                                "row": None,
                                "code": "MISSING_QUANTITY_COLUMN"
                            })
                        
                        # Enhanced data quality estimation
                        if not errors:
                            sample_size = min(10, len(rows))
                            sample_rows = rows[:sample_size]
                            quality_issues = 0
                            
                            for i, row in enumerate(sample_rows):
                                parsed_data = await self._parse_row_data(row, column_mapping, i + 1)
                                if parsed_data.get("skip") or parsed_data.get("errors"):
                                    quality_issues += 1
                            
                            estimated_error_rate = (quality_issues / sample_size) * 100 if sample_size > 0 else 0
                            estimated_invalid = int(validation_result.total_records * estimated_error_rate / 100)
                            
                            validation_result.valid_records = validation_result.total_records - estimated_invalid
                            validation_result.invalid_records = estimated_invalid
                            
                            # STRICT: Any data quality issues should fail validation for production
                            if estimated_error_rate > 0:
                                errors.append({
                                    "type": "data_quality_error",
                                    "message": f"Falló la validación de calidad: {estimated_error_rate:.0f}% de los datos tienen errores críticos",
                                    "field": "data", 
                                    "row": None,
                                    "code": "DATA_QUALITY_FAILED"
                                })
                                
                                # Add specific error details
                                if estimated_error_rate > 50:
                                    errors.append({
                                        "type": "data_quality_critical",
                                        "message": f"Calidad de datos crítica: más del 50% de los registros tienen errores",
                                        "field": "data",
                                        "row": None, 
                                        "code": "DATA_QUALITY_CRITICAL"
                                    })
                                elif estimated_error_rate > 20:
                                    errors.append({
                                        "type": "data_quality_high",
                                        "message": f"Alta tasa de errores detectada: {estimated_error_rate:.0f}% de los datos requieren corrección",
                                        "field": "data",
                                        "row": None,
                                        "code": "DATA_QUALITY_HIGH_ERROR_RATE" 
                                    })
                                else:
                                    # Even small error rates are now treated as validation failures
                                    errors.append({
                                        "type": "data_quality_detected",
                                        "message": f"Se detectaron errores de validación en {estimated_error_rate:.0f}% de los datos",
                                        "field": "data",
                                        "row": None,
                                        "code": "DATA_QUALITY_ERRORS_FOUND"
                                    })
                        else:
                            validation_result.valid_records = 0
                            validation_result.invalid_records = validation_result.total_records
                            
                except Exception as csv_error:
                    logger.warning("Enhanced CSV analysis failed", error=str(csv_error))
                    warnings.append({
                        "type": "analysis_warning",
                        "message": f"No se pudo analizar completamente el CSV: {str(csv_error)}",
                        "field": "data",
                        "row": None,
                        "code": "CSV_ANALYSIS_WARNING"
                    })
            
            # Set validation result
            validation_result.is_valid = len(errors) == 0
            validation_result.errors = errors
            validation_result.warnings = warnings
            
            # Enhanced summary generation
            validation_result.summary = {
                "status": "valid" if validation_result.is_valid else "invalid",
                "file_format": format_type,
                "file_size_bytes": data_size,
                "file_size_mb": round(data_size / (1024 * 1024), 2),
                "estimated_processing_time_seconds": max(1, validation_result.total_records // 100),
                "validation_timestamp": datetime.utcnow().isoformat(),
                "detected_columns": list(self._detect_columns(list(csv.DictReader(io.StringIO(data_content)).fieldnames or [])).keys()) if format_type == "csv" and data_content else [],
                "suggestions": self._generate_suggestions(validation_result, format_type, len(warnings))
            }
            
            logger.info("Enhanced import validation completed",
                       is_valid=validation_result.is_valid,
                       total_records=validation_result.total_records,
                       error_count=len(errors),
                       warning_count=len(warnings))
            
            return validation_result
            
        except Exception as e:
            logger.error("Enhanced validation process failed", error=str(e))
            
            return SalesValidationResult(
                is_valid=False,
                total_records=0,
                valid_records=0,
                invalid_records=0,
                errors=[{
                    "type": "system_error",
                    "message": f"Error en el proceso de validación: {str(e)}",
                    "field": None,
                    "row": None,
                    "code": "SYSTEM_ERROR"
                }],
                warnings=[],
                summary={
                    "status": "error",
                    "file_format": data.get("data_format", "unknown"),
                    "error_type": "system_error",
                    "suggestions": [
                        "Intenta de nuevo con un archivo diferente",
                        "Contacta soporte si el problema persiste"
                    ]
                }
            )
    
    async def process_import(
        self, 
        tenant_id: str, 
        content: str, 
        file_format: str, 
        filename: Optional[str] = None
    ) -> SalesImportResult:
        """Enhanced data import processing with better error handling"""
        start_time = datetime.utcnow()
        
        try:
            # Clear cache for new import session
            self._clear_import_cache()
            
            logger.info("Starting enhanced data import",
                       filename=filename,
                       format=file_format,
                       tenant_id=tenant_id)
            
            async with get_db_transaction() as db:
                repository = SalesRepository(db)
                
                # Process data based on format
                if file_format.lower() == 'csv':
                    result = await self._process_csv_data(tenant_id, content, repository, filename)
                elif file_format.lower() == 'json':
                    result = await self._process_json_data(tenant_id, content, repository, filename)
                elif file_format.lower() in ['excel', 'xlsx']:
                    result = await self._process_excel_data(tenant_id, content, repository, filename)
                else:
                    raise ValueError(f"Unsupported format: {file_format}")
                
                # Calculate processing time
                end_time = datetime.utcnow()
                processing_time = (end_time - start_time).total_seconds()
                
                # Build enhanced final result
                final_result = SalesImportResult(
                    success=result.get("success", False),
                    records_processed=result.get("total_rows", 0),
                    records_created=result.get("records_created", 0),
                    records_updated=0,  # We don't update, only create
                    records_failed=result.get("total_rows", 0) - result.get("records_created", 0),
                    errors=self._structure_messages(result.get("errors", [])),
                    warnings=self._structure_messages(result.get("warnings", [])),
                    processing_time_seconds=processing_time
                )
                
                logger.info("Enhanced data import completed successfully",
                           records_created=final_result.records_created,
                           processing_time=processing_time)
                
                return final_result
                
        except Exception as e:
            end_time = datetime.utcnow()
            processing_time = (end_time - start_time).total_seconds()
            
            logger.error("Enhanced data import failed", error=str(e), tenant_id=tenant_id)
            
            return SalesImportResult(
                success=False,
                records_processed=0,
                records_created=0,
                records_updated=0,
                records_failed=0,
                errors=[{
                    "type": "import_error",
                    "message": f"Import failed: {str(e)}",
                    "field": None,
                    "row": None,
                    "code": "IMPORT_FAILURE"
                }],
                warnings=[],
                processing_time_seconds=processing_time
            )
    
    async def _process_csv_data(
        self, 
        tenant_id: str, 
        csv_content: str, 
        repository: SalesRepository, 
        filename: Optional[str] = None
    ) -> Dict[str, Any]:
        """Enhanced CSV processing with batch product resolution for better reliability"""
        try:
            reader = csv.DictReader(io.StringIO(csv_content))
            rows = list(reader)
            
            if not rows:
                return {
                    "success": False,
                    "total_rows": 0,
                    "records_created": 0,
                    "errors": ["CSV file is empty"],
                    "warnings": []
                }
            
            # Enhanced column mapping
            column_mapping = self._detect_columns(list(rows[0].keys()))
            
            # Pre-process to extract unique products for batch creation
            unique_products = set()
            parsed_rows = []
            
            logger.info(f"Pre-processing {len(rows)} records to identify unique products")
            
            for index, row in enumerate(rows):
                try:
                    # Enhanced data parsing and validation
                    parsed_data = await self._parse_row_data(row, column_mapping, index + 1)
                    if not parsed_data.get("skip"):
                        unique_products.add((
                            parsed_data["product_name"],
                            parsed_data.get("product_category", "general")
                        ))
                        parsed_rows.append((index, parsed_data))
                except Exception as e:
                    logger.warning(f"Failed to parse row {index + 1}: {e}")
                    continue
            
            logger.info(f"Found {len(unique_products)} unique products, attempting batch resolution")
            
            # Try to resolve/create all unique products in batch
            await self._batch_resolve_products(unique_products, tenant_id)
            
            # Now process the actual sales records
            records_created = 0
            errors = []
            warnings = []
            
            logger.info(f"Processing {len(parsed_rows)} validated records for sales creation")
            
            for index, parsed_data in parsed_rows:
                try:
                    # Resolve product name to inventory_product_id (should be cached now)
                    inventory_product_id = await self._resolve_product_to_inventory_id(
                        parsed_data["product_name"], 
                        parsed_data.get("product_category"),
                        tenant_id
                    )
                    
                    if not inventory_product_id:
                        error_msg = f"Row {index + 1}: Could not resolve product '{parsed_data['product_name']}' to inventory ID"
                        errors.append(error_msg)
                        logger.warning("Product resolution failed", error=error_msg)
                        continue
                    
                    # Create sales record with enhanced data
                    sales_data = SalesDataCreate(
                        tenant_id=tenant_id,
                        date=parsed_data["date"],
                        inventory_product_id=inventory_product_id,
                        quantity_sold=parsed_data["quantity_sold"],
                        unit_price=parsed_data.get("unit_price"),
                        revenue=parsed_data.get("revenue"),
                        location_id=parsed_data.get("location_id"),
                        source="csv"
                    )
                    
                    created_record = await repository.create_sales_record(sales_data, tenant_id)
                    records_created += 1
                    
                    # Enhanced progress logging
                    if records_created % 100 == 0:
                        logger.info(f"Enhanced processing: {records_created}/{len(rows)} records completed...")
                        
                except Exception as e:
                    error_msg = f"Row {index + 1}: {str(e)}"
                    errors.append(error_msg)
                    logger.warning("Enhanced record processing failed", error=error_msg)
            
            success_rate = (records_created / len(rows)) * 100 if rows else 0
            
            return {
                "success": records_created > 0,
                "total_rows": len(rows),
                "records_created": records_created,
                "success_rate": success_rate,
                "errors": errors,
                "warnings": warnings
            }
            
        except Exception as e:
            logger.error("Enhanced CSV processing failed", error=str(e))
            raise
    
    async def _process_json_data(
        self, 
        tenant_id: str, 
        json_content: str, 
        repository: SalesRepository, 
        filename: Optional[str] = None
    ) -> Dict[str, Any]:
        """Enhanced JSON processing with pandas integration"""
        try:
            # Parse JSON with base64 support
            if json_content.startswith('data:'):
                json_content = base64.b64decode(json_content.split(',')[1]).decode('utf-8')
            
            data = json.loads(json_content)
            
            # Handle different JSON structures
            if isinstance(data, dict):
                if 'data' in data:
                    records = data['data']
                elif 'records' in data:
                    records = data['records']
                elif 'sales' in data:
                    records = data['sales']
                else:
                    records = [data]  # Single record
            elif isinstance(data, list):
                records = data
            else:
                raise ValueError("Invalid JSON format")
            
            # Convert to DataFrame for enhanced processing
            if records:
                df = pd.DataFrame(records)
                df.columns = df.columns.str.strip().str.lower()
                
                return await self._process_dataframe(tenant_id, df, repository, "json", filename)
            else:
                return {
                    "success": False,
                    "total_rows": 0,
                    "records_created": 0,
                    "errors": ["No records found in JSON"],
                    "warnings": []
                }
            
        except json.JSONDecodeError as e:
            raise ValueError(f"Invalid JSON: {str(e)}")
        except Exception as e:
            logger.error("Enhanced JSON processing failed", error=str(e))
            raise
    
    async def _process_excel_data(
        self, 
        tenant_id: str, 
        excel_content: str, 
        repository: SalesRepository, 
        filename: Optional[str] = None
    ) -> Dict[str, Any]:
        """Enhanced Excel processing with base64 support"""
        try:
            # Decode base64 content
            if excel_content.startswith('data:'):
                excel_bytes = base64.b64decode(excel_content.split(',')[1])
            else:
                excel_bytes = base64.b64decode(excel_content)
            
            # Read Excel file with pandas
            df = pd.read_excel(io.BytesIO(excel_bytes), sheet_name=0)
            
            # Enhanced column cleaning
            df.columns = df.columns.str.strip().str.lower()
            
            # Remove empty rows
            df = df.dropna(how='all')
            
            return await self._process_dataframe(tenant_id, df, repository, "excel", filename)
            
        except Exception as e:
            logger.error("Enhanced Excel processing failed", error=str(e))
            raise
    
    async def _process_dataframe(
        self, 
        tenant_id: str, 
        df: pd.DataFrame, 
        repository: SalesRepository,
        source: str,
        filename: Optional[str] = None
    ) -> Dict[str, Any]:
        """Enhanced DataFrame processing with better error handling"""
        try:
            # Enhanced column mapping
            column_mapping = self._detect_columns(df.columns.tolist())
            
            if not column_mapping.get('date') or not column_mapping.get('product'):
                required_missing = []
                if not column_mapping.get('date'):
                    required_missing.append("date")
                if not column_mapping.get('product'):
                    required_missing.append("product")
                
                raise ValueError(f"Required columns missing: {', '.join(required_missing)}")
            
            records_created = 0
            errors = []
            warnings = []
            
            logger.info(f"Enhanced processing of {len(df)} records from {source}")
            
            for index, row in df.iterrows():
                try:
                    # Convert pandas row to dict
                    row_dict = {}
                    for col in df.columns:
                        val = row[col]
                        # Handle pandas NaN values
                        if pd.isna(val):
                            row_dict[col] = None
                        else:
                            row_dict[col] = val
                    
                    # Enhanced data parsing
                    parsed_data = await self._parse_row_data(row_dict, column_mapping, index + 1)
                    if parsed_data.get("skip"):
                        errors.extend(parsed_data.get("errors", []))
                        warnings.extend(parsed_data.get("warnings", []))
                        continue
                    
                    # Resolve product name to inventory_product_id
                    inventory_product_id = await self._resolve_product_to_inventory_id(
                        parsed_data["product_name"], 
                        parsed_data.get("product_category"),
                        tenant_id
                    )
                    
                    if not inventory_product_id:
                        error_msg = f"Row {index + 1}: Could not resolve product '{parsed_data['product_name']}' to inventory ID"
                        errors.append(error_msg)
                        logger.warning("Product resolution failed", error=error_msg)
                        continue
                    
                    # Create enhanced sales record
                    sales_data = SalesDataCreate(
                        tenant_id=tenant_id,
                        date=parsed_data["date"],
                        inventory_product_id=inventory_product_id,
                        quantity_sold=parsed_data["quantity_sold"],
                        unit_price=parsed_data.get("unit_price"),
                        revenue=parsed_data.get("revenue"),
                        location_id=parsed_data.get("location_id"),
                        source=source
                    )
                    
                    created_record = await repository.create_sales_record(sales_data, tenant_id)
                    records_created += 1
                    
                    # Progress logging for large datasets
                    if records_created % 100 == 0:
                        logger.info(f"Enhanced DataFrame processing: {records_created}/{len(df)} records completed...")
                        
                except Exception as e:
                    error_msg = f"Row {index + 1}: {str(e)}"
                    errors.append(error_msg)
                    logger.warning("Enhanced record processing failed", error=error_msg)
            
            success_rate = (records_created / len(df)) * 100 if len(df) > 0 else 0
            
            return {
                "success": records_created > 0,
                "total_rows": len(df),
                "records_created": records_created,
                "success_rate": success_rate,
                "errors": errors[:10],  # Limit errors for performance
                "warnings": warnings[:10]  # Limit warnings
            }
            
        except ValueError:
            raise
        except Exception as e:
            logger.error("Enhanced DataFrame processing failed", error=str(e))
            raise
    
    async def _parse_row_data(
        self, 
        row: Dict[str, Any], 
        column_mapping: Dict[str, str], 
        row_number: int
    ) -> Dict[str, Any]:
        """Enhanced row data parsing with better validation"""
        errors = []
        warnings = []
        
        try:
            # Enhanced date extraction and validation
            date_str = str(row.get(column_mapping.get('date', ''), '')).strip()
            if not date_str or date_str.lower() in ['nan', 'null', 'none', '']:
                errors.append(f"Row {row_number}: Missing date")
                return {"skip": True, "errors": errors, "warnings": warnings}
            
            parsed_date = self._parse_date(date_str)
            if not parsed_date:
                errors.append(f"Row {row_number}: Invalid date format: {date_str}")
                return {"skip": True, "errors": errors, "warnings": warnings}
            
            # Enhanced product name extraction and cleaning
            product_name = str(row.get(column_mapping.get('product', ''), '')).strip()
            if not product_name or product_name.lower() in ['nan', 'null', 'none', '']:
                errors.append(f"Row {row_number}: Missing product name")
                return {"skip": True, "errors": errors, "warnings": warnings}
            
            product_name = self._clean_product_name(product_name)
            
            # STRICT quantity validation for production data quality
            quantity_raw = row.get(column_mapping.get('quantity', 'quantity'), 1)
            try:
                if pd.isna(quantity_raw):
                    # Allow default quantity of 1 for missing values
                    quantity = 1
                else:
                    quantity = int(float(str(quantity_raw).replace(',', '.')))
                    if quantity <= 0:
                        # STRICT: Treat invalid quantities as ERRORS, not warnings
                        errors.append(f"Row {row_number}: Invalid quantity ({quantity}) - quantities must be positive")
                        return {"skip": True, "errors": errors, "warnings": warnings}
                    elif self.STRICT_VALIDATION and quantity > self.MAX_QUANTITY_PER_DAY:
                        # STRICT: Check for unrealistic quantities 
                        errors.append(f"Row {row_number}: Unrealistic quantity ({quantity}) - exceeds maximum expected daily sales ({self.MAX_QUANTITY_PER_DAY})")
                        return {"skip": True, "errors": errors, "warnings": warnings}
            except (ValueError, TypeError):
                # STRICT: Treat non-numeric quantities as ERRORS
                errors.append(f"Row {row_number}: Invalid quantity format ({quantity_raw}) - must be a positive number")
                return {"skip": True, "errors": errors, "warnings": warnings}
            
            # Enhanced revenue extraction
            revenue = None
            unit_price = None
            if 'revenue' in column_mapping and column_mapping['revenue'] in row:
                revenue_raw = row.get(column_mapping['revenue'])
                if revenue_raw and not pd.isna(revenue_raw) and str(revenue_raw).lower() not in ['nan', 'null', 'none', '']:
                    try:
                        revenue = float(str(revenue_raw).replace(',', '.').replace('€', '').replace('$', '').strip())
                        if revenue < 0:
                            # STRICT: Treat negative revenue as ERROR, not warning
                            errors.append(f"Row {row_number}: Negative revenue ({revenue}) - revenue must be positive or zero")
                            return {"skip": True, "errors": errors, "warnings": warnings}
                        else:
                            # STRICT: Check for unrealistic revenue values
                            if self.STRICT_VALIDATION and revenue > self.MAX_REVENUE_PER_ITEM:
                                errors.append(f"Row {row_number}: Unrealistic revenue ({revenue}) - exceeds maximum expected value ({self.MAX_REVENUE_PER_ITEM})")
                                return {"skip": True, "errors": errors, "warnings": warnings}
                            
                            # Calculate unit price if we have both revenue and quantity
                            unit_price = revenue / quantity if quantity > 0 else None
                            
                            # STRICT: Validate unit price reasonableness
                            if unit_price and unit_price > 10000:  # More than €10,000 per unit seems unrealistic for bakery
                                errors.append(f"Row {row_number}: Unrealistic unit price ({unit_price:.2f}) - check quantity and revenue values")
                                return {"skip": True, "errors": errors, "warnings": warnings}
                    except (ValueError, TypeError):
                        # STRICT: Treat invalid revenue format as ERROR
                        errors.append(f"Row {row_number}: Invalid revenue format ({revenue_raw}) - must be a valid number")
                        return {"skip": True, "errors": errors, "warnings": warnings}
            
            # Enhanced location extraction
            location_id = None
            if 'location' in column_mapping and column_mapping['location'] in row:
                location_raw = row.get(column_mapping['location'])
                if location_raw and not pd.isna(location_raw) and str(location_raw).lower() not in ['nan', 'null', 'none', '']:
                    location_id = str(location_raw).strip()
            
            # Enhanced product category extraction
            product_category = None
            if 'category' in column_mapping and column_mapping['category'] in row:
                category_raw = row.get(column_mapping['category'])
                if category_raw and not pd.isna(category_raw) and str(category_raw).lower() not in ['nan', 'null', 'none', '']:
                    product_category = str(category_raw).strip()
            
            return {
                "skip": False,
                "date": parsed_date,
                "product_name": product_name,
                "product_category": product_category,
                "quantity_sold": quantity,
                "unit_price": unit_price,
                "revenue": revenue,
                "location_id": location_id,
                "errors": errors,
                "warnings": warnings
            }
            
        except Exception as e:
            errors.append(f"Row {row_number}: Enhanced parsing error: {str(e)}")
            return {"skip": True, "errors": errors, "warnings": warnings}
    
    def _detect_columns(self, columns: List[str]) -> Dict[str, str]:
        """Enhanced column detection with fuzzy matching"""
        mapping = {}
        columns_lower = [col.lower().strip() for col in columns]
        
        for standard_name, possible_names in self.COLUMN_MAPPINGS.items():
            best_match = None
            best_score = 0
            
            for col_idx, col in enumerate(columns_lower):
                for possible in possible_names:
                    # Exact match (highest priority)
                    if possible == col:
                        best_match = columns[col_idx]
                        best_score = 100
                        break
                    # Contains match
                    elif possible in col or col in possible:
                        score = len(possible) / len(col) * 90
                        if score > best_score:
                            best_match = columns[col_idx]
                            best_score = score
                
                if best_score == 100:  # Found exact match
                    break
            
            if best_match and best_score > 70:  # Threshold for matches
                mapping[standard_name] = best_match
        
        # Enhanced alias mapping
        if 'product' not in mapping and 'product_name' in mapping:
            mapping['product'] = mapping['product_name']
        if 'quantity' not in mapping and 'quantity_sold' in mapping:
            mapping['quantity'] = mapping['quantity_sold']
        if 'location' not in mapping and 'location_id' in mapping:
            mapping['location'] = mapping['location_id']
        
        return mapping
    
    def _parse_date(self, date_str: str) -> Optional[datetime]:
        """Enhanced date parsing with explicit format handling for CSV dates"""
        if not date_str or str(date_str).lower() in ['nan', 'null', 'none']:
            return None
        
        date_str = str(date_str).strip()
        
        # For CSV format like "2024/10/01", try specific formats first to avoid ambiguity
        # Priority order: YYYY/MM/DD (most likely for machine-generated data)
        priority_formats = [
            '%Y/%m/%d',     # 2024/10/01 (October 1, 2024) - most likely for CSV exports
            '%Y-%m-%d',     # 2024-10-01 
            '%d/%m/%Y',     # 01/10/2024 (European format)
            '%m/%d/%Y',     # 10/01/2024 (US format)
        ]
        
        # Try priority formats first
        for fmt in priority_formats:
            try:
                parsed_dt = datetime.strptime(date_str, fmt)
                if parsed_dt.tzinfo is None:
                    parsed_dt = parsed_dt.replace(tzinfo=timezone.utc)
                logger.debug(f"Successfully parsed date '{date_str}' using format '{fmt}' -> {parsed_dt}")
                return parsed_dt
            except ValueError:
                continue
        
        # Try pandas as fallback with explicit format inference
        try:
            # For YYYY/MM/DD format, disable dayfirst to prevent misinterpretation
            if '/' in date_str and len(date_str.split('/')[0]) == 4:
                # Looks like YYYY/MM/DD format, so don't use dayfirst
                parsed_dt = pd.to_datetime(date_str, dayfirst=False)
            else:
                # For other formats, use dayfirst=True for European-style dates
                parsed_dt = pd.to_datetime(date_str, dayfirst=True)
                
            if hasattr(parsed_dt, 'to_pydatetime'):
                parsed_dt = parsed_dt.to_pydatetime()
            
            if parsed_dt.tzinfo is None:
                parsed_dt = parsed_dt.replace(tzinfo=timezone.utc)
            
            logger.debug(f"Successfully parsed date '{date_str}' using pandas -> {parsed_dt}")
            return parsed_dt
        except Exception as e:
            logger.debug(f"Pandas date parsing failed for '{date_str}': {e}")
            pass
        
        # Try remaining formats as last fallback
        for fmt in self.DATE_FORMATS:
            if fmt not in priority_formats:  # Skip already tried formats
                try:
                    parsed_dt = datetime.strptime(date_str, fmt)
                    if parsed_dt.tzinfo is None:
                        parsed_dt = parsed_dt.replace(tzinfo=timezone.utc)
                    logger.debug(f"Successfully parsed date '{date_str}' using fallback format '{fmt}' -> {parsed_dt}")
                    return parsed_dt
                except ValueError:
                    continue
        
        logger.warning(f"Could not parse date: {date_str}")
        return None
    
    def _clean_product_name(self, product_name: str) -> str:
        """Enhanced product name cleaning and standardization"""
        if not product_name:
            return "Producto sin nombre"
        
        # Remove extra whitespace
        cleaned = re.sub(r'\s+', ' ', str(product_name).strip())
        
        # Remove special characters but keep Spanish characters
        cleaned = re.sub(r'[^\w\s\-áéíóúñçüÁÉÍÓÚÑÇÜ]', '', cleaned)
        
        # Capitalize first letter of each word
        cleaned = cleaned.title()
        
        # Enhanced corrections for Spanish bakeries
        replacements = {
            'Pan De': 'Pan de',
            'Café Con': 'Café con',
            'Te ': 'Té ',
            'Bocadillo De': 'Bocadillo de',
            'Dulce De': 'Dulce de',
            'Tarta De': 'Tarta de',
        }
        
        for old, new in replacements.items():
            cleaned = cleaned.replace(old, new)
        
        return cleaned if cleaned else "Producto sin nombre"
    
    def _clear_import_cache(self):
        """Clear the product resolution cache for a new import session"""
        self.product_cache.clear()
        self.failed_products.clear()
        logger.info("Import cache cleared for new session")
    
    async def _resolve_product_to_inventory_id(self, product_name: str, product_category: Optional[str], tenant_id: UUID) -> Optional[UUID]:
        """Resolve a product name to an inventory_product_id via the inventory service with improved error handling and fallback"""
        
        # Check cache first
        if product_name in self.product_cache:
            logger.debug("Product resolved from cache", product_name=product_name, tenant_id=tenant_id)
            return self.product_cache[product_name]
        
        # Skip if this product already failed to resolve after all attempts
        if product_name in self.failed_products:
            logger.debug("Skipping previously failed product", product_name=product_name, tenant_id=tenant_id)
            return None
        
        max_retries = 5  # Increased retries
        base_delay = 2.0  # Increased base delay
        fallback_retry_delay = 10.0  # Longer delay for fallback attempts
        
        for attempt in range(max_retries):
            try:
                # Add progressive delay to avoid rate limiting
                if attempt > 0:
                    # Use longer delays for later attempts
                    if attempt >= 3:
                        delay = fallback_retry_delay  # Use fallback delay for later attempts
                    else:
                        delay = base_delay * (2 ** (attempt - 1))  # Exponential backoff
                    
                    logger.info(f"Retrying product resolution after {delay}s delay", 
                              product_name=product_name, attempt=attempt, tenant_id=tenant_id)
                    await asyncio.sleep(delay)
                
                # First try to search for existing product by name
                try:
                    products = await self.inventory_client.search_products(product_name, tenant_id)
                    
                    if products:
                        # Return the first matching product's ID
                        product_id = products[0].get('id')
                        if product_id:
                            uuid_id = UUID(str(product_id))
                            self.product_cache[product_name] = uuid_id  # Cache for future use
                            logger.info("Resolved product to existing inventory ID", 
                                      product_name=product_name, product_id=product_id, tenant_id=tenant_id)
                            return uuid_id
                except Exception as search_error:
                    logger.warning("Product search failed, trying direct creation", 
                                 product_name=product_name, error=str(search_error), tenant_id=tenant_id)
                
                # Add delay before creation attempt to avoid hitting rate limits
                await asyncio.sleep(1.0)
                
                # If not found or search failed, create a new ingredient/product in inventory
                ingredient_data = {
                    'name': product_name,
                    'type': 'finished_product',  # Assuming sales are of finished products
                    'unit': 'unit',  # Default unit
                    'current_stock': 0,  # No stock initially
                    'reorder_point': 0,
                    'cost_per_unit': 0,
                    'category': product_category or 'general'
                }
                
                try:
                    created_product = await self.inventory_client.create_ingredient(ingredient_data, str(tenant_id))
                    if created_product and created_product.get('id'):
                        product_id = created_product['id']
                        uuid_id = UUID(str(product_id))
                        self.product_cache[product_name] = uuid_id  # Cache for future use
                        logger.info("Created new inventory product for sales data", 
                                  product_name=product_name, product_id=product_id, tenant_id=tenant_id)
                        return uuid_id
                except Exception as creation_error:
                    logger.warning("Product creation failed", 
                                 product_name=product_name, error=str(creation_error), tenant_id=tenant_id)
                
                logger.warning("Failed to resolve or create product in inventory", 
                             product_name=product_name, tenant_id=tenant_id, attempt=attempt)
                             
            except Exception as e:
                error_str = str(e)
                if "429" in error_str or "rate limit" in error_str.lower() or "too many requests" in error_str.lower():
                    logger.warning("Rate limit or service overload detected, retrying with longer delay", 
                                 product_name=product_name, attempt=attempt, error=error_str, tenant_id=tenant_id)
                    if attempt < max_retries - 1:
                        continue  # Retry with exponential backoff
                elif "503" in error_str or "502" in error_str or "service unavailable" in error_str.lower():
                    logger.warning("Service unavailable, retrying with backoff", 
                                 product_name=product_name, attempt=attempt, error=error_str, tenant_id=tenant_id)
                    if attempt < max_retries - 1:
                        continue  # Retry for service unavailable errors
                elif "timeout" in error_str.lower() or "connection" in error_str.lower():
                    logger.warning("Network issue detected, retrying", 
                                 product_name=product_name, attempt=attempt, error=error_str, tenant_id=tenant_id)
                    if attempt < max_retries - 1:
                        continue  # Retry for network issues
                else:
                    logger.error("Non-retryable error resolving product to inventory ID", 
                               error=error_str, product_name=product_name, tenant_id=tenant_id)
                    if attempt < max_retries - 1:
                        # Still retry even for other errors, in case it's transient
                        continue
                    else:
                        break  # Don't retry on final attempt
        
        # If all retries failed, log detailed error but don't mark as permanently failed yet
        # Instead, we'll implement a fallback mechanism
        logger.error("Failed to resolve product after all retries, attempting fallback", 
                   product_name=product_name, tenant_id=tenant_id)
        
        # FALLBACK: Try to create a temporary product with minimal data
        try:
            # Use a simplified approach with minimal data
            fallback_data = {
                'name': product_name,
                'type': 'finished_product',
                'unit': 'unit',
                'current_stock': 0,
                'cost_per_unit': 0
            }
            
            logger.info("Attempting fallback product creation with minimal data", 
                      product_name=product_name, tenant_id=tenant_id)
            
            created_product = await self.inventory_client.create_ingredient(fallback_data, str(tenant_id))
            if created_product and created_product.get('id'):
                product_id = created_product['id']
                uuid_id = UUID(str(product_id))
                self.product_cache[product_name] = uuid_id
                logger.info("SUCCESS: Fallback product creation succeeded", 
                          product_name=product_name, product_id=product_id, tenant_id=tenant_id)
                return uuid_id
        except Exception as fallback_error:
            logger.error("Fallback product creation also failed", 
                       product_name=product_name, error=str(fallback_error), tenant_id=tenant_id)
        
        # Only mark as permanently failed after all attempts including fallback
        self.failed_products.add(product_name)
        logger.error("CRITICAL: Permanently failed to resolve product - this will result in missing training data", 
                   product_name=product_name, tenant_id=tenant_id)
        return None
    
    async def _batch_resolve_products(self, unique_products: set, tenant_id: str) -> None:
        """Batch resolve/create products to reduce API calls and improve success rate"""
        
        if not unique_products:
            return
        
        logger.info(f"Starting batch product resolution for {len(unique_products)} unique products")
        
        # Convert set to list for easier handling
        products_list = list(unique_products)
        batch_size = 5  # Process in smaller batches to avoid overwhelming the inventory service
        
        for i in range(0, len(products_list), batch_size):
            batch = products_list[i:i + batch_size]
            logger.info(f"Processing batch {i//batch_size + 1}/{(len(products_list) + batch_size - 1)//batch_size}")
            
            # Process each product in the batch with retry logic
            for product_name, product_category in batch:
                try:
                    # Skip if already in cache or failed list
                    if product_name in self.product_cache or product_name in self.failed_products:
                        continue
                    
                    # Try to resolve the product
                    await self._resolve_product_to_inventory_id(product_name, product_category, tenant_id)
                    
                    # Add small delay between products to be gentle on the API
                    await asyncio.sleep(0.5)
                    
                except Exception as e:
                    logger.warning(f"Failed to batch process product {product_name}: {e}")
                    continue
            
            # Add delay between batches
            if i + batch_size < len(products_list):
                logger.info("Waiting between batches to avoid rate limiting...")
                await asyncio.sleep(2.0)
        
        successful_resolutions = len([p for p, _ in products_list if p in self.product_cache])
        failed_resolutions = len([p for p, _ in products_list if p in self.failed_products])
        
        logger.info(f"Batch product resolution completed: {successful_resolutions} successful, {failed_resolutions} failed")
        
        if failed_resolutions > 0:
            logger.warning(f"ATTENTION: {failed_resolutions} products failed to resolve - these will be missing from training data")
            
        return
    
    def _structure_messages(self, messages: List[Union[str, Dict]]) -> List[Dict[str, Any]]:
        """Convert string messages to structured format"""
        structured = []
        for msg in messages:
            if isinstance(msg, str):
                structured.append({
                    "type": "general_message",
                    "message": msg,
                    "field": None,
                    "row": None,
                    "code": "GENERAL_MESSAGE"
                })
            else:
                structured.append(msg)
        return structured
    
    def _generate_suggestions(
        self, 
        validation_result: SalesValidationResult, 
        format_type: str, 
        warning_count: int
    ) -> List[str]:
        """Generate enhanced contextual suggestions"""
        suggestions = []
        
        if validation_result.is_valid:
            suggestions.append("El archivo está listo para procesamiento")
            suggestions.append(f"Se procesarán aproximadamente {validation_result.total_records} registros")
            
            if validation_result.total_records > 1000:
                suggestions.append("Archivo grande: el procesamiento puede tomar varios minutos")
                suggestions.append("Considera dividir archivos muy grandes para mejor rendimiento")
            
            if warning_count > 0:
                suggestions.append("Revisa las advertencias antes de continuar")
                suggestions.append("Los datos con advertencias se procesarán con valores por defecto")
                
            # Format-specific suggestions
            if format_type == "csv":
                suggestions.append("Asegúrate de que las fechas estén en formato DD/MM/YYYY")
                suggestions.append("Verifica que los números usen punto decimal (no coma)")
            elif format_type in ["excel", "xlsx"]:
                suggestions.append("Solo se procesará la primera hoja del archivo Excel")
                suggestions.append("Evita celdas combinadas y fórmulas complejas")
        else:
            suggestions.append("Corrige los errores antes de continuar")
            suggestions.append("Verifica que el archivo tenga el formato correcto")
            
            if format_type not in ["csv", "excel", "xlsx", "json"]:
                suggestions.append("Usa formato CSV o Excel para mejores resultados")
                suggestions.append("El formato JSON es para usuarios avanzados")
            
            if validation_result.total_records == 0:
                suggestions.append("Asegúrate de que el archivo contenga datos")
                suggestions.append("Verifica que el archivo no esté corrupto")
                
            # Missing column suggestions
            error_codes = [error.get("code", "") for error in validation_result.errors if isinstance(error, dict)]
            if "MISSING_DATE_COLUMN" in error_codes:
                suggestions.append("Incluye una columna de fecha (fecha, date, dia)")
            if "MISSING_PRODUCT_COLUMN" in error_codes:
                suggestions.append("Incluye una columna de producto (producto, product, item)")
        
        return suggestions


# Main DataImportService class with enhanced functionality