From 7556a00db7f9437d98d380262b8a67772431c9e3 Mon Sep 17 00:00:00 2001 From: Urtzi Alfaro Date: Sun, 12 Oct 2025 18:47:33 +0200 Subject: [PATCH] Improve the demo feature of the project --- DEMO_ARCHITECTURE.md | 499 ---- DEMO_IMPLEMENTATION_SUMMARY.md | 584 ---- DEPLOYMENT_COMMANDS.md | 322 -- EXTERNAL_DATA_REDESIGN_IMPLEMENTATION.md | 141 - EXTERNAL_DATA_SERVICE_REDESIGN.md | 2660 ----------------- FRONTEND_ALIGNMENT_STRATEGY.md | 757 ----- FRONTEND_INTEGRATION_GUIDE.md | 748 ----- IMPLEMENTATION_SUMMARY.md | 308 -- MIGRATION_SCRIPTS_README.md | 451 --- MODEL_STORAGE_FIX.md | 167 -- PROCUREMENT_IMPLEMENTATION_SUMMARY.md | 591 ---- SERVICE_INITIALIZATION_ARCHITECTURE.md | 532 ---- SSE_IMPLEMENTATION_COMPLETE.md | 363 --- SSE_SECURITY_MITIGATIONS.md | 291 -- TIMEZONE_AWARE_DATETIME_FIX.md | 234 -- Tiltfile | 64 +- WEBSOCKET_CLEAN_IMPLEMENTATION_STATUS.md | 215 -- WEBSOCKET_IMPLEMENTATION_COMPLETE.md | 278 -- bakery-ia-ca.crt | 14 +- .../docker-compose.yml | 0 docs/FRONTEND.md | 223 -- docs/IMPLEMENTATION_SUMMARY.md | 567 ---- docs/MVP_GAP_ANALYSIS_REPORT.md | 216 -- docs/ONBOARDING_AUTOMATION_IMPLEMENTATION.md | 324 -- docs/PRODUCTION_PLANNING_SYSTEM.md | 718 ----- docs/ROADMAP.md | 172 -- docs/SCHEDULER_QUICKSTART.md | 414 --- docs/SCHEDULER_RUNBOOK.md | 530 ---- frontend/package-lock.json | 7 + frontend/package.json | 1 + frontend/src/api/client/apiClient.ts | 4 + frontend/src/api/services/demo.ts | 95 +- .../src/components/demo/DemoErrorScreen.tsx | 105 + .../components/demo/DemoProgressIndicator.tsx | 132 + .../components/layout/AppShell/AppShell.tsx | 6 +- .../layout/DemoBanner/DemoBanner.tsx | 227 +- .../src/components/layout/Header/Header.tsx | 22 +- .../src/components/layout/Sidebar/Sidebar.tsx | 12 +- .../src/features/demo-onboarding/README.md | 209 ++ .../demo-onboarding/config/driver-config.ts | 41 + .../demo-onboarding/config/tour-steps.ts | 176 ++ .../demo-onboarding/hooks/useDemoTour.ts | 170 ++ .../src/features/demo-onboarding/index.ts | 4 + .../src/features/demo-onboarding/styles.css | 179 ++ .../src/features/demo-onboarding/types.ts | 26 + .../demo-onboarding/utils/tour-analytics.ts | 40 + .../demo-onboarding/utils/tour-state.ts | 84 + frontend/src/pages/app/DashboardPage.tsx | 66 +- .../operations/inventory/InventoryPage.tsx | 5 + .../subscription/SubscriptionPage.tsx | 34 +- frontend/src/pages/public/DemoPage.tsx | 38 +- frontend/src/pages/public/DemoSetupPage.tsx | 236 ++ frontend/src/router/AppRouter.tsx | 2 + frontend/src/stores/useTenantInitializer.ts | 31 +- gateway/app/middleware/auth.py | 6 +- gateway/app/middleware/demo_middleware.py | 30 +- gateway/app/routes/user.py | 3 +- .../base/jobs/demo-clone-job-template.yaml | 55 - .../base/jobs/demo-seed-recipes-job.yaml | 63 + .../base/jobs/demo-seed-sales-job.yaml | 63 + .../jobs/demo-seed-subscriptions-job.yaml | 56 + .../base/jobs/demo-seed-suppliers-job.yaml | 63 + .../kubernetes/base/kustomization.yaml | 4 + .../alert-processor-migration-job.yaml | 2 +- .../base/migrations/auth-migration-job.yaml | 2 +- .../demo-session-migration-job.yaml | 2 +- .../migrations/external-migration-job.yaml | 2 +- .../migrations/forecasting-migration-job.yaml | 2 +- .../migrations/inventory-migration-job.yaml | 2 +- .../notification-migration-job.yaml | 2 +- .../base/migrations/orders-migration-job.yaml | 2 +- .../base/migrations/pos-migration-job.yaml | 2 +- .../migrations/production-migration-job.yaml | 2 +- .../migrations/recipes-migration-job.yaml | 2 +- .../base/migrations/sales-migration-job.yaml | 2 +- .../migrations/suppliers-migration-job.yaml | 2 +- .../base/migrations/tenant-migration-job.yaml | 2 +- .../migrations/training-migration-job.yaml | 2 +- .../secrets/demo-internal-api-key-secret.yaml | 10 + .../cleanup_databases_k8s.sh | 0 .../complete-cleanup.sh | 0 scripts/demo/__init__.py | 1 - scripts/demo/clone_demo_tenant.py | 234 -- scripts/demo/seed_demo_ai_models.py | 278 -- scripts/demo/seed_demo_inventory.py | 338 --- scripts/demo/seed_demo_tenants.py | 144 - {services/auth => scripts}/docker-compose.yml | 0 scripts/manual_seed_demo.py | 49 - .../regenerate_all_migrations.sh | 0 .../regenerate_migrations_k8s.sh | 0 setup-https.sh => scripts/setup-https.sh | 0 services/alert_processor/Dockerfile | 3 +- services/auth/Dockerfile | 3 +- .../auth/scripts}/demo/seed_demo_users.py | 0 .../demo_session/app/api/demo_sessions.py | 132 +- services/demo_session/app/core/config.py | 6 +- services/demo_session/app/models/__init__.py | 4 +- .../demo_session/app/models/demo_session.py | 32 +- .../app/services/cleanup_service.py | 55 +- .../app/services/clone_orchestrator.py | 330 ++ .../app/services/k8s_job_cloner.py | 166 - .../app/services/session_manager.py | 185 +- .../002_add_cloning_status_tracking.py | 81 + services/external/Dockerfile | 3 +- services/external/IMPLEMENTATION_COMPLETE.md | 477 --- services/forecasting/Dockerfile | 3 +- services/forecasting/app/api/internal_demo.py | 221 ++ services/forecasting/app/main.py | 3 +- services/inventory/Dockerfile | 3 +- services/inventory/app/api/internal_demo.py | 182 ++ services/inventory/app/main.py | 4 +- services/inventory/app/schemas/inventory.py | 14 +- .../scripts/demo/ingredientes_es.json | 449 +++ .../scripts/demo/seed_demo_inventory.py | 325 ++ services/notification/Dockerfile | 3 +- services/orders/Dockerfile | 3 +- services/orders/app/api/customers.py | 68 +- services/orders/app/api/internal_demo.py | 352 +++ services/orders/app/api/orders.py | 80 +- services/orders/app/main.py | 8 +- services/orders/scripts/demo/clientes_es.json | 229 ++ services/pos/Dockerfile | 3 +- services/production/Dockerfile | 3 +- services/production/app/api/internal_demo.py | 462 +++ .../production/app/api/quality_templates.py | 490 +++ services/production/app/main.py | 7 +- .../test_transformation_integration.py | 246 -- services/production/verify_integration.py | 221 -- services/recipes/Dockerfile | 3 +- services/recipes/app/api/internal_demo.py | 377 +++ services/recipes/app/main.py | 3 +- services/recipes/app/models/recipes.py | 20 +- .../app/repositories/recipe_repository.py | 17 +- services/recipes/scripts/demo/recetas_es.json | 447 +++ .../recipes/scripts/demo/seed_demo_recipes.py | 387 +++ services/sales/Dockerfile | 3 +- services/sales/app/api/internal_demo.py | 188 ++ services/sales/app/main.py | 5 +- .../sales/scripts/demo/seed_demo_sales.py | 360 +++ services/suppliers/Dockerfile | 3 +- services/suppliers/app/api/internal_demo.py | 539 ++++ services/suppliers/app/main.py | 3 +- .../scripts/demo/proveedores_es.json | 367 +++ .../scripts/demo/seed_demo_suppliers.py | 430 +++ services/tenant/Dockerfile | 3 +- services/tenant/app/api/internal_demo.py | 308 ++ services/tenant/app/main.py | 3 +- services/tenant/app/models/tenants.py | 5 +- .../services/subscription_limit_service.py | 33 +- ...c00c1244_add_metadata_column_to_tenants.py | 28 + .../scripts/demo/seed_demo_subscriptions.py | 235 ++ .../tenant/scripts/demo/seed_demo_tenants.py | 263 ++ .../COMPLETE_IMPLEMENTATION_REPORT.md | 645 ---- services/training/DEVELOPER_GUIDE.md | 230 -- services/training/Dockerfile | 3 +- services/training/IMPLEMENTATION_SUMMARY.md | 274 -- services/training/PHASE_2_ENHANCEMENTS.md | 540 ---- .../scripts/demo/seed_demo_ai_models.py | 271 ++ shared/database/README.md | 123 - shared/database/demo_inventory_data.sql | 659 ---- shared/routing/route_builder.py | 14 +- {scripts => shared/scripts}/run_migrations.py | 0 tests/test_alert_quick.sh | 61 - tests/test_alert_working.py | 137 - tests/test_onboarding_flow.sh | 1084 ------- tests/test_out_of_stock_alert.py | 94 - tests/test_recommendation_alert.py | 95 - tests/test_traffic_storage.py | 93 - 168 files changed, 10102 insertions(+), 18869 deletions(-) delete mode 100644 DEMO_ARCHITECTURE.md delete mode 100644 DEMO_IMPLEMENTATION_SUMMARY.md delete mode 100644 DEPLOYMENT_COMMANDS.md delete mode 100644 EXTERNAL_DATA_REDESIGN_IMPLEMENTATION.md delete mode 100644 EXTERNAL_DATA_SERVICE_REDESIGN.md delete mode 100644 FRONTEND_ALIGNMENT_STRATEGY.md delete mode 100644 FRONTEND_INTEGRATION_GUIDE.md delete mode 100644 IMPLEMENTATION_SUMMARY.md delete mode 100644 MIGRATION_SCRIPTS_README.md delete mode 100644 MODEL_STORAGE_FIX.md delete mode 100644 PROCUREMENT_IMPLEMENTATION_SUMMARY.md delete mode 100644 SERVICE_INITIALIZATION_ARCHITECTURE.md delete mode 100644 SSE_IMPLEMENTATION_COMPLETE.md delete mode 100644 SSE_SECURITY_MITIGATIONS.md delete mode 100644 TIMEZONE_AWARE_DATETIME_FIX.md delete mode 100644 WEBSOCKET_CLEAN_IMPLEMENTATION_STATUS.md delete mode 100644 WEBSOCKET_IMPLEMENTATION_COMPLETE.md rename docker-compose.yml => config/docker-compose.yml (100%) delete mode 100644 docs/FRONTEND.md delete mode 100644 docs/IMPLEMENTATION_SUMMARY.md delete mode 100644 docs/MVP_GAP_ANALYSIS_REPORT.md delete mode 100644 docs/ONBOARDING_AUTOMATION_IMPLEMENTATION.md delete mode 100644 docs/PRODUCTION_PLANNING_SYSTEM.md delete mode 100644 docs/ROADMAP.md delete mode 100644 docs/SCHEDULER_QUICKSTART.md delete mode 100644 docs/SCHEDULER_RUNBOOK.md create mode 100644 frontend/src/components/demo/DemoErrorScreen.tsx create mode 100644 frontend/src/components/demo/DemoProgressIndicator.tsx create mode 100644 frontend/src/features/demo-onboarding/README.md create mode 100644 frontend/src/features/demo-onboarding/config/driver-config.ts create mode 100644 frontend/src/features/demo-onboarding/config/tour-steps.ts create mode 100644 frontend/src/features/demo-onboarding/hooks/useDemoTour.ts create mode 100644 frontend/src/features/demo-onboarding/index.ts create mode 100644 frontend/src/features/demo-onboarding/styles.css create mode 100644 frontend/src/features/demo-onboarding/types.ts create mode 100644 frontend/src/features/demo-onboarding/utils/tour-analytics.ts create mode 100644 frontend/src/features/demo-onboarding/utils/tour-state.ts create mode 100644 frontend/src/pages/public/DemoSetupPage.tsx delete mode 100644 infrastructure/kubernetes/base/jobs/demo-clone-job-template.yaml create mode 100644 infrastructure/kubernetes/base/jobs/demo-seed-recipes-job.yaml create mode 100644 infrastructure/kubernetes/base/jobs/demo-seed-sales-job.yaml create mode 100644 infrastructure/kubernetes/base/jobs/demo-seed-subscriptions-job.yaml create mode 100644 infrastructure/kubernetes/base/jobs/demo-seed-suppliers-job.yaml create mode 100644 infrastructure/kubernetes/base/secrets/demo-internal-api-key-secret.yaml rename cleanup_databases_k8s.sh => scripts/cleanup_databases_k8s.sh (100%) rename complete-cleanup.sh => scripts/complete-cleanup.sh (100%) delete mode 100644 scripts/demo/__init__.py delete mode 100644 scripts/demo/clone_demo_tenant.py delete mode 100644 scripts/demo/seed_demo_ai_models.py delete mode 100644 scripts/demo/seed_demo_inventory.py delete mode 100644 scripts/demo/seed_demo_tenants.py rename {services/auth => scripts}/docker-compose.yml (100%) delete mode 100644 scripts/manual_seed_demo.py rename regenerate_all_migrations.sh => scripts/regenerate_all_migrations.sh (100%) rename regenerate_migrations_k8s.sh => scripts/regenerate_migrations_k8s.sh (100%) rename setup-https.sh => scripts/setup-https.sh (100%) rename {scripts => services/auth/scripts}/demo/seed_demo_users.py (100%) create mode 100644 services/demo_session/app/services/clone_orchestrator.py delete mode 100644 services/demo_session/app/services/k8s_job_cloner.py create mode 100644 services/demo_session/migrations/versions/002_add_cloning_status_tracking.py delete mode 100644 services/external/IMPLEMENTATION_COMPLETE.md create mode 100644 services/forecasting/app/api/internal_demo.py create mode 100644 services/inventory/app/api/internal_demo.py create mode 100644 services/inventory/scripts/demo/ingredientes_es.json create mode 100644 services/inventory/scripts/demo/seed_demo_inventory.py create mode 100644 services/orders/app/api/internal_demo.py create mode 100644 services/orders/scripts/demo/clientes_es.json create mode 100644 services/production/app/api/internal_demo.py create mode 100644 services/production/app/api/quality_templates.py delete mode 100644 services/production/test_transformation_integration.py delete mode 100644 services/production/verify_integration.py create mode 100644 services/recipes/app/api/internal_demo.py create mode 100644 services/recipes/scripts/demo/recetas_es.json create mode 100755 services/recipes/scripts/demo/seed_demo_recipes.py create mode 100644 services/sales/app/api/internal_demo.py create mode 100755 services/sales/scripts/demo/seed_demo_sales.py create mode 100644 services/suppliers/app/api/internal_demo.py create mode 100644 services/suppliers/scripts/demo/proveedores_es.json create mode 100755 services/suppliers/scripts/demo/seed_demo_suppliers.py create mode 100644 services/tenant/app/api/internal_demo.py create mode 100644 services/tenant/migrations/versions/20251011_1247_865dc00c1244_add_metadata_column_to_tenants.py create mode 100755 services/tenant/scripts/demo/seed_demo_subscriptions.py create mode 100755 services/tenant/scripts/demo/seed_demo_tenants.py delete mode 100644 services/training/COMPLETE_IMPLEMENTATION_REPORT.md delete mode 100644 services/training/DEVELOPER_GUIDE.md delete mode 100644 services/training/IMPLEMENTATION_SUMMARY.md delete mode 100644 services/training/PHASE_2_ENHANCEMENTS.md create mode 100644 services/training/scripts/demo/seed_demo_ai_models.py delete mode 100644 shared/database/README.md delete mode 100644 shared/database/demo_inventory_data.sql rename {scripts => shared/scripts}/run_migrations.py (100%) delete mode 100755 tests/test_alert_quick.sh delete mode 100644 tests/test_alert_working.py delete mode 100755 tests/test_onboarding_flow.sh delete mode 100644 tests/test_out_of_stock_alert.py delete mode 100644 tests/test_recommendation_alert.py delete mode 100644 tests/test_traffic_storage.py diff --git a/DEMO_ARCHITECTURE.md b/DEMO_ARCHITECTURE.md deleted file mode 100644 index 328be31b..00000000 --- a/DEMO_ARCHITECTURE.md +++ /dev/null @@ -1,499 +0,0 @@ -# Demo Architecture - Production Demo System - -## Overview - -This document describes the complete demo architecture for providing prospects with isolated, ephemeral demo sessions to explore the Bakery IA platform. - -## Key Features - -- ✅ **Session Isolation**: Each prospect gets their own isolated copy of demo data -- ✅ **Spanish Content**: All demo data in Spanish for the Spanish market -- ✅ **Two Business Models**: Individual bakery and central baker satellite -- ✅ **Automatic Cleanup**: Sessions automatically expire after 30 minutes -- ✅ **Read-Mostly Access**: Prospects can explore but critical operations are restricted -- ✅ **Production Ready**: Scalable to 200+ concurrent demo sessions - -## Architecture Components - -### 1. Demo Session Service - -**Location**: `services/demo_session/` - -**Responsibilities**: -- Create isolated demo sessions -- Manage session lifecycle (create, extend, destroy) -- Clone base demo data to virtual tenants -- Track session metrics and activity - -**Key Endpoints**: -``` -GET /api/demo/accounts # Get public demo account info -POST /api/demo/session/create # Create new demo session -POST /api/demo/session/extend # Extend session expiration -POST /api/demo/session/destroy # Destroy session -GET /api/demo/session/{id} # Get session info -GET /api/demo/stats # Get usage statistics -``` - -### 2. Demo Data Seeding - -**Location**: `scripts/demo/` - -**Scripts**: -- `seed_demo_users.py` - Creates demo user accounts -- `seed_demo_tenants.py` - Creates base demo tenants (templates) -- `seed_demo_inventory.py` - Populates inventory with Spanish data (25 ingredients per template) -- `clone_demo_tenant.py` - Clones data from base template to virtual tenant (runs as K8s Job) - -**Demo Accounts**: - -#### Individual Bakery (Panadería San Pablo) -``` -Email: demo.individual@panaderiasanpablo.com -Password: DemoSanPablo2024! -Business Model: Producción Local -Features: Production, Recipes, Inventory, Forecasting, POS, Sales -``` - -#### Central Baker Satellite (Panadería La Espiga) -``` -Email: demo.central@panaderialaespiga.com -Password: DemoLaEspiga2024! -Business Model: Obrador Central + Punto de Venta -Features: Suppliers, Inventory, Orders, POS, Sales, Forecasting -``` - -### 3. Gateway Middleware - -**Location**: `gateway/app/middleware/demo_middleware.py` - -**Responsibilities**: -- Intercept requests with demo session IDs -- Inject virtual tenant ID -- Enforce operation restrictions -- Track session activity - -**Allowed Operations**: -```python -# Read - All allowed -GET, HEAD, OPTIONS: * - -# Limited Write - Realistic testing -POST: /api/pos/sales, /api/orders, /api/inventory/adjustments -PUT: /api/pos/sales/*, /api/orders/* - -# Blocked -DELETE: All (read-only for destructive operations) -``` - -### 4. Redis Cache Layer - -**Purpose**: Store frequently accessed demo session data - -**Data Cached**: -- Session metadata -- Inventory summaries -- POS session data -- Recent sales - -**TTL**: 30 minutes (auto-cleanup) - -### 5. Kubernetes Resources - -**Databases**: -- `demo-session-db` - Tracks session records - -**Services**: -- `demo-session-service` - Main demo service (2 replicas) - -**Jobs** (Initialization): -- `demo-seed-users` - Creates demo users -- `demo-seed-tenants` - Creates demo tenant templates -- `demo-seed-inventory` - Populates inventory data (25 ingredients per tenant) - -**Dynamic Jobs** (Runtime): -- `demo-clone-{virtual_tenant_id}` - Created per session to clone data from template - -**CronJob** (Maintenance): -- `demo-session-cleanup` - Runs hourly to cleanup expired sessions - -**RBAC**: -- `demo-session-sa` - ServiceAccount for demo-session-service -- `demo-session-job-creator` - Role allowing job creation and pod management -- `demo-seed-role` - Role for seed jobs to access databases - -## Data Flow - -### Session Creation - -``` -1. User clicks "Probar Demo" on website - ↓ -2. Frontend calls POST /api/demo/session/create - { - "demo_account_type": "individual_bakery" - } - ↓ -3. Demo Session Service: - - Generates unique session_id: "demo_abc123..." - - Creates virtual_tenant_id: UUID - - Stores session in database - - Returns session_token (JWT) - ↓ -4. Kubernetes Job Cloning (background): - - Demo service triggers K8s Job with clone script - - Job container uses CLONE_JOB_IMAGE (inventory-service image) - - Clones inventory data from base template tenant - - Uses ORM models for safe data copying - - Job runs with IfNotPresent pull policy (works in dev & prod) - ↓ -5. Frontend receives: - { - "session_id": "demo_abc123...", - "virtual_tenant_id": "uuid-here", - "expires_at": "2025-10-02T12:30:00Z", - "session_token": "eyJ..." - } - ↓ -6. Frontend stores session_token in cookie/localStorage - All subsequent requests include: - Header: X-Demo-Session-Id: demo_abc123... -``` - -### Request Handling - -``` -1. Request arrives at Gateway - ↓ -2. Demo Middleware checks: - - Is X-Demo-Session-Id present? - - Is session still active? - - Is operation allowed? - ↓ -3. If valid: - - Injects X-Tenant-Id: {virtual_tenant_id} - - Routes to appropriate service - ↓ -4. Service processes request: - - Reads/writes data for virtual tenant - - No knowledge of demo vs. real tenant - ↓ -5. Response returned to user -``` - -### Session Cleanup - -``` -Every hour (CronJob): - -1. Demo Cleanup Service queries: - SELECT * FROM demo_sessions - WHERE status = 'active' - AND expires_at < NOW() - ↓ -2. For each expired session: - - Mark as 'expired' - - Delete all virtual tenant data - - Delete Redis keys - - Update statistics - ↓ -3. Weekly cleanup: - DELETE FROM demo_sessions - WHERE status = 'destroyed' - AND destroyed_at < NOW() - INTERVAL '7 days' -``` - -## Database Schema - -### demo_sessions Table - -```sql -CREATE TABLE demo_sessions ( - id UUID PRIMARY KEY, - session_id VARCHAR(100) UNIQUE NOT NULL, - - -- Ownership - user_id UUID, - ip_address VARCHAR(45), - user_agent VARCHAR(500), - - -- Demo linking - base_demo_tenant_id UUID NOT NULL, - virtual_tenant_id UUID NOT NULL, - demo_account_type VARCHAR(50) NOT NULL, - - -- Lifecycle - status VARCHAR(20) NOT NULL, -- active, expired, destroyed - created_at TIMESTAMP WITH TIME ZONE NOT NULL, - expires_at TIMESTAMP WITH TIME ZONE NOT NULL, - last_activity_at TIMESTAMP WITH TIME ZONE, - destroyed_at TIMESTAMP WITH TIME ZONE, - - -- Metrics - request_count INTEGER DEFAULT 0, - data_cloned BOOLEAN DEFAULT FALSE, - redis_populated BOOLEAN DEFAULT FALSE, - - -- Metadata - metadata JSONB -); - -CREATE INDEX idx_session_id ON demo_sessions(session_id); -CREATE INDEX idx_virtual_tenant ON demo_sessions(virtual_tenant_id); -CREATE INDEX idx_status ON demo_sessions(status); -CREATE INDEX idx_expires_at ON demo_sessions(expires_at); -``` - -### tenants Table (Updated) - -```sql -ALTER TABLE tenants ADD COLUMN is_demo BOOLEAN DEFAULT FALSE; -ALTER TABLE tenants ADD COLUMN is_demo_template BOOLEAN DEFAULT FALSE; -ALTER TABLE tenants ADD COLUMN base_demo_tenant_id UUID; -ALTER TABLE tenants ADD COLUMN demo_session_id VARCHAR(100); -ALTER TABLE tenants ADD COLUMN demo_expires_at TIMESTAMP WITH TIME ZONE; - -CREATE INDEX idx_is_demo ON tenants(is_demo); -CREATE INDEX idx_demo_session ON tenants(demo_session_id); -``` - -## Deployment - -### Initial Deployment - -```bash -# 1. Deploy infrastructure (databases, redis, rabbitmq) -kubectl apply -k infrastructure/kubernetes/overlays/prod - -# 2. Run migrations -# (Automatically handled by migration jobs) - -# 3. Seed demo data -# (Automatically handled by demo-seed-* jobs) - -# 4. Verify demo system -kubectl get jobs -n bakery-ia | grep demo-seed -kubectl logs -f job/demo-seed-users -n bakery-ia -kubectl logs -f job/demo-seed-tenants -n bakery-ia -kubectl logs -f job/demo-seed-inventory -n bakery-ia - -# 5. Test demo session creation -curl -X POST http://your-domain/api/demo/session/create \ - -H "Content-Type: application/json" \ - -d '{"demo_account_type": "individual_bakery"}' -``` - -### Using Tilt (Local Development) - -```bash -# Start Tilt -tilt up - -# Demo resources in Tilt UI: -# - databases: demo-session-db -# - migrations: demo-session-migration -# - services: demo-session-service -# - demo-init: demo-seed-users, demo-seed-tenants, demo-seed-inventory -# - config: patch-demo-session-env (sets CLONE_JOB_IMAGE dynamically) - -# Tilt automatically: -# 1. Gets inventory-service image tag (e.g., tilt-abc123) -# 2. Patches demo-session-service with CLONE_JOB_IMAGE env var -# 3. Clone jobs use this image with IfNotPresent pull policy -``` - -## Monitoring - -### Key Metrics - -```python -# Session Statistics -GET /api/demo/stats - -{ - "total_sessions": 1250, - "active_sessions": 45, - "expired_sessions": 980, - "destroyed_sessions": 225, - "avg_duration_minutes": 18.5, - "total_requests": 125000 -} -``` - -### Health Checks - -```bash -# Demo Session Service -curl http://demo-session-service:8000/health - -# Check active sessions -kubectl exec -it deployment/demo-session-service -- \ - python -c "from app.services import *; print(get_active_sessions())" -``` - -### Logs - -```bash -# Demo session service logs -kubectl logs -f deployment/demo-session-service -n bakery-ia - -# Demo seed job logs -kubectl logs job/demo-seed-inventory -n bakery-ia - -# Cleanup cron job logs -kubectl logs -l app=demo-cleanup -n bakery-ia --tail=100 -``` - -## Scaling Considerations - -### Current Limits - -- **Concurrent Sessions**: ~200 (2 replicas × ~100 sessions each) -- **Redis Memory**: ~1-2 GB (10 MB per session × 200) -- **PostgreSQL**: ~5-10 GB (30 MB per virtual tenant × 200) -- **Session Duration**: 30 minutes (configurable) -- **Extensions**: Maximum 3 per session - -### Scaling Up - -```yaml -# Scale demo-session-service -kubectl scale deployment/demo-session-service --replicas=4 -n bakery-ia - -# Increase Redis memory (if needed) -# Edit redis deployment, increase memory limits - -# Adjust session settings -# Edit demo-session configmap: -DEMO_SESSION_DURATION_MINUTES: 45 # Increase session time -DEMO_SESSION_MAX_EXTENSIONS: 5 # Allow more extensions -``` - -## Security - -### Public Demo Credentials - -Demo credentials are **intentionally public** for prospect access: -- Published on marketing website -- Included in demo documentation -- Safe because sessions are isolated and ephemeral - -### Restrictions - -1. **No Destructive Operations**: DELETE blocked -2. **Limited Modifications**: Only realistic testing operations -3. **No Sensitive Data Access**: Cannot change passwords, billing, etc. -4. **Automatic Expiration**: Sessions auto-destroy after 30 minutes -5. **Rate Limiting**: Standard gateway rate limits apply -6. **No AI Training**: Forecast API blocked for demo accounts (no trained models) -7. **Scheduler Prevention**: Procurement scheduler filters out demo tenants - -### Data Privacy - -- No real customer data in demo tenants -- Session data automatically deleted -- Anonymized analytics only - -## Troubleshooting - -### Session Creation Fails - -```bash -# Check demo-session-service health -kubectl get pods -l app=demo-session-service -n bakery-ia - -# Check logs -kubectl logs deployment/demo-session-service -n bakery-ia --tail=50 - -# Verify base demo tenants exist -kubectl exec -it deployment/tenant-service -- \ - psql $TENANT_DATABASE_URL -c \ - "SELECT id, name, subdomain FROM tenants WHERE is_demo_template = true;" -``` - -### Sessions Not Cleaning Up - -```bash -# Check cleanup cronjob -kubectl get cronjobs -n bakery-ia -kubectl get jobs -l app=demo-cleanup -n bakery-ia - -# Manually trigger cleanup -kubectl create job --from=cronjob/demo-session-cleanup manual-cleanup-$(date +%s) -n bakery-ia - -# Check for orphaned sessions -kubectl exec -it deployment/demo-session-service -- \ - psql $DEMO_SESSION_DATABASE_URL -c \ - "SELECT status, COUNT(*) FROM demo_sessions GROUP BY status;" -``` - -### Redis Connection Issues - -```bash -# Test Redis connectivity -kubectl exec -it deployment/demo-session-service -- \ - python -c "import redis; r=redis.Redis(host='redis-service'); print(r.ping())" - -# Check Redis memory usage -kubectl exec -it deployment/redis -- redis-cli INFO memory -``` - -## Technical Implementation Details - -### Data Cloning Architecture - -**Choice: Kubernetes Job-based Cloning** (selected over service-based endpoints) - -**Why K8s Jobs**: -- Database-level operations (faster than API calls) -- Scalable (one job per session, isolated execution) -- No service-specific clone endpoints needed -- Works in both dev (Tilt) and production - -**How it Works**: -1. Demo-session-service creates K8s Job via K8s API -2. Job uses `CLONE_JOB_IMAGE` environment variable (configured image) -3. In **Dev (Tilt)**: `patch-demo-session-env` sets dynamic Tilt image tag -4. In **Production**: Deployment manifest has stable release tag -5. Job runs `clone_demo_tenant.py` with `imagePullPolicy: IfNotPresent` -6. Script uses ORM models to clone data safely - -**Environment-based Image Configuration**: -```yaml -# Demo-session deployment -env: -- name: CLONE_JOB_IMAGE - value: "bakery/inventory-service:latest" # Overridden by Tilt in dev - -# Tilt automatically patches this to match actual inventory-service tag -# e.g., bakery/inventory-service:tilt-abc123 -``` - -### AI Model Restrictions - -**Fake Models in Database**: -- Demo tenants have AI model records in database -- No actual model files (.pkl, .h5) stored -- Forecast API blocked at gateway level for demo accounts -- Returns user-friendly error message - -**Scheduler Prevention**: -- Procurement scheduler filters `is_demo = true` tenants -- Prevents automated procurement runs on demo data -- Manual procurement still allowed for realistic testing - -## Future Enhancements - -1. **Analytics Dashboard**: Track demo → paid conversion rates -2. **Guided Tours**: In-app tutorials for demo users -3. **Custom Demo Scenarios**: Let prospects choose specific features -4. **Demo Recordings**: Capture anonymized session recordings -5. **Multi-Region**: Deploy demo infrastructure in EU, US, LATAM -6. **Sales & Orders Cloning**: Extend clone script to copy sales and orders data - -## References - -- [Demo Session Service API](services/demo_session/README.md) -- [Demo Data Seeding](scripts/demo/README.md) -- [Gateway Middleware](gateway/app/middleware/README.md) -- [Kubernetes Manifests](infrastructure/kubernetes/base/components/demo-session/) diff --git a/DEMO_IMPLEMENTATION_SUMMARY.md b/DEMO_IMPLEMENTATION_SUMMARY.md deleted file mode 100644 index a02544e6..00000000 --- a/DEMO_IMPLEMENTATION_SUMMARY.md +++ /dev/null @@ -1,584 +0,0 @@ -# Demo Architecture Implementation Summary - -## ✅ Implementation Complete - -All components of the production demo system have been implemented. This document provides a summary of what was created and how to use it. - ---- - -## 📁 Files Created - -### Demo Session Service (New Microservice) - -``` -services/demo_session/ -├── app/ -│ ├── __init__.py -│ ├── main.py # FastAPI application -│ ├── api/ -│ │ ├── __init__.py -│ │ ├── routes.py # API endpoints -│ │ └── schemas.py # Pydantic models -│ ├── core/ -│ │ ├── __init__.py -│ │ ├── config.py # Settings -│ │ ├── database.py # Database manager -│ │ └── redis_client.py # Redis client -│ ├── models/ -│ │ ├── __init__.py -│ │ └── demo_session.py # Session model -│ └── services/ -│ ├── __init__.py -│ ├── session_manager.py # Session lifecycle -│ ├── data_cloner.py # Data cloning -│ └── cleanup_service.py # Cleanup logic -├── migrations/ -│ ├── env.py -│ ├── script.py.mako -│ └── versions/ -├── requirements.txt -├── Dockerfile -└── alembic.ini -``` - -### Demo Seeding Scripts - -``` -scripts/demo/ -├── __init__.py -├── seed_demo_users.py # Creates demo users -├── seed_demo_tenants.py # Creates demo tenants -├── seed_demo_inventory.py # Populates Spanish inventory (25 ingredients) -└── clone_demo_tenant.py # Clones data from template (runs as K8s Job) -``` - -### Gateway Middleware - -``` -gateway/app/middleware/ -└── demo_middleware.py # Demo session handling -``` - -### Kubernetes Resources - -``` -infrastructure/kubernetes/base/ -├── components/demo-session/ -│ ├── deployment.yaml # Service deployment (with CLONE_JOB_IMAGE env) -│ ├── service.yaml # K8s service -│ ├── database.yaml # PostgreSQL DB -│ └── rbac.yaml # RBAC for job creation -├── migrations/ -│ └── demo-session-migration-job.yaml # Migration job -├── jobs/ -│ ├── demo-seed-users-job.yaml # User seeding -│ ├── demo-seed-tenants-job.yaml # Tenant seeding -│ ├── demo-seed-inventory-job.yaml # Inventory seeding -│ ├── demo-seed-rbac.yaml # RBAC permissions for seed jobs -│ └── demo-clone-job-template.yaml # Reference template for clone jobs -└── cronjobs/ - └── demo-cleanup-cronjob.yaml # Hourly cleanup -``` - -### Documentation - -``` -DEMO_ARCHITECTURE.md # Complete architecture guide -DEMO_IMPLEMENTATION_SUMMARY.md # This file -``` - -### Updated Files - -``` -services/tenant/app/models/tenants.py # Added demo flags -services/demo_session/app/services/k8s_job_cloner.py # K8s Job cloning implementation -gateway/app/main.py # Added demo middleware -gateway/app/middleware/demo_middleware.py # Converted to BaseHTTPMiddleware -Tiltfile # Added demo resources + CLONE_JOB_IMAGE patching -shared/config/base.py # Added demo-related settings -``` - ---- - -## 🎯 Key Features Implemented - -### 1. Session Isolation ✅ -- Each prospect gets isolated virtual tenant -- No data interference between sessions -- Automatic resource cleanup - -### 2. Spanish Demo Data ✅ -- **Panadería San Pablo** (Individual Bakery) - - Raw ingredients: Harina, Levadura, Mantequilla, etc. - - Local production focus - - Full recipe management - -- **Panadería La Espiga** (Central Baker Satellite) - - Pre-baked products from central baker - - Supplier management - - Order tracking - -### 3. Redis Caching ✅ -- Hot data cached for fast access -- Automatic TTL (30 minutes) -- Session metadata storage - -### 4. Gateway Integration ✅ -- Demo session detection -- Operation restrictions -- Virtual tenant injection - -### 5. Automatic Cleanup ✅ -- Hourly CronJob cleanup -- Expired session detection -- Database and Redis cleanup - -### 6. K8s Job-based Data Cloning ✅ -- Database-level cloning (faster than API calls) -- Environment-based image configuration -- Works in dev (Tilt dynamic tags) and production (stable tags) -- Uses ORM models for safe data copying -- `imagePullPolicy: IfNotPresent` for local images - -### 7. AI & Scheduler Restrictions ✅ -- Fake AI models in database (no real files) -- Forecast API blocked at gateway for demo accounts -- Procurement scheduler filters out demo tenants -- Manual operations still allowed for realistic testing - ---- - -## 🚀 Quick Start - -### Local Development with Tilt - -```bash -# Start all services including demo system -tilt up - -# Watch demo initialization -tilt logs demo-seed-users -tilt logs demo-seed-tenants -tilt logs demo-seed-inventory - -# Check demo service -tilt logs demo-session-service -``` - -### Test Demo Session Creation - -```bash -# Get demo accounts info -curl http://localhost/api/demo/accounts | jq - -# Create demo session -curl -X POST http://localhost/api/demo/session/create \ - -H "Content-Type: application/json" \ - -d '{ - "demo_account_type": "individual_bakery", - "ip_address": "127.0.0.1" - }' | jq - -# Response: -# { -# "session_id": "demo_abc123...", -# "virtual_tenant_id": "uuid-here", -# "expires_at": "2025-10-02T12:30:00Z", -# "session_token": "eyJ..." -# } -``` - -### Use Demo Session - -```bash -# Make request with demo session -curl http://localhost/api/inventory/ingredients \ - -H "X-Demo-Session-Id: demo_abc123..." \ - -H "Content-Type: application/json" - -# Try restricted operation (should fail) -curl -X DELETE http://localhost/api/inventory/ingredients/uuid \ - -H "X-Demo-Session-Id: demo_abc123..." - -# Response: -# { -# "error": "demo_restriction", -# "message": "Esta operación no está permitida en cuentas demo..." -# } -``` - ---- - -## 📊 Demo Accounts - -### Account 1: Individual Bakery - -```yaml -Name: Panadería San Pablo - Demo -Email: demo.individual@panaderiasanpablo.com -Password: DemoSanPablo2024! -Business Model: individual_bakery -Location: Madrid, Spain - -Features: - - Production Management ✓ - - Recipe Management ✓ - - Inventory Tracking ✓ - - Demand Forecasting ✓ - - POS System ✓ - - Sales Analytics ✓ - -Data: - - 20+ raw ingredients - - 5+ finished products - - Multiple stock lots - - Production batches - - Sales history -``` - -### Account 2: Central Baker Satellite - -```yaml -Name: Panadería La Espiga - Demo -Email: demo.central@panaderialaespiga.com -Password: DemoLaEspiga2024! -Business Model: central_baker_satellite -Location: Barcelona, Spain - -Features: - - Supplier Management ✓ - - Inventory Tracking ✓ - - Order Management ✓ - - POS System ✓ - - Sales Analytics ✓ - - Demand Forecasting ✓ - -Data: - - 15+ par-baked products - - 10+ finished products - - Supplier relationships - - Delivery tracking - - Sales history -``` - ---- - -## 🔧 Configuration - -### Session Settings - -Edit `services/demo_session/app/core/config.py`: - -```python -DEMO_SESSION_DURATION_MINUTES = 30 # Session lifetime -DEMO_SESSION_MAX_EXTENSIONS = 3 # Max extensions allowed -REDIS_SESSION_TTL = 1800 # Redis cache TTL (seconds) -``` - -### Operation Restrictions - -Edit `gateway/app/middleware/demo_middleware.py`: - -```python -DEMO_ALLOWED_OPERATIONS = { - "GET": ["*"], - "POST": [ - "/api/pos/sales", # Allow sales - "/api/orders", # Allow orders - "/api/inventory/adjustments" # Allow adjustments - ], - "DELETE": [] # Block all deletes -} -``` - -### Cleanup Schedule - -Edit `infrastructure/kubernetes/base/cronjobs/demo-cleanup-cronjob.yaml`: - -```yaml -spec: - schedule: "0 * * * *" # Every hour - # Or: - # schedule: "*/30 * * * *" # Every 30 minutes - # schedule: "0 */3 * * *" # Every 3 hours -``` - ---- - -## 📈 Monitoring - -### Check Active Sessions - -```bash -# Get statistics -curl http://localhost/api/demo/stats | jq - -# Get specific session -curl http://localhost/api/demo/session/{session_id} | jq -``` - -### View Logs - -```bash -# Demo session service -kubectl logs -f deployment/demo-session-service -n bakery-ia - -# Cleanup job -kubectl logs -l app=demo-cleanup -n bakery-ia --tail=100 - -# Seed jobs -kubectl logs job/demo-seed-inventory -n bakery-ia -``` - -### Metrics - -```bash -# Database queries -kubectl exec -it deployment/demo-session-service -n bakery-ia -- \ - psql $DEMO_SESSION_DATABASE_URL -c \ - "SELECT status, COUNT(*) FROM demo_sessions GROUP BY status;" - -# Redis memory -kubectl exec -it deployment/redis -n bakery-ia -- \ - redis-cli INFO memory -``` - ---- - -## 🔄 Maintenance - -### Manual Cleanup - -```bash -# Trigger cleanup manually -kubectl create job --from=cronjob/demo-session-cleanup \ - manual-cleanup-$(date +%s) -n bakery-ia - -# Watch cleanup progress -kubectl logs -f job/manual-cleanup-xxxxx -n bakery-ia -``` - -### Reseed Demo Data - -```bash -# Delete and recreate seed jobs -kubectl delete job demo-seed-inventory -n bakery-ia -kubectl apply -f infrastructure/kubernetes/base/jobs/demo-seed-inventory-job.yaml - -# Watch progress -kubectl logs -f job/demo-seed-inventory -n bakery-ia -``` - -### Scale Demo Service - -```bash -# Scale up for high load -kubectl scale deployment/demo-session-service --replicas=4 -n bakery-ia - -# Scale down for maintenance -kubectl scale deployment/demo-session-service --replicas=1 -n bakery-ia -``` - ---- - -## 🛠 Troubleshooting - -### Sessions Not Creating - -1. **Check demo-session-service health** - ```bash - kubectl get pods -l app=demo-session-service -n bakery-ia - kubectl logs deployment/demo-session-service -n bakery-ia --tail=50 - ``` - -2. **Verify base tenants exist** - ```bash - kubectl exec -it deployment/tenant-service -n bakery-ia -- \ - psql $TENANT_DATABASE_URL -c \ - "SELECT id, name, is_demo_template FROM tenants WHERE is_demo = true;" - ``` - -3. **Check Redis connection** - ```bash - kubectl exec -it deployment/demo-session-service -n bakery-ia -- \ - python -c "import redis; r=redis.Redis(host='redis-service'); print(r.ping())" - ``` - -### Sessions Not Cleaning Up - -1. **Check CronJob status** - ```bash - kubectl get cronjobs -n bakery-ia - kubectl get jobs -l app=demo-cleanup -n bakery-ia - ``` - -2. **Manually trigger cleanup** - ```bash - curl -X POST http://localhost/api/demo/cleanup/run - ``` - -3. **Check for stuck sessions** - ```bash - kubectl exec -it deployment/demo-session-service -n bakery-ia -- \ - psql $DEMO_SESSION_DATABASE_URL -c \ - "SELECT session_id, status, expires_at FROM demo_sessions WHERE status = 'active';" - ``` - -### Gateway Not Injecting Virtual Tenant - -1. **Check middleware is loaded** - ```bash - kubectl logs deployment/gateway -n bakery-ia | grep -i demo - ``` - -2. **Verify session ID in request** - ```bash - curl -v http://localhost/api/inventory/ingredients \ - -H "X-Demo-Session-Id: your-session-id" - ``` - -3. **Check demo middleware logic** - - Review [demo_middleware.py](gateway/app/middleware/demo_middleware.py) - - Ensure session is active - - Verify operation is allowed - ---- - -## 🎉 Success Criteria - -✅ **Demo session creates successfully** -- Session ID returned -- Virtual tenant ID generated -- Expiration time set - -✅ **Data is isolated** -- Multiple sessions don't interfere -- Each session has unique tenant ID - -✅ **Spanish demo data loads** -- Ingredients in Spanish -- Realistic bakery scenarios -- Both business models represented - -✅ **Operations restricted** -- Read operations allowed -- Write operations limited -- Delete operations blocked - -✅ **Automatic cleanup works** -- Sessions expire after 30 minutes -- CronJob removes expired sessions -- Redis keys cleaned up - -✅ **Gateway integration works** -- Middleware detects sessions -- Virtual tenant injected -- Restrictions enforced - -✅ **K8s Job cloning works** -- Dynamic image detection in Tilt (dev) -- Environment variable configuration -- Automatic data cloning per session -- No service-specific clone endpoints needed - -✅ **AI & Scheduler protection works** -- Forecast API blocked for demo accounts -- Scheduler filters demo tenants -- Fake models in database only - ---- - -## 📚 Next Steps - -### For Frontend Integration - -1. Create demo login page showing both accounts -2. Implement session token storage (cookie/localStorage) -3. Add session timer UI component -4. Show "DEMO MODE" badge in header -5. Display session expiration warnings - -### For Marketing - -1. Publish demo credentials on website -2. Create demo walkthrough videos -3. Add "Probar Demo" CTA buttons -4. Track demo → signup conversion - -### For Operations - -1. Set up monitoring dashboards -2. Configure alerts for cleanup failures -3. Track session metrics (duration, usage) -4. Optimize Redis cache strategy - ---- - -## 📞 Support - -For issues or questions: -- Review [DEMO_ARCHITECTURE.md](DEMO_ARCHITECTURE.md) for detailed documentation -- Check logs: `tilt logs demo-session-service` -- Inspect database: `psql $DEMO_SESSION_DATABASE_URL` - ---- - -## 🔧 Technical Architecture Decisions - -### Data Cloning: Why Kubernetes Jobs? - -**Problem**: Need to clone demo data from base template tenants to virtual tenants for each session. - -**Options Considered**: -1. ❌ **Service-based clone endpoints** - Would require `/internal/demo/clone` in every service -2. ❌ **PostgreSQL Foreign Data Wrapper** - Complex setup, doesn't work across databases -3. ✅ **Kubernetes Jobs** - Selected approach - -**Why K8s Jobs Won**: -- Database-level operations (ORM-based, faster than API calls) -- Scalable (one job per session, isolated execution) -- No service coupling (don't need clone endpoints in every service) -- Works in all environments (dev & production) - -### Image Configuration: Environment Variables - -**Problem**: K8s Jobs need container images, but Tilt uses dynamic tags (e.g., `tilt-abc123`) while production uses stable tags. - -**Solution**: Environment variable `CLONE_JOB_IMAGE` -```yaml -# Demo-session deployment has default -env: -- name: CLONE_JOB_IMAGE - value: "bakery/inventory-service:latest" - -# Tilt patches it dynamically -# Tiltfile line 231-237 -inventory_image_ref = kubectl get deployment inventory-service ... -kubectl set env deployment/demo-session-service CLONE_JOB_IMAGE=$inventory_image_ref -``` - -**Benefits**: -- ✅ General solution (not tied to specific service) -- ✅ Works in dev (dynamic Tilt tags) -- ✅ Works in production (stable release tags) -- ✅ Easy to change image via env var - -### Middleware: BaseHTTPMiddleware Pattern - -**Problem**: Initial function-based middleware using `@app.middleware("http")` wasn't executing. - -**Solution**: Converted to class-based `BaseHTTPMiddleware` -```python -class DemoMiddleware(BaseHTTPMiddleware): - async def dispatch(self, request: Request, call_next): - # ... middleware logic -``` - -**Why**: FastAPI's `BaseHTTPMiddleware` provides better lifecycle hooks and guaranteed execution order. - ---- - -**Implementation Date**: 2025-10-02 -**Last Updated**: 2025-10-03 -**Status**: ✅ Complete - Ready for Production -**Next**: Frontend integration and end-to-end testing diff --git a/DEPLOYMENT_COMMANDS.md b/DEPLOYMENT_COMMANDS.md deleted file mode 100644 index b16baff0..00000000 --- a/DEPLOYMENT_COMMANDS.md +++ /dev/null @@ -1,322 +0,0 @@ -# Deployment Commands - Quick Reference - -## Implementation Complete ✅ - -All changes are implemented. Services now only verify database readiness - they never run migrations. - ---- - -## Deploy the New Architecture - -### Option 1: Skaffold (Recommended) - -```bash -# Development mode (auto-rebuild on changes) -skaffold dev - -# Production deployment -skaffold run -``` - -### Option 2: Manual Deployment - -```bash -# 1. Build all service images -for service in auth orders inventory external pos sales recipes \ - training suppliers tenant notification forecasting \ - production alert-processor; do - docker build -t bakery/${service}-service:latest services/${service}/ -done - -# 2. Apply Kubernetes manifests -kubectl apply -f infrastructure/kubernetes/base/ - -# 3. Wait for rollout -kubectl rollout status deployment --all -n bakery-ia -``` - ---- - -## Verification Commands - -### Check Services Are Using New Code: - -```bash -# Check external service logs for verification (not migration) -kubectl logs -n bakery-ia deployment/external-service | grep -i "verification" - -# Expected output: -# [info] Database verification mode - checking database is ready -# [info] Database verification successful - -# Should NOT see (old behavior): -# [info] Running pending migrations -``` - -### Check All Services: - -```bash -# Check all service logs -for service in auth orders inventory external pos sales recipes \ - training suppliers tenant notification forecasting \ - production alert-processor; do - echo "=== Checking $service-service ===" - kubectl logs -n bakery-ia deployment/${service}-service --tail=20 | grep -E "(verification|migration)" || echo "No logs yet" -done -``` - -### Check Startup Times: - -```bash -# Watch pod startup times -kubectl get events -n bakery-ia --sort-by='.lastTimestamp' --watch - -# Or check specific service -kubectl describe pod -n bakery-ia -l app.kubernetes.io/name=external-service | grep -A 5 "Events:" -``` - ---- - -## Troubleshooting - -### Service Won't Start - "Database is empty" - -```bash -# 1. Check migration job status -kubectl get jobs -n bakery-ia | grep migration - -# 2. Check specific migration job -kubectl logs -n bakery-ia job/external-migration - -# 3. Re-run migration job if needed -kubectl delete job external-migration -n bakery-ia -kubectl apply -f infrastructure/kubernetes/base/migrations/external-migration.yaml -``` - -### Service Won't Start - "No migration files found" - -```bash -# 1. Check if migrations exist in image -kubectl exec -n bakery-ia deployment/external-service -- ls -la /app/migrations/versions/ - -# 2. If missing, regenerate and rebuild -./regenerate_migrations_k8s.sh --verbose -skaffold build -kubectl rollout restart deployment/external-service -n bakery-ia -``` - -### Check Migration Job Logs: - -```bash -# List all migration jobs -kubectl get jobs -n bakery-ia | grep migration - -# Check specific job logs -kubectl logs -n bakery-ia job/-migration - -# Example: -kubectl logs -n bakery-ia job/auth-migration -``` - ---- - -## Performance Testing - -### Measure Startup Time Improvement: - -```bash -# 1. Record current startup times -kubectl get events -n bakery-ia --sort-by='.lastTimestamp' | grep "Started container" > before.txt - -# 2. Deploy new code -skaffold run - -# 3. Restart services to measure -kubectl rollout restart deployment --all -n bakery-ia - -# 4. Record new startup times -kubectl get events -n bakery-ia --sort-by='.lastTimestamp' | grep "Started container" > after.txt - -# 5. Compare (should be 50-80% faster) -diff before.txt after.txt -``` - -### Monitor Database Load: - -```bash -# Check database connections during startup -kubectl exec -n bakery-ia external-db- -- \ - psql -U external_user -d external_db -c \ - "SELECT count(*) FROM pg_stat_activity WHERE datname='external_db';" -``` - ---- - -## Rollback (If Needed) - -### Rollback Deployments: - -```bash -# Rollback specific service -kubectl rollout undo deployment/external-service -n bakery-ia - -# Rollback all services -kubectl rollout undo deployment --all -n bakery-ia - -# Check rollout status -kubectl rollout status deployment --all -n bakery-ia -``` - -### Rollback to Specific Revision: - -```bash -# List revisions -kubectl rollout history deployment/external-service -n bakery-ia - -# Rollback to specific revision -kubectl rollout undo deployment/external-service --to-revision=2 -n bakery-ia -``` - ---- - -## Clean Deployment - -### If You Want Fresh Start: - -```bash -# 1. Delete everything -kubectl delete namespace bakery-ia - -# 2. Recreate namespace -kubectl create namespace bakery-ia - -# 3. Apply all manifests -kubectl apply -f infrastructure/kubernetes/base/ - -# 4. Wait for all to be ready -kubectl wait --for=condition=ready pod --all -n bakery-ia --timeout=300s -``` - ---- - -## Health Checks - -### Check All Pods: - -```bash -kubectl get pods -n bakery-ia -``` - -### Check Services Are Ready: - -```bash -# Check all services -kubectl get deployments -n bakery-ia - -# Check specific service health -kubectl exec -n bakery-ia deployment/external-service -- \ - curl -s http://localhost:8000/health/live -``` - -### Check Migration Jobs Completed: - -```bash -# Should all show "Complete" -kubectl get jobs -n bakery-ia | grep migration -``` - ---- - -## Useful Aliases - -Add to your `~/.bashrc` or `~/.zshrc`: - -```bash -# Kubernetes bakery-ia shortcuts -alias k='kubectl' -alias kn='kubectl -n bakery-ia' -alias kp='kubectl get pods -n bakery-ia' -alias kd='kubectl get deployments -n bakery-ia' -alias kj='kubectl get jobs -n bakery-ia' -alias kl='kubectl logs -n bakery-ia' -alias kdesc='kubectl describe -n bakery-ia' - -# Quick log checks -alias klogs='kubectl logs -n bakery-ia deployment/' - -# Example usage: -# klogs external-service | grep verification -``` - ---- - -## Expected Output Examples - -### Migration Job (Successful): - -``` -[info] Migration job starting service=external -[info] Migration mode - running database migrations -[info] Running pending migrations -INFO [alembic.runtime.migration] Context impl PostgresqlImpl. -INFO [alembic.runtime.migration] Will assume transactional DDL. -[info] Migrations applied successfully -[info] Migration job completed successfully -``` - -### Service Startup (New Behavior): - -``` -[info] Starting external-service version=1.0.0 -[info] Database connection established -[info] Database verification mode - checking database is ready -[info] Database state checked -[info] Database verification successful - migration_count=1 current_revision=374752db316e table_count=6 -[info] Database verification completed -[info] external-service started successfully -``` - ---- - -## CI/CD Integration - -### GitHub Actions Example: - -```yaml -name: Deploy to Kubernetes -on: - push: - branches: [main] - -jobs: - deploy: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - - name: Build and push images - run: skaffold build - - - name: Deploy to cluster - run: skaffold run - - - name: Verify deployment - run: | - kubectl rollout status deployment --all -n bakery-ia - kubectl get pods -n bakery-ia -``` - ---- - -## Summary - -**To Deploy**: Just run `skaffold dev` or `skaffold run` - -**To Verify**: Check logs show "verification" not "migration" - -**To Troubleshoot**: Check migration job logs first - -**Expected Result**: Services start 50-80% faster, no redundant migration execution - -**Status**: ✅ Ready to deploy! diff --git a/EXTERNAL_DATA_REDESIGN_IMPLEMENTATION.md b/EXTERNAL_DATA_REDESIGN_IMPLEMENTATION.md deleted file mode 100644 index fcf53ee2..00000000 --- a/EXTERNAL_DATA_REDESIGN_IMPLEMENTATION.md +++ /dev/null @@ -1,141 +0,0 @@ -# External Data Service Redesign - Implementation Summary - -**Status:** ✅ **COMPLETE** -**Date:** October 7, 2025 -**Version:** 2.0.0 - ---- - -## 🎯 Objective - -Redesign the external data service to eliminate redundant per-tenant fetching, enable multi-city support, implement automated 24-month rolling windows, and leverage Kubernetes for lifecycle management. - ---- - -## ✅ All Deliverables Completed - -### 1. Backend Implementation (Python/FastAPI) - -#### City Registry & Geolocation -- ✅ `services/external/app/registry/city_registry.py` -- ✅ `services/external/app/registry/geolocation_mapper.py` - -#### Data Adapters -- ✅ `services/external/app/ingestion/base_adapter.py` -- ✅ `services/external/app/ingestion/adapters/madrid_adapter.py` -- ✅ `services/external/app/ingestion/adapters/__init__.py` -- ✅ `services/external/app/ingestion/ingestion_manager.py` - -#### Database Layer -- ✅ `services/external/app/models/city_weather.py` -- ✅ `services/external/app/models/city_traffic.py` -- ✅ `services/external/app/repositories/city_data_repository.py` -- ✅ `services/external/migrations/versions/20251007_0733_add_city_data_tables.py` - -#### Cache Layer -- ✅ `services/external/app/cache/redis_cache.py` - -#### API Layer -- ✅ `services/external/app/schemas/city_data.py` -- ✅ `services/external/app/api/city_operations.py` -- ✅ Updated `services/external/app/main.py` (router registration) - -#### Job Scripts -- ✅ `services/external/app/jobs/initialize_data.py` -- ✅ `services/external/app/jobs/rotate_data.py` - -### 2. Infrastructure (Kubernetes) - -- ✅ `infrastructure/kubernetes/external/init-job.yaml` -- ✅ `infrastructure/kubernetes/external/cronjob.yaml` -- ✅ `infrastructure/kubernetes/external/deployment.yaml` -- ✅ `infrastructure/kubernetes/external/configmap.yaml` -- ✅ `infrastructure/kubernetes/external/secrets.yaml` - -### 3. Frontend (TypeScript) - -- ✅ `frontend/src/api/types/external.ts` (added CityInfoResponse, DataAvailabilityResponse) -- ✅ `frontend/src/api/services/external.ts` (complete service client) - -### 4. Documentation - -- ✅ `EXTERNAL_DATA_SERVICE_REDESIGN.md` (complete architecture) -- ✅ `services/external/IMPLEMENTATION_COMPLETE.md` (deployment guide) -- ✅ `EXTERNAL_DATA_REDESIGN_IMPLEMENTATION.md` (this file) - ---- - -## 📊 Performance Improvements - -| Metric | Before | After | Improvement | -|--------|--------|-------|-------------| -| **Historical Weather (1 month)** | 3-5 sec | <100ms | **30-50x faster** | -| **Historical Traffic (1 month)** | 5-10 sec | <100ms | **50-100x faster** | -| **Training Data Load (24 months)** | 60-120 sec | 1-2 sec | **60x faster** | -| **Data Redundancy** | N tenants × fetch | 1 fetch shared | **100% deduplication** | -| **Cache Hit Rate** | 0% | >70% | **70% reduction in DB load** | - ---- - -## 🚀 Quick Start - -### 1. Run Database Migration - -```bash -cd services/external -alembic upgrade head -``` - -### 2. Configure Secrets - -```bash -cd infrastructure/kubernetes/external -# Edit secrets.yaml with actual API keys -kubectl apply -f secrets.yaml -kubectl apply -f configmap.yaml -``` - -### 3. Initialize Data (One-time) - -```bash -kubectl apply -f init-job.yaml -kubectl logs -f job/external-data-init -n bakery-ia -``` - -### 4. Deploy Service - -```bash -kubectl apply -f deployment.yaml -kubectl wait --for=condition=ready pod -l app=external-service -n bakery-ia -``` - -### 5. Schedule Monthly Rotation - -```bash -kubectl apply -f cronjob.yaml -``` - ---- - -## 🎉 Success Criteria - All Met! - -✅ **No redundant fetching** - City-based storage eliminates per-tenant downloads -✅ **Multi-city support** - Architecture supports Madrid, Valencia, Barcelona, etc. -✅ **Sub-100ms access** - Redis cache provides instant training data -✅ **Automated rotation** - Kubernetes CronJob handles 24-month window -✅ **Zero downtime** - Init job ensures data before service start -✅ **Type-safe frontend** - Full TypeScript integration -✅ **Production-ready** - No TODOs, complete observability - ---- - -## 📚 Additional Resources - -- **Full Architecture:** `/Users/urtzialfaro/Documents/bakery-ia/EXTERNAL_DATA_SERVICE_REDESIGN.md` -- **Deployment Guide:** `/Users/urtzialfaro/Documents/bakery-ia/services/external/IMPLEMENTATION_COMPLETE.md` -- **API Documentation:** `http://localhost:8000/docs` (when service is running) - ---- - -**Implementation completed:** October 7, 2025 -**Compliance:** ✅ All constraints met (no backward compatibility, no legacy code, production-ready) diff --git a/EXTERNAL_DATA_SERVICE_REDESIGN.md b/EXTERNAL_DATA_SERVICE_REDESIGN.md deleted file mode 100644 index 3ee5eb01..00000000 --- a/EXTERNAL_DATA_SERVICE_REDESIGN.md +++ /dev/null @@ -1,2660 +0,0 @@ -# External Data Service Architectural Redesign - -**Project:** Bakery IA - External Data Service -**Version:** 2.0.0 -**Date:** 2025-10-07 -**Status:** Complete Architecture & Implementation Plan - ---- - -## Executive Summary - -This document provides a complete architectural redesign of the external data service to eliminate redundant per-tenant data fetching, enable multi-city support, implement automated 24-month rolling windows, and leverage Kubernetes for lifecycle management. - -### Key Problems Addressed - -1. ✅ **Per-tenant redundant fetching** → Centralized city-based data storage -2. ✅ **Geographic limitation (Madrid only)** → Multi-city extensible architecture -3. ✅ **Redundant downloads for same city** → Shared data layer with geolocation mapping -4. ✅ **Slow training pipeline** → Pre-populated historical datasets via K8s Jobs -5. ✅ **Static data windows** → Automated 24-month rolling updates via CronJobs - ---- - -## Part 1: High-Level Architecture - -### 1.1 Architecture Overview - -``` -┌─────────────────────────────────────────────────────────────────────┐ -│ KUBERNETES ORCHESTRATION │ -├─────────────────────────────────────────────────────────────────────┤ -│ │ -│ ┌──────────────────┐ ┌──────────────────┐ │ -│ │ Init Job │ │ Monthly CronJob │ │ -│ │ (One-time) │ │ (Scheduled) │ │ -│ ├──────────────────┤ ├──────────────────┤ │ -│ │ • Load 24 months │ │ • Expire old │ │ -│ │ • All cities │ │ • Ingest new │ │ -│ │ • Traffic + Wx │ │ • Rotate window │ │ -│ └────────┬─────────┘ └────────┬─────────┘ │ -│ │ │ │ -│ └────────┬────────────────┘ │ -│ │ │ -│ ▼ │ -│ ┌──────────────────────────────────────────────────────────┐ │ -│ │ Data Ingestion Manager │ │ -│ │ ┌────────────┐ ┌────────────┐ ┌────────────┐ │ │ -│ │ │ Madrid │ │ Valencia │ │ Barcelona │ │ │ -│ │ │ Adapter │ │ Adapter │ │ Adapter │ ... │ │ -│ │ └────────────┘ └────────────┘ └────────────┘ │ │ -│ └─────────────────────────┬────────────────────────────────┘ │ -│ │ │ -│ ▼ │ -│ ┌──────────────────────────────────────────────────────────┐ │ -│ │ Shared Storage Layer (PostgreSQL + Redis) │ │ -│ │ - City-based historical data (24-month window) │ │ -│ │ - Traffic: city_traffic_data table │ │ -│ │ - Weather: city_weather_data table │ │ -│ │ - Redis cache for fast access during training │ │ -│ └─────────────────────────┬────────────────────────────────┘ │ -│ │ │ -└────────────────────────────┼─────────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────────────────┐ -│ External Data Service (FastAPI) │ -├─────────────────────────────────────────────────────────────────────┤ -│ │ -│ ┌──────────────────────────────────────────────────────────┐ │ -│ │ Geolocation Mapper: Tenant → City │ │ -│ │ - Maps (lat, lon) to nearest supported city │ │ -│ │ - Returns city-specific cached data │ │ -│ └──────────────────────────────────────────────────────────┘ │ -│ │ │ -│ ▼ │ -│ ┌──────────────────────────────────────────────────────────┐ │ -│ │ API Endpoints │ │ -│ │ GET /api/v1/tenants/{id}/external/historical-weather │ │ -│ │ GET /api/v1/tenants/{id}/external/historical-traffic │ │ -│ │ GET /api/v1/cities │ │ -│ │ GET /api/v1/cities/{city_id}/data-availability │ │ -│ └──────────────────────────────────────────────────────────┘ │ -│ │ -└───────────────────────────────┬───────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────────────────┐ -│ Training Service Consumer │ -│ - Requests historical data for tenant location │ -│ - Receives pre-populated city data (instant response) │ -│ - No waiting for external API calls │ -└─────────────────────────────────────────────────────────────────────┘ -``` - -### 1.2 Data Flow - -#### **Initialization Phase (Kubernetes Job)** -``` -1. Job starts → Read city registry config -2. For each city: - a. Instantiate city-specific adapter (Madrid, Valencia, etc.) - b. Fetch last 24 months of traffic data - c. Fetch last 24 months of weather data - d. Store in shared PostgreSQL tables (city_id indexed) - e. Warm Redis cache -3. Job completes → Service deployment readiness probe passes -``` - -#### **Monthly Maintenance (Kubernetes CronJob)** -``` -1. CronJob triggers (1st of month, 2am UTC) -2. For each city: - a. Delete data older than 24 months - b. Fetch latest available month's data - c. Append to shared tables - d. Invalidate old cache entries -3. Log completion metrics -``` - -#### **Runtime Request Flow** -``` -1. Training service → GET /api/v1/tenants/{id}/external/historical-traffic -2. External service: - a. Extract tenant lat/lon from tenant profile - b. Geolocation mapper → Find nearest city - c. Query city_traffic_data WHERE city_id=X AND date BETWEEN ... - d. Return cached results (< 100ms) -3. Training service receives data instantly -``` - ---- - -## Part 2: Component Breakdown - -### 2.1 City Registry & Geolocation Mapper - -**File:** `services/external/app/registry/city_registry.py` - -```python -# services/external/app/registry/city_registry.py -""" -City Registry - Configuration-driven multi-city support -""" - -from dataclasses import dataclass -from typing import List, Optional, Dict, Any -from enum import Enum -import math - - -class Country(str, Enum): - SPAIN = "ES" - FRANCE = "FR" - # Extensible - - -class WeatherProvider(str, Enum): - AEMET = "aemet" # Spain - METEO_FRANCE = "meteo_france" # France - OPEN_WEATHER = "open_weather" # Global fallback - - -class TrafficProvider(str, Enum): - MADRID_OPENDATA = "madrid_opendata" - VALENCIA_OPENDATA = "valencia_opendata" - BARCELONA_OPENDATA = "barcelona_opendata" - - -@dataclass -class CityDefinition: - """City configuration with data source specifications""" - city_id: str - name: str - country: Country - latitude: float - longitude: float - radius_km: float # Coverage radius - - # Data providers - weather_provider: WeatherProvider - weather_config: Dict[str, Any] # Provider-specific config - traffic_provider: TrafficProvider - traffic_config: Dict[str, Any] - - # Metadata - timezone: str - population: int - enabled: bool = True - - -class CityRegistry: - """Central registry of supported cities""" - - CITIES: List[CityDefinition] = [ - CityDefinition( - city_id="madrid", - name="Madrid", - country=Country.SPAIN, - latitude=40.4168, - longitude=-3.7038, - radius_km=30.0, - weather_provider=WeatherProvider.AEMET, - weather_config={ - "station_ids": ["3195", "3129", "3197"], - "municipality_code": "28079" - }, - traffic_provider=TrafficProvider.MADRID_OPENDATA, - traffic_config={ - "current_xml_url": "https://datos.madrid.es/egob/catalogo/...", - "historical_base_url": "https://datos.madrid.es/...", - "measurement_points_csv": "https://datos.madrid.es/..." - }, - timezone="Europe/Madrid", - population=3_200_000 - ), - CityDefinition( - city_id="valencia", - name="Valencia", - country=Country.SPAIN, - latitude=39.4699, - longitude=-0.3763, - radius_km=25.0, - weather_provider=WeatherProvider.AEMET, - weather_config={ - "station_ids": ["8416"], - "municipality_code": "46250" - }, - traffic_provider=TrafficProvider.VALENCIA_OPENDATA, - traffic_config={ - "api_endpoint": "https://valencia.opendatasoft.com/api/..." - }, - timezone="Europe/Madrid", - population=800_000, - enabled=False # Not yet implemented - ), - CityDefinition( - city_id="barcelona", - name="Barcelona", - country=Country.SPAIN, - latitude=41.3851, - longitude=2.1734, - radius_km=30.0, - weather_provider=WeatherProvider.AEMET, - weather_config={ - "station_ids": ["0076"], - "municipality_code": "08019" - }, - traffic_provider=TrafficProvider.BARCELONA_OPENDATA, - traffic_config={ - "api_endpoint": "https://opendata-ajuntament.barcelona.cat/..." - }, - timezone="Europe/Madrid", - population=1_600_000, - enabled=False # Not yet implemented - ) - ] - - @classmethod - def get_enabled_cities(cls) -> List[CityDefinition]: - """Get all enabled cities""" - return [city for city in cls.CITIES if city.enabled] - - @classmethod - def get_city(cls, city_id: str) -> Optional[CityDefinition]: - """Get city by ID""" - for city in cls.CITIES: - if city.city_id == city_id: - return city - return None - - @classmethod - def find_nearest_city(cls, latitude: float, longitude: float) -> Optional[CityDefinition]: - """Find nearest enabled city to coordinates""" - enabled_cities = cls.get_enabled_cities() - if not enabled_cities: - return None - - min_distance = float('inf') - nearest_city = None - - for city in enabled_cities: - distance = cls._haversine_distance( - latitude, longitude, - city.latitude, city.longitude - ) - if distance <= city.radius_km and distance < min_distance: - min_distance = distance - nearest_city = city - - return nearest_city - - @staticmethod - def _haversine_distance(lat1: float, lon1: float, lat2: float, lon2: float) -> float: - """Calculate distance in km between two coordinates""" - R = 6371 # Earth radius in km - - dlat = math.radians(lat2 - lat1) - dlon = math.radians(lon2 - lon1) - - a = (math.sin(dlat/2) ** 2 + - math.cos(math.radians(lat1)) * math.cos(math.radians(lat2)) * - math.sin(dlon/2) ** 2) - - c = 2 * math.atan2(math.sqrt(a), math.sqrt(1-a)) - return R * c -``` - -**File:** `services/external/app/registry/geolocation_mapper.py` - -```python -# services/external/app/registry/geolocation_mapper.py -""" -Geolocation Mapper - Maps tenant locations to cities -""" - -from typing import Optional, Tuple -import structlog -from .city_registry import CityRegistry, CityDefinition - -logger = structlog.get_logger() - - -class GeolocationMapper: - """Maps tenant coordinates to nearest supported city""" - - def __init__(self): - self.registry = CityRegistry() - - def map_tenant_to_city( - self, - latitude: float, - longitude: float - ) -> Optional[Tuple[CityDefinition, float]]: - """ - Map tenant coordinates to nearest city - - Returns: - Tuple of (CityDefinition, distance_km) or None if no match - """ - nearest_city = self.registry.find_nearest_city(latitude, longitude) - - if not nearest_city: - logger.warning( - "No supported city found for coordinates", - lat=latitude, - lon=longitude - ) - return None - - distance = self.registry._haversine_distance( - latitude, longitude, - nearest_city.latitude, nearest_city.longitude - ) - - logger.info( - "Mapped tenant to city", - lat=latitude, - lon=longitude, - city=nearest_city.name, - distance_km=round(distance, 2) - ) - - return (nearest_city, distance) - - def validate_location_support(self, latitude: float, longitude: float) -> bool: - """Check if coordinates are supported""" - result = self.map_tenant_to_city(latitude, longitude) - return result is not None -``` - -### 2.2 Data Ingestion Manager with Adapter Pattern - -**File:** `services/external/app/ingestion/base_adapter.py` - -```python -# services/external/app/ingestion/base_adapter.py -""" -Base adapter interface for city-specific data sources -""" - -from abc import ABC, abstractmethod -from typing import List, Dict, Any -from datetime import datetime - - -class CityDataAdapter(ABC): - """Abstract base class for city-specific data adapters""" - - def __init__(self, city_id: str, config: Dict[str, Any]): - self.city_id = city_id - self.config = config - - @abstractmethod - async def fetch_historical_weather( - self, - start_date: datetime, - end_date: datetime - ) -> List[Dict[str, Any]]: - """Fetch historical weather data for date range""" - pass - - @abstractmethod - async def fetch_historical_traffic( - self, - start_date: datetime, - end_date: datetime - ) -> List[Dict[str, Any]]: - """Fetch historical traffic data for date range""" - pass - - @abstractmethod - async def validate_connection(self) -> bool: - """Validate connection to data source""" - pass - - def get_city_id(self) -> str: - """Get city identifier""" - return self.city_id -``` - -**File:** `services/external/app/ingestion/adapters/madrid_adapter.py` - -```python -# services/external/app/ingestion/adapters/madrid_adapter.py -""" -Madrid city data adapter - Uses existing AEMET and Madrid OpenData clients -""" - -from typing import List, Dict, Any -from datetime import datetime -import structlog - -from ..base_adapter import CityDataAdapter -from app.external.aemet import AEMETClient -from app.external.apis.madrid_traffic_client import MadridTrafficClient - -logger = structlog.get_logger() - - -class MadridAdapter(CityDataAdapter): - """Adapter for Madrid using AEMET + Madrid OpenData""" - - def __init__(self, city_id: str, config: Dict[str, Any]): - super().__init__(city_id, config) - self.aemet_client = AEMETClient() - self.traffic_client = MadridTrafficClient() - - # Madrid center coordinates - self.madrid_lat = 40.4168 - self.madrid_lon = -3.7038 - - async def fetch_historical_weather( - self, - start_date: datetime, - end_date: datetime - ) -> List[Dict[str, Any]]: - """Fetch historical weather from AEMET""" - try: - logger.info( - "Fetching Madrid historical weather", - start=start_date.isoformat(), - end=end_date.isoformat() - ) - - weather_data = await self.aemet_client.get_historical_weather( - self.madrid_lat, - self.madrid_lon, - start_date, - end_date - ) - - # Enrich with city_id - for record in weather_data: - record['city_id'] = self.city_id - record['city_name'] = 'Madrid' - - logger.info( - "Madrid weather data fetched", - records=len(weather_data) - ) - - return weather_data - - except Exception as e: - logger.error("Error fetching Madrid weather", error=str(e)) - return [] - - async def fetch_historical_traffic( - self, - start_date: datetime, - end_date: datetime - ) -> List[Dict[str, Any]]: - """Fetch historical traffic from Madrid OpenData""" - try: - logger.info( - "Fetching Madrid historical traffic", - start=start_date.isoformat(), - end=end_date.isoformat() - ) - - traffic_data = await self.traffic_client.get_historical_traffic( - self.madrid_lat, - self.madrid_lon, - start_date, - end_date - ) - - # Enrich with city_id - for record in traffic_data: - record['city_id'] = self.city_id - record['city_name'] = 'Madrid' - - logger.info( - "Madrid traffic data fetched", - records=len(traffic_data) - ) - - return traffic_data - - except Exception as e: - logger.error("Error fetching Madrid traffic", error=str(e)) - return [] - - async def validate_connection(self) -> bool: - """Validate connection to AEMET and Madrid OpenData""" - try: - # Test weather connection - test_weather = await self.aemet_client.get_current_weather( - self.madrid_lat, - self.madrid_lon - ) - - # Test traffic connection - test_traffic = await self.traffic_client.get_current_traffic( - self.madrid_lat, - self.madrid_lon - ) - - return test_weather is not None and test_traffic is not None - - except Exception as e: - logger.error("Madrid adapter connection validation failed", error=str(e)) - return False -``` - -**File:** `services/external/app/ingestion/adapters/__init__.py` - -```python -# services/external/app/ingestion/adapters/__init__.py -""" -Adapter registry - Maps city IDs to adapter implementations -""" - -from typing import Dict, Type -from ..base_adapter import CityDataAdapter -from .madrid_adapter import MadridAdapter - -# Registry: city_id → Adapter class -ADAPTER_REGISTRY: Dict[str, Type[CityDataAdapter]] = { - "madrid": MadridAdapter, - # "valencia": ValenciaAdapter, # Future - # "barcelona": BarcelonaAdapter, # Future -} - - -def get_adapter(city_id: str, config: Dict) -> CityDataAdapter: - """Factory to instantiate appropriate adapter""" - adapter_class = ADAPTER_REGISTRY.get(city_id) - if not adapter_class: - raise ValueError(f"No adapter registered for city: {city_id}") - return adapter_class(city_id, config) -``` - -**File:** `services/external/app/ingestion/ingestion_manager.py` - -```python -# services/external/app/ingestion/ingestion_manager.py -""" -Data Ingestion Manager - Coordinates multi-city data collection -""" - -from typing import List, Dict, Any -from datetime import datetime, timedelta -import structlog -import asyncio - -from app.registry.city_registry import CityRegistry -from .adapters import get_adapter -from app.repositories.city_data_repository import CityDataRepository -from app.core.database import database_manager - -logger = structlog.get_logger() - - -class DataIngestionManager: - """Orchestrates data ingestion across all cities""" - - def __init__(self): - self.registry = CityRegistry() - self.database_manager = database_manager - - async def initialize_all_cities(self, months: int = 24): - """ - Initialize historical data for all enabled cities - Called by Kubernetes Init Job - """ - enabled_cities = self.registry.get_enabled_cities() - - logger.info( - "Starting full data initialization", - cities=len(enabled_cities), - months=months - ) - - # Calculate date range - end_date = datetime.now() - start_date = end_date - timedelta(days=months * 30) - - # Process cities concurrently - tasks = [ - self.initialize_city(city.city_id, start_date, end_date) - for city in enabled_cities - ] - - results = await asyncio.gather(*tasks, return_exceptions=True) - - # Log results - successes = sum(1 for r in results if r is True) - failures = len(results) - successes - - logger.info( - "Data initialization complete", - total=len(results), - successes=successes, - failures=failures - ) - - return successes == len(results) - - async def initialize_city( - self, - city_id: str, - start_date: datetime, - end_date: datetime - ) -> bool: - """Initialize historical data for a single city""" - try: - city = self.registry.get_city(city_id) - if not city: - logger.error("City not found", city_id=city_id) - return False - - logger.info( - "Initializing city data", - city=city.name, - start=start_date.date(), - end=end_date.date() - ) - - # Get appropriate adapter - adapter = get_adapter( - city_id, - { - "weather_config": city.weather_config, - "traffic_config": city.traffic_config - } - ) - - # Validate connection - if not await adapter.validate_connection(): - logger.error("Adapter validation failed", city=city.name) - return False - - # Fetch weather data - weather_data = await adapter.fetch_historical_weather( - start_date, end_date - ) - - # Fetch traffic data - traffic_data = await adapter.fetch_historical_traffic( - start_date, end_date - ) - - # Store in database - async with self.database_manager.get_session() as session: - repo = CityDataRepository(session) - - weather_stored = await repo.bulk_store_weather( - city_id, weather_data - ) - traffic_stored = await repo.bulk_store_traffic( - city_id, traffic_data - ) - - logger.info( - "City initialization complete", - city=city.name, - weather_records=weather_stored, - traffic_records=traffic_stored - ) - - return True - - except Exception as e: - logger.error( - "City initialization failed", - city_id=city_id, - error=str(e) - ) - return False - - async def rotate_monthly_data(self): - """ - Rotate 24-month window: delete old, ingest new - Called by Kubernetes CronJob monthly - """ - enabled_cities = self.registry.get_enabled_cities() - - logger.info("Starting monthly data rotation", cities=len(enabled_cities)) - - now = datetime.now() - cutoff_date = now - timedelta(days=24 * 30) # 24 months ago - - # Last month's date range - last_month_end = now.replace(day=1) - timedelta(days=1) - last_month_start = last_month_end.replace(day=1) - - tasks = [] - for city in enabled_cities: - tasks.append( - self._rotate_city_data( - city.city_id, - cutoff_date, - last_month_start, - last_month_end - ) - ) - - results = await asyncio.gather(*tasks, return_exceptions=True) - - successes = sum(1 for r in results if r is True) - logger.info( - "Monthly rotation complete", - total=len(results), - successes=successes - ) - - async def _rotate_city_data( - self, - city_id: str, - cutoff_date: datetime, - new_start: datetime, - new_end: datetime - ) -> bool: - """Rotate data for a single city""" - try: - city = self.registry.get_city(city_id) - if not city: - return False - - logger.info( - "Rotating city data", - city=city.name, - cutoff=cutoff_date.date(), - new_month=new_start.strftime("%Y-%m") - ) - - async with self.database_manager.get_session() as session: - repo = CityDataRepository(session) - - # Delete old data - deleted_weather = await repo.delete_weather_before( - city_id, cutoff_date - ) - deleted_traffic = await repo.delete_traffic_before( - city_id, cutoff_date - ) - - logger.info( - "Old data deleted", - city=city.name, - weather_deleted=deleted_weather, - traffic_deleted=deleted_traffic - ) - - # Fetch new month's data - adapter = get_adapter(city_id, { - "weather_config": city.weather_config, - "traffic_config": city.traffic_config - }) - - new_weather = await adapter.fetch_historical_weather( - new_start, new_end - ) - new_traffic = await adapter.fetch_historical_traffic( - new_start, new_end - ) - - # Store new data - async with self.database_manager.get_session() as session: - repo = CityDataRepository(session) - - weather_stored = await repo.bulk_store_weather( - city_id, new_weather - ) - traffic_stored = await repo.bulk_store_traffic( - city_id, new_traffic - ) - - logger.info( - "New data ingested", - city=city.name, - weather_added=weather_stored, - traffic_added=traffic_stored - ) - - return True - - except Exception as e: - logger.error( - "City rotation failed", - city_id=city_id, - error=str(e) - ) - return False -``` - -### 2.3 Shared Storage/Cache Interface - -**File:** `services/external/app/repositories/city_data_repository.py` - -```python -# services/external/app/repositories/city_data_repository.py -""" -City Data Repository - Manages shared city-based data storage -""" - -from typing import List, Dict, Any, Optional -from datetime import datetime -from sqlalchemy import select, delete, and_ -from sqlalchemy.ext.asyncio import AsyncSession -import structlog - -from app.models.city_weather import CityWeatherData -from app.models.city_traffic import CityTrafficData - -logger = structlog.get_logger() - - -class CityDataRepository: - """Repository for city-based historical data""" - - def __init__(self, session: AsyncSession): - self.session = session - - # ============= WEATHER OPERATIONS ============= - - async def bulk_store_weather( - self, - city_id: str, - weather_records: List[Dict[str, Any]] - ) -> int: - """Bulk insert weather records for a city""" - if not weather_records: - return 0 - - try: - objects = [] - for record in weather_records: - obj = CityWeatherData( - city_id=city_id, - date=record.get('date'), - temperature=record.get('temperature'), - precipitation=record.get('precipitation'), - humidity=record.get('humidity'), - wind_speed=record.get('wind_speed'), - pressure=record.get('pressure'), - description=record.get('description'), - source=record.get('source', 'ingestion'), - raw_data=record.get('raw_data') - ) - objects.append(obj) - - self.session.add_all(objects) - await self.session.commit() - - logger.info( - "Weather data stored", - city_id=city_id, - records=len(objects) - ) - - return len(objects) - - except Exception as e: - await self.session.rollback() - logger.error( - "Error storing weather data", - city_id=city_id, - error=str(e) - ) - raise - - async def get_weather_by_city_and_range( - self, - city_id: str, - start_date: datetime, - end_date: datetime - ) -> List[CityWeatherData]: - """Get weather data for city within date range""" - stmt = select(CityWeatherData).where( - and_( - CityWeatherData.city_id == city_id, - CityWeatherData.date >= start_date, - CityWeatherData.date <= end_date - ) - ).order_by(CityWeatherData.date) - - result = await self.session.execute(stmt) - return result.scalars().all() - - async def delete_weather_before( - self, - city_id: str, - cutoff_date: datetime - ) -> int: - """Delete weather records older than cutoff date""" - stmt = delete(CityWeatherData).where( - and_( - CityWeatherData.city_id == city_id, - CityWeatherData.date < cutoff_date - ) - ) - - result = await self.session.execute(stmt) - await self.session.commit() - - return result.rowcount - - # ============= TRAFFIC OPERATIONS ============= - - async def bulk_store_traffic( - self, - city_id: str, - traffic_records: List[Dict[str, Any]] - ) -> int: - """Bulk insert traffic records for a city""" - if not traffic_records: - return 0 - - try: - objects = [] - for record in traffic_records: - obj = CityTrafficData( - city_id=city_id, - date=record.get('date'), - traffic_volume=record.get('traffic_volume'), - pedestrian_count=record.get('pedestrian_count'), - congestion_level=record.get('congestion_level'), - average_speed=record.get('average_speed'), - source=record.get('source', 'ingestion'), - raw_data=record.get('raw_data') - ) - objects.append(obj) - - self.session.add_all(objects) - await self.session.commit() - - logger.info( - "Traffic data stored", - city_id=city_id, - records=len(objects) - ) - - return len(objects) - - except Exception as e: - await self.session.rollback() - logger.error( - "Error storing traffic data", - city_id=city_id, - error=str(e) - ) - raise - - async def get_traffic_by_city_and_range( - self, - city_id: str, - start_date: datetime, - end_date: datetime - ) -> List[CityTrafficData]: - """Get traffic data for city within date range""" - stmt = select(CityTrafficData).where( - and_( - CityTrafficData.city_id == city_id, - CityTrafficData.date >= start_date, - CityTrafficData.date <= end_date - ) - ).order_by(CityTrafficData.date) - - result = await self.session.execute(stmt) - return result.scalars().all() - - async def delete_traffic_before( - self, - city_id: str, - cutoff_date: datetime - ) -> int: - """Delete traffic records older than cutoff date""" - stmt = delete(CityTrafficData).where( - and_( - CityTrafficData.city_id == city_id, - CityTrafficData.date < cutoff_date - ) - ) - - result = await self.session.execute(stmt) - await self.session.commit() - - return result.rowcount -``` - -**Database Models:** - -**File:** `services/external/app/models/city_weather.py` - -```python -# services/external/app/models/city_weather.py -""" -City Weather Data Model - Shared city-based weather storage -""" - -from sqlalchemy import Column, String, Float, DateTime, Text, Index -from sqlalchemy.dialects.postgresql import UUID, JSONB -from datetime import datetime -import uuid - -from app.core.database import Base - - -class CityWeatherData(Base): - """City-based historical weather data""" - - __tablename__ = "city_weather_data" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - city_id = Column(String(50), nullable=False, index=True) - date = Column(DateTime(timezone=True), nullable=False, index=True) - - # Weather metrics - temperature = Column(Float, nullable=True) - precipitation = Column(Float, nullable=True) - humidity = Column(Float, nullable=True) - wind_speed = Column(Float, nullable=True) - pressure = Column(Float, nullable=True) - description = Column(String(200), nullable=True) - - # Metadata - source = Column(String(50), nullable=False) - raw_data = Column(JSONB, nullable=True) - - # Timestamps - created_at = Column(DateTime(timezone=True), default=datetime.utcnow) - updated_at = Column(DateTime(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow) - - # Composite index for fast queries - __table_args__ = ( - Index('idx_city_weather_lookup', 'city_id', 'date'), - ) -``` - -**File:** `services/external/app/models/city_traffic.py` - -```python -# services/external/app/models/city_traffic.py -""" -City Traffic Data Model - Shared city-based traffic storage -""" - -from sqlalchemy import Column, String, Integer, Float, DateTime, Text, Index -from sqlalchemy.dialects.postgresql import UUID, JSONB -from datetime import datetime -import uuid - -from app.core.database import Base - - -class CityTrafficData(Base): - """City-based historical traffic data""" - - __tablename__ = "city_traffic_data" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - city_id = Column(String(50), nullable=False, index=True) - date = Column(DateTime(timezone=True), nullable=False, index=True) - - # Traffic metrics - traffic_volume = Column(Integer, nullable=True) - pedestrian_count = Column(Integer, nullable=True) - congestion_level = Column(String(20), nullable=True) - average_speed = Column(Float, nullable=True) - - # Metadata - source = Column(String(50), nullable=False) - raw_data = Column(JSONB, nullable=True) - - # Timestamps - created_at = Column(DateTime(timezone=True), default=datetime.utcnow) - updated_at = Column(DateTime(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow) - - # Composite index for fast queries - __table_args__ = ( - Index('idx_city_traffic_lookup', 'city_id', 'date'), - ) -``` - -### 2.4 Redis Cache Layer - -**File:** `services/external/app/cache/redis_cache.py` - -```python -# services/external/app/cache/redis_cache.py -""" -Redis cache layer for fast training data access -""" - -from typing import List, Dict, Any, Optional -import json -from datetime import datetime, timedelta -import structlog -import redis.asyncio as redis - -from app.core.config import settings - -logger = structlog.get_logger() - - -class ExternalDataCache: - """Redis cache for external data service""" - - def __init__(self): - self.redis_client = redis.from_url( - settings.REDIS_URL, - encoding="utf-8", - decode_responses=True - ) - self.ttl = 86400 * 7 # 7 days - - # ============= WEATHER CACHE ============= - - def _weather_cache_key( - self, - city_id: str, - start_date: datetime, - end_date: datetime - ) -> str: - """Generate cache key for weather data""" - return f"weather:{city_id}:{start_date.date()}:{end_date.date()}" - - async def get_cached_weather( - self, - city_id: str, - start_date: datetime, - end_date: datetime - ) -> Optional[List[Dict[str, Any]]]: - """Get cached weather data""" - try: - key = self._weather_cache_key(city_id, start_date, end_date) - cached = await self.redis_client.get(key) - - if cached: - logger.debug("Weather cache hit", city_id=city_id, key=key) - return json.loads(cached) - - logger.debug("Weather cache miss", city_id=city_id, key=key) - return None - - except Exception as e: - logger.error("Error reading weather cache", error=str(e)) - return None - - async def set_cached_weather( - self, - city_id: str, - start_date: datetime, - end_date: datetime, - data: List[Dict[str, Any]] - ): - """Set cached weather data""" - try: - key = self._weather_cache_key(city_id, start_date, end_date) - - # Serialize datetime objects - serializable_data = [] - for record in data: - record_copy = record.copy() - if isinstance(record_copy.get('date'), datetime): - record_copy['date'] = record_copy['date'].isoformat() - serializable_data.append(record_copy) - - await self.redis_client.setex( - key, - self.ttl, - json.dumps(serializable_data) - ) - - logger.debug("Weather data cached", city_id=city_id, records=len(data)) - - except Exception as e: - logger.error("Error caching weather data", error=str(e)) - - # ============= TRAFFIC CACHE ============= - - def _traffic_cache_key( - self, - city_id: str, - start_date: datetime, - end_date: datetime - ) -> str: - """Generate cache key for traffic data""" - return f"traffic:{city_id}:{start_date.date()}:{end_date.date()}" - - async def get_cached_traffic( - self, - city_id: str, - start_date: datetime, - end_date: datetime - ) -> Optional[List[Dict[str, Any]]]: - """Get cached traffic data""" - try: - key = self._traffic_cache_key(city_id, start_date, end_date) - cached = await self.redis_client.get(key) - - if cached: - logger.debug("Traffic cache hit", city_id=city_id, key=key) - return json.loads(cached) - - logger.debug("Traffic cache miss", city_id=city_id, key=key) - return None - - except Exception as e: - logger.error("Error reading traffic cache", error=str(e)) - return None - - async def set_cached_traffic( - self, - city_id: str, - start_date: datetime, - end_date: datetime, - data: List[Dict[str, Any]] - ): - """Set cached traffic data""" - try: - key = self._traffic_cache_key(city_id, start_date, end_date) - - # Serialize datetime objects - serializable_data = [] - for record in data: - record_copy = record.copy() - if isinstance(record_copy.get('date'), datetime): - record_copy['date'] = record_copy['date'].isoformat() - serializable_data.append(record_copy) - - await self.redis_client.setex( - key, - self.ttl, - json.dumps(serializable_data) - ) - - logger.debug("Traffic data cached", city_id=city_id, records=len(data)) - - except Exception as e: - logger.error("Error caching traffic data", error=str(e)) - - async def invalidate_city_cache(self, city_id: str): - """Invalidate all cache entries for a city""" - try: - pattern = f"*:{city_id}:*" - async for key in self.redis_client.scan_iter(match=pattern): - await self.redis_client.delete(key) - - logger.info("City cache invalidated", city_id=city_id) - - except Exception as e: - logger.error("Error invalidating cache", error=str(e)) -``` - ---- - -## Part 3: Kubernetes Manifests - -### 3.1 Init Job - Initial Data Load - -**File:** `infrastructure/kubernetes/external/init-job.yaml` - -```yaml -# infrastructure/kubernetes/external/init-job.yaml -apiVersion: batch/v1 -kind: Job -metadata: - name: external-data-init - namespace: bakery-ia - labels: - app: external-service - component: data-initialization -spec: - ttlSecondsAfterFinished: 86400 # Clean up after 1 day - backoffLimit: 3 - template: - metadata: - labels: - app: external-service - job: data-init - spec: - restartPolicy: OnFailure - - initContainers: - # Wait for database to be ready - - name: wait-for-db - image: postgres:15-alpine - command: - - sh - - -c - - | - until pg_isready -h external-db -p 5432 -U external_user; do - echo "Waiting for database..." - sleep 2 - done - echo "Database is ready" - env: - - name: PGPASSWORD - valueFrom: - secretKeyRef: - name: external-db-secret - key: password - - containers: - - name: data-loader - image: bakery-ia/external-service:latest - imagePullPolicy: Always - - command: - - python - - -m - - app.jobs.initialize_data - - args: - - "--months=24" - - "--log-level=INFO" - - env: - # Database - - name: DATABASE_URL - valueFrom: - secretKeyRef: - name: external-db-secret - key: url - - # Redis - - name: REDIS_URL - valueFrom: - configMapKeyRef: - name: external-config - key: redis-url - - # API Keys - - name: AEMET_API_KEY - valueFrom: - secretKeyRef: - name: external-api-keys - key: aemet-key - - - name: MADRID_OPENDATA_API_KEY - valueFrom: - secretKeyRef: - name: external-api-keys - key: madrid-key - - # Job configuration - - name: JOB_MODE - value: "initialize" - - - name: LOG_LEVEL - value: "INFO" - - resources: - requests: - memory: "1Gi" - cpu: "500m" - limits: - memory: "2Gi" - cpu: "1000m" - - volumeMounts: - - name: config - mountPath: /app/config - - volumes: - - name: config - configMap: - name: external-config -``` - -### 3.2 Monthly CronJob - Data Rotation - -**File:** `infrastructure/kubernetes/external/cronjob.yaml` - -```yaml -# infrastructure/kubernetes/external/cronjob.yaml -apiVersion: batch/v1 -kind: CronJob -metadata: - name: external-data-rotation - namespace: bakery-ia - labels: - app: external-service - component: data-rotation -spec: - # Run on 1st of each month at 2:00 AM UTC - schedule: "0 2 1 * *" - - # Keep last 3 successful jobs for debugging - successfulJobsHistoryLimit: 3 - failedJobsHistoryLimit: 3 - - # Don't start new job if previous is still running - concurrencyPolicy: Forbid - - jobTemplate: - metadata: - labels: - app: external-service - job: data-rotation - spec: - ttlSecondsAfterFinished: 172800 # 2 days - backoffLimit: 2 - - template: - metadata: - labels: - app: external-service - cronjob: data-rotation - spec: - restartPolicy: OnFailure - - containers: - - name: data-rotator - image: bakery-ia/external-service:latest - imagePullPolicy: Always - - command: - - python - - -m - - app.jobs.rotate_data - - args: - - "--log-level=INFO" - - "--notify-slack=true" - - env: - # Database - - name: DATABASE_URL - valueFrom: - secretKeyRef: - name: external-db-secret - key: url - - # Redis - - name: REDIS_URL - valueFrom: - configMapKeyRef: - name: external-config - key: redis-url - - # API Keys - - name: AEMET_API_KEY - valueFrom: - secretKeyRef: - name: external-api-keys - key: aemet-key - - - name: MADRID_OPENDATA_API_KEY - valueFrom: - secretKeyRef: - name: external-api-keys - key: madrid-key - - # Slack notification - - name: SLACK_WEBHOOK_URL - valueFrom: - secretKeyRef: - name: slack-secrets - key: webhook-url - optional: true - - # Job configuration - - name: JOB_MODE - value: "rotate" - - - name: LOG_LEVEL - value: "INFO" - - resources: - requests: - memory: "512Mi" - cpu: "250m" - limits: - memory: "1Gi" - cpu: "500m" - - volumeMounts: - - name: config - mountPath: /app/config - - volumes: - - name: config - configMap: - name: external-config -``` - -### 3.3 Main Service Deployment - -**File:** `infrastructure/kubernetes/external/deployment.yaml` - -```yaml -# infrastructure/kubernetes/external/deployment.yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: external-service - namespace: bakery-ia - labels: - app: external-service - version: "2.0" -spec: - replicas: 2 - - selector: - matchLabels: - app: external-service - - template: - metadata: - labels: - app: external-service - version: "2.0" - spec: - # Wait for init job to complete before deploying - initContainers: - - name: check-data-initialized - image: postgres:15-alpine - command: - - sh - - -c - - | - echo "Checking if data initialization is complete..." - until psql "$DATABASE_URL" -c "SELECT COUNT(*) FROM city_weather_data LIMIT 1;" > /dev/null 2>&1; do - echo "Waiting for initial data load..." - sleep 10 - done - echo "Data is initialized" - env: - - name: DATABASE_URL - valueFrom: - secretKeyRef: - name: external-db-secret - key: url - - containers: - - name: external-api - image: bakery-ia/external-service:latest - imagePullPolicy: Always - - ports: - - name: http - containerPort: 8000 - protocol: TCP - - env: - # Database - - name: DATABASE_URL - valueFrom: - secretKeyRef: - name: external-db-secret - key: url - - # Redis - - name: REDIS_URL - valueFrom: - configMapKeyRef: - name: external-config - key: redis-url - - # API Keys - - name: AEMET_API_KEY - valueFrom: - secretKeyRef: - name: external-api-keys - key: aemet-key - - - name: MADRID_OPENDATA_API_KEY - valueFrom: - secretKeyRef: - name: external-api-keys - key: madrid-key - - # Service config - - name: LOG_LEVEL - value: "INFO" - - - name: CORS_ORIGINS - value: "*" - - # Readiness probe - checks if data is available - readinessProbe: - httpGet: - path: /health/ready - port: http - initialDelaySeconds: 10 - periodSeconds: 5 - timeoutSeconds: 3 - failureThreshold: 3 - - # Liveness probe - livenessProbe: - httpGet: - path: /health/live - port: http - initialDelaySeconds: 30 - periodSeconds: 10 - timeoutSeconds: 3 - failureThreshold: 3 - - resources: - requests: - memory: "256Mi" - cpu: "100m" - limits: - memory: "512Mi" - cpu: "500m" - - volumeMounts: - - name: config - mountPath: /app/config - - volumes: - - name: config - configMap: - name: external-config -``` - -### 3.4 ConfigMap and Secrets - -**File:** `infrastructure/kubernetes/external/configmap.yaml` - -```yaml -# infrastructure/kubernetes/external/configmap.yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: external-config - namespace: bakery-ia -data: - redis-url: "redis://external-redis:6379/0" - - # City configuration (can be overridden) - enabled-cities: "madrid" - - # Data retention - retention-months: "24" - - # Cache TTL - cache-ttl-days: "7" -``` - -**File:** `infrastructure/kubernetes/external/secrets.yaml` (template) - -```yaml -# infrastructure/kubernetes/external/secrets.yaml -# NOTE: In production, use sealed-secrets or external secrets operator -apiVersion: v1 -kind: Secret -metadata: - name: external-api-keys - namespace: bakery-ia -type: Opaque -stringData: - aemet-key: "YOUR_AEMET_API_KEY_HERE" - madrid-key: "YOUR_MADRID_OPENDATA_KEY_HERE" ---- -apiVersion: v1 -kind: Secret -metadata: - name: external-db-secret - namespace: bakery-ia -type: Opaque -stringData: - url: "postgresql+asyncpg://external_user:password@external-db:5432/external_db" - password: "YOUR_DB_PASSWORD_HERE" -``` - -### 3.5 Job Scripts - -**File:** `services/external/app/jobs/initialize_data.py` - -```python -# services/external/app/jobs/initialize_data.py -""" -Kubernetes Init Job - Initialize 24-month historical data -""" - -import asyncio -import argparse -import sys -import structlog - -from app.ingestion.ingestion_manager import DataIngestionManager -from app.core.database import database_manager - -logger = structlog.get_logger() - - -async def main(months: int = 24): - """Initialize historical data for all enabled cities""" - logger.info("Starting data initialization job", months=months) - - try: - # Initialize database - await database_manager.initialize() - - # Run ingestion - manager = DataIngestionManager() - success = await manager.initialize_all_cities(months=months) - - if success: - logger.info("✅ Data initialization completed successfully") - sys.exit(0) - else: - logger.error("❌ Data initialization failed") - sys.exit(1) - - except Exception as e: - logger.error("❌ Fatal error during initialization", error=str(e)) - sys.exit(1) - finally: - await database_manager.close() - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Initialize historical data") - parser.add_argument("--months", type=int, default=24, help="Number of months to load") - parser.add_argument("--log-level", default="INFO", help="Log level") - - args = parser.parse_args() - - # Configure logging - structlog.configure( - wrapper_class=structlog.make_filtering_bound_logger(args.log_level) - ) - - asyncio.run(main(months=args.months)) -``` - -**File:** `services/external/app/jobs/rotate_data.py` - -```python -# services/external/app/jobs/rotate_data.py -""" -Kubernetes CronJob - Monthly data rotation (24-month window) -""" - -import asyncio -import argparse -import sys -import structlog - -from app.ingestion.ingestion_manager import DataIngestionManager -from app.core.database import database_manager - -logger = structlog.get_logger() - - -async def main(): - """Rotate 24-month data window""" - logger.info("Starting monthly data rotation job") - - try: - # Initialize database - await database_manager.initialize() - - # Run rotation - manager = DataIngestionManager() - await manager.rotate_monthly_data() - - logger.info("✅ Data rotation completed successfully") - sys.exit(0) - - except Exception as e: - logger.error("❌ Fatal error during rotation", error=str(e)) - sys.exit(1) - finally: - await database_manager.close() - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Rotate historical data") - parser.add_argument("--log-level", default="INFO", help="Log level") - parser.add_argument("--notify-slack", type=bool, default=False, help="Send Slack notification") - - args = parser.parse_args() - - # Configure logging - structlog.configure( - wrapper_class=structlog.make_filtering_bound_logger(args.log_level) - ) - - asyncio.run(main()) -``` - ---- - -## Part 4: Updated API Endpoints - -### 4.1 New City-Based Endpoints - -**File:** `services/external/app/api/city_operations.py` - -```python -# services/external/app/api/city_operations.py -""" -City Operations API - New endpoints for city-based data access -""" - -from fastapi import APIRouter, Depends, HTTPException, Query, Path -from typing import List -from datetime import datetime -from uuid import UUID -import structlog - -from app.schemas.city_data import CityInfoResponse, DataAvailabilityResponse -from app.schemas.weather import WeatherDataResponse -from app.schemas.traffic import TrafficDataResponse -from app.registry.city_registry import CityRegistry -from app.registry.geolocation_mapper import GeolocationMapper -from app.repositories.city_data_repository import CityDataRepository -from app.cache.redis_cache import ExternalDataCache -from shared.routing.route_builder import RouteBuilder -from sqlalchemy.ext.asyncio import AsyncSession -from app.core.database import get_db - -route_builder = RouteBuilder('external') -router = APIRouter(tags=["city-operations"]) -logger = structlog.get_logger() - - -@router.get( - route_builder.build_base_route("cities"), - response_model=List[CityInfoResponse] -) -async def list_supported_cities(): - """List all enabled cities with data availability""" - registry = CityRegistry() - cities = registry.get_enabled_cities() - - return [ - CityInfoResponse( - city_id=city.city_id, - name=city.name, - country=city.country.value, - latitude=city.latitude, - longitude=city.longitude, - radius_km=city.radius_km, - weather_provider=city.weather_provider.value, - traffic_provider=city.traffic_provider.value, - enabled=city.enabled - ) - for city in cities - ] - - -@router.get( - route_builder.build_operations_route("cities/{city_id}/availability"), - response_model=DataAvailabilityResponse -) -async def get_city_data_availability( - city_id: str = Path(..., description="City ID"), - db: AsyncSession = Depends(get_db) -): - """Get data availability for a specific city""" - registry = CityRegistry() - city = registry.get_city(city_id) - - if not city: - raise HTTPException(status_code=404, detail="City not found") - - repo = CityDataRepository(db) - - # Query min/max dates - weather_stmt = await db.execute( - "SELECT MIN(date), MAX(date), COUNT(*) FROM city_weather_data WHERE city_id = :city_id", - {"city_id": city_id} - ) - weather_min, weather_max, weather_count = weather_stmt.fetchone() - - traffic_stmt = await db.execute( - "SELECT MIN(date), MAX(date), COUNT(*) FROM city_traffic_data WHERE city_id = :city_id", - {"city_id": city_id} - ) - traffic_min, traffic_max, traffic_count = traffic_stmt.fetchone() - - return DataAvailabilityResponse( - city_id=city_id, - city_name=city.name, - weather_available=weather_count > 0, - weather_start_date=weather_min.isoformat() if weather_min else None, - weather_end_date=weather_max.isoformat() if weather_max else None, - weather_record_count=weather_count, - traffic_available=traffic_count > 0, - traffic_start_date=traffic_min.isoformat() if traffic_min else None, - traffic_end_date=traffic_max.isoformat() if traffic_max else None, - traffic_record_count=traffic_count - ) - - -@router.get( - route_builder.build_operations_route("historical-weather-optimized"), - response_model=List[WeatherDataResponse] -) -async def get_historical_weather_optimized( - tenant_id: UUID = Path(..., description="Tenant ID"), - latitude: float = Query(..., description="Latitude"), - longitude: float = Query(..., description="Longitude"), - start_date: datetime = Query(..., description="Start date"), - end_date: datetime = Query(..., description="End date"), - db: AsyncSession = Depends(get_db) -): - """ - Get historical weather data using city-based cached data - This is the FAST endpoint for training service - """ - try: - # Map tenant location to city - mapper = GeolocationMapper() - mapping = mapper.map_tenant_to_city(latitude, longitude) - - if not mapping: - raise HTTPException( - status_code=404, - detail="No supported city found for this location" - ) - - city, distance = mapping - - logger.info( - "Fetching historical weather from cache", - tenant_id=tenant_id, - city=city.name, - distance_km=round(distance, 2) - ) - - # Try cache first - cache = ExternalDataCache() - cached_data = await cache.get_cached_weather( - city.city_id, start_date, end_date - ) - - if cached_data: - logger.info("Weather cache hit", records=len(cached_data)) - return cached_data - - # Cache miss - query database - repo = CityDataRepository(db) - db_records = await repo.get_weather_by_city_and_range( - city.city_id, start_date, end_date - ) - - # Convert to response format - response_data = [ - WeatherDataResponse( - id=str(record.id), - location_id=f"{city.city_id}_{record.date.date()}", - date=record.date.isoformat(), - temperature=record.temperature, - precipitation=record.precipitation, - humidity=record.humidity, - wind_speed=record.wind_speed, - pressure=record.pressure, - description=record.description, - source=record.source, - created_at=record.created_at.isoformat(), - updated_at=record.updated_at.isoformat() - ) - for record in db_records - ] - - # Store in cache for next time - await cache.set_cached_weather( - city.city_id, start_date, end_date, response_data - ) - - logger.info( - "Historical weather data retrieved", - records=len(response_data), - source="database" - ) - - return response_data - - except HTTPException: - raise - except Exception as e: - logger.error("Error fetching historical weather", error=str(e)) - raise HTTPException(status_code=500, detail="Internal server error") - - -@router.get( - route_builder.build_operations_route("historical-traffic-optimized"), - response_model=List[TrafficDataResponse] -) -async def get_historical_traffic_optimized( - tenant_id: UUID = Path(..., description="Tenant ID"), - latitude: float = Query(..., description="Latitude"), - longitude: float = Query(..., description="Longitude"), - start_date: datetime = Query(..., description="Start date"), - end_date: datetime = Query(..., description="End date"), - db: AsyncSession = Depends(get_db) -): - """ - Get historical traffic data using city-based cached data - This is the FAST endpoint for training service - """ - try: - # Map tenant location to city - mapper = GeolocationMapper() - mapping = mapper.map_tenant_to_city(latitude, longitude) - - if not mapping: - raise HTTPException( - status_code=404, - detail="No supported city found for this location" - ) - - city, distance = mapping - - logger.info( - "Fetching historical traffic from cache", - tenant_id=tenant_id, - city=city.name, - distance_km=round(distance, 2) - ) - - # Try cache first - cache = ExternalDataCache() - cached_data = await cache.get_cached_traffic( - city.city_id, start_date, end_date - ) - - if cached_data: - logger.info("Traffic cache hit", records=len(cached_data)) - return cached_data - - # Cache miss - query database - repo = CityDataRepository(db) - db_records = await repo.get_traffic_by_city_and_range( - city.city_id, start_date, end_date - ) - - # Convert to response format - response_data = [ - TrafficDataResponse( - date=record.date.isoformat(), - traffic_volume=record.traffic_volume, - pedestrian_count=record.pedestrian_count, - congestion_level=record.congestion_level, - average_speed=record.average_speed, - source=record.source - ) - for record in db_records - ] - - # Store in cache for next time - await cache.set_cached_traffic( - city.city_id, start_date, end_date, response_data - ) - - logger.info( - "Historical traffic data retrieved", - records=len(response_data), - source="database" - ) - - return response_data - - except HTTPException: - raise - except Exception as e: - logger.error("Error fetching historical traffic", error=str(e)) - raise HTTPException(status_code=500, detail="Internal server error") -``` - -### 4.2 Schema Definitions - -**File:** `services/external/app/schemas/city_data.py` - -```python -# services/external/app/schemas/city_data.py -""" -City Data Schemas - New response types for city-based operations -""" - -from pydantic import BaseModel, Field -from typing import Optional - - -class CityInfoResponse(BaseModel): - """Information about a supported city""" - city_id: str - name: str - country: str - latitude: float - longitude: float - radius_km: float - weather_provider: str - traffic_provider: str - enabled: bool - - -class DataAvailabilityResponse(BaseModel): - """Data availability for a city""" - city_id: str - city_name: str - - # Weather availability - weather_available: bool - weather_start_date: Optional[str] = None - weather_end_date: Optional[str] = None - weather_record_count: int = 0 - - # Traffic availability - traffic_available: bool - traffic_start_date: Optional[str] = None - traffic_end_date: Optional[str] = None - traffic_record_count: int = 0 -``` - ---- - -## Part 5: Frontend Integration - -### 5.1 Updated TypeScript Types - -**File:** `frontend/src/api/types/external.ts` (additions) - -```typescript -// frontend/src/api/types/external.ts -// ADD TO EXISTING FILE - -// ================================================================ -// CITY-BASED DATA TYPES (NEW) -// ================================================================ - -/** - * City information response - * Backend: services/external/app/schemas/city_data.py:CityInfoResponse - */ -export interface CityInfoResponse { - city_id: string; - name: string; - country: string; - latitude: number; - longitude: number; - radius_km: number; - weather_provider: string; - traffic_provider: string; - enabled: boolean; -} - -/** - * Data availability response - * Backend: services/external/app/schemas/city_data.py:DataAvailabilityResponse - */ -export interface DataAvailabilityResponse { - city_id: string; - city_name: string; - - // Weather availability - weather_available: boolean; - weather_start_date: string | null; - weather_end_date: string | null; - weather_record_count: number; - - // Traffic availability - traffic_available: boolean; - traffic_start_date: string | null; - traffic_end_date: string | null; - traffic_record_count: number; -} -``` - -### 5.2 API Service Methods - -**File:** `frontend/src/api/services/external.ts` (new file) - -```typescript -// frontend/src/api/services/external.ts -/** - * External Data API Service - * Handles weather and traffic data operations - */ - -import { apiClient } from '../client'; -import type { - CityInfoResponse, - DataAvailabilityResponse, - WeatherDataResponse, - TrafficDataResponse, - HistoricalWeatherRequest, - HistoricalTrafficRequest, -} from '../types/external'; - -class ExternalDataService { - /** - * List all supported cities - */ - async listCities(): Promise { - const response = await apiClient.get( - '/api/v1/external/cities' - ); - return response.data; - } - - /** - * Get data availability for a specific city - */ - async getCityAvailability(cityId: string): Promise { - const response = await apiClient.get( - `/api/v1/external/operations/cities/${cityId}/availability` - ); - return response.data; - } - - /** - * Get historical weather data (optimized city-based endpoint) - */ - async getHistoricalWeatherOptimized( - tenantId: string, - params: { - latitude: number; - longitude: number; - start_date: string; - end_date: string; - } - ): Promise { - const response = await apiClient.get( - `/api/v1/tenants/${tenantId}/external/operations/historical-weather-optimized`, - { params } - ); - return response.data; - } - - /** - * Get historical traffic data (optimized city-based endpoint) - */ - async getHistoricalTrafficOptimized( - tenantId: string, - params: { - latitude: number; - longitude: number; - start_date: string; - end_date: string; - } - ): Promise { - const response = await apiClient.get( - `/api/v1/tenants/${tenantId}/external/operations/historical-traffic-optimized`, - { params } - ); - return response.data; - } - - /** - * Legacy: Get historical weather (non-optimized) - * @deprecated Use getHistoricalWeatherOptimized instead - */ - async getHistoricalWeather( - tenantId: string, - request: HistoricalWeatherRequest - ): Promise { - const response = await apiClient.post( - `/api/v1/tenants/${tenantId}/external/operations/weather/historical`, - request - ); - return response.data; - } - - /** - * Legacy: Get historical traffic (non-optimized) - * @deprecated Use getHistoricalTrafficOptimized instead - */ - async getHistoricalTraffic( - tenantId: string, - request: HistoricalTrafficRequest - ): Promise { - const response = await apiClient.post( - `/api/v1/tenants/${tenantId}/external/operations/traffic/historical`, - request - ); - return response.data; - } -} - -export const externalDataService = new ExternalDataService(); -export default externalDataService; -``` - -### 5.3 Contract Synchronization Process - -**Document:** Frontend API contract sync workflow - -```markdown -# Frontend-Backend Contract Synchronization - -## When to Update - -Trigger frontend updates when ANY of these occur: -1. New API endpoint added -2. Request/response schema changed -3. Enum values modified -4. Required/optional fields changed - -## Process - -### Step 1: Detect Backend Changes -```bash -# Monitor these files for changes: -services/external/app/schemas/*.py -services/external/app/api/*.py -``` - -### Step 2: Update TypeScript Types -```bash -# Location: frontend/src/api/types/external.ts -# 1. Compare backend Pydantic models with TS interfaces -# 2. Add/update interfaces to match -# 3. Add JSDoc comments with backend file references -``` - -### Step 3: Update API Service Methods -```bash -# Location: frontend/src/api/services/external.ts -# 1. Add new methods for new endpoints -# 2. Update method signatures for schema changes -# 3. Update endpoint URLs to match route_builder output -``` - -### Step 4: Validate -```bash -# Run type check -npm run type-check - -# Test compilation -npm run build -``` - -### Step 5: Integration Test -```bash -# Test actual API calls -npm run test:integration -``` - -## Example: Adding New Endpoint - -**Backend (Python):** -```python -@router.get("/cities/{city_id}/stats", response_model=CityStatsResponse) -async def get_city_stats(city_id: str): - ... -``` - -**Frontend Steps:** -1. Add type: `frontend/src/api/types/external.ts` - ```typescript - export interface CityStatsResponse { - city_id: string; - total_records: number; - last_updated: string; - } - ``` - -2. Add method: `frontend/src/api/services/external.ts` - ```typescript - async getCityStats(cityId: string): Promise { - const response = await apiClient.get( - `/api/v1/external/cities/${cityId}/stats` - ); - return response.data; - } - ``` - -3. Verify type safety: - ```typescript - const stats = await externalDataService.getCityStats('madrid'); - console.log(stats.total_records); // TypeScript autocomplete works! - ``` - -## Automation (Future) - -Consider implementing: -- OpenAPI spec generation from FastAPI -- TypeScript type generation from OpenAPI -- Contract testing (Pact, etc.) -``` - ---- - -## Part 6: Migration Plan - -### 6.1 Migration Phases - -#### Phase 1: Infrastructure Setup (Week 1) -- ✅ Create new database tables (`city_weather_data`, `city_traffic_data`) -- ✅ Deploy Redis for caching -- ✅ Create Kubernetes secrets and configmaps -- ✅ Deploy init job (without running) - -#### Phase 2: Code Implementation (Week 2-3) -- ✅ Implement city registry and geolocation mapper -- ✅ Implement Madrid adapter (reuse existing clients) -- ✅ Implement ingestion manager -- ✅ Implement city data repository -- ✅ Implement Redis cache layer -- ✅ Create init and rotation job scripts - -#### Phase 3: Initial Data Load (Week 4) -- ✅ Test init job in staging -- ✅ Run init job in production (24-month load) -- ✅ Validate data integrity -- ✅ Warm Redis cache - -#### Phase 4: API Migration (Week 5) -- ✅ Deploy new city-based endpoints -- ✅ Update training service to use optimized endpoints -- ✅ Update frontend types and services -- ✅ Run parallel (old + new endpoints) - -#### Phase 5: Cutover (Week 6) -- ✅ Switch training service to new endpoints -- ✅ Monitor performance (should be <100ms) -- ✅ Verify cache hit rates -- ✅ Deprecate old endpoints - -#### Phase 6: Cleanup (Week 7) -- ✅ Remove old per-tenant data fetching code -- ✅ Schedule first monthly CronJob -- ✅ Document new architecture -- ✅ Remove backward compatibility code - -### 6.2 Rollback Plan - -If issues occur during cutover: - -```yaml -# Rollback steps -1. Update training service config: - USE_OPTIMIZED_EXTERNAL_ENDPOINTS: false - -2. Traffic routes back to old endpoints - -3. New infrastructure remains running (no data loss) - -4. Investigate issues, fix, retry cutover -``` - -### 6.3 Testing Strategy - -**Unit Tests:** -```python -# tests/unit/test_geolocation_mapper.py -def test_map_tenant_to_madrid(): - mapper = GeolocationMapper() - city, distance = mapper.map_tenant_to_city(40.42, -3.70) - assert city.city_id == "madrid" - assert distance < 5.0 -``` - -**Integration Tests:** -```python -# tests/integration/test_ingestion.py -async def test_initialize_city_data(): - manager = DataIngestionManager() - success = await manager.initialize_city( - "madrid", - datetime(2023, 1, 1), - datetime(2023, 1, 31) - ) - assert success -``` - -**Performance Tests:** -```python -# tests/performance/test_cache_performance.py -async def test_historical_weather_response_time(): - start = time.time() - data = await get_historical_weather_optimized(...) - duration = time.time() - start - assert duration < 0.1 # <100ms - assert len(data) > 0 -``` - ---- - -## Part 7: Observability & Monitoring - -### 7.1 Metrics to Track - -```python -# services/external/app/metrics/city_metrics.py -from prometheus_client import Counter, Histogram, Gauge - -# Data ingestion metrics -ingestion_records_total = Counter( - 'external_ingestion_records_total', - 'Total records ingested', - ['city_id', 'data_type'] -) - -ingestion_duration_seconds = Histogram( - 'external_ingestion_duration_seconds', - 'Ingestion duration', - ['city_id', 'data_type'] -) - -# Cache metrics -cache_hit_total = Counter( - 'external_cache_hit_total', - 'Cache hits', - ['data_type'] -) - -cache_miss_total = Counter( - 'external_cache_miss_total', - 'Cache misses', - ['data_type'] -) - -# Data availability -city_data_records_gauge = Gauge( - 'external_city_data_records', - 'Current record count per city', - ['city_id', 'data_type'] -) - -# API performance -api_request_duration_seconds = Histogram( - 'external_api_request_duration_seconds', - 'API request duration', - ['endpoint', 'city_id'] -) -``` - -### 7.2 Logging Strategy - -```python -# Structured logging examples - -# Ingestion -logger.info( - "City data initialization started", - city=city.name, - start_date=start_date.isoformat(), - end_date=end_date.isoformat(), - expected_records=estimated_count -) - -# Cache -logger.info( - "Cache hit", - cache_key=key, - city_id=city_id, - hit_rate=hit_rate, - response_time_ms=duration * 1000 -) - -# API -logger.info( - "Historical data request", - tenant_id=tenant_id, - city=city.name, - distance_km=distance, - date_range_days=(end_date - start_date).days, - records_returned=len(data), - source="cache" if cached else "database" -) -``` - -### 7.3 Alerts - -```yaml -# Prometheus alert rules -groups: - - name: external_data_service - interval: 30s - rules: - # Data freshness - - alert: ExternalDataStale - expr: | - (time() - external_city_data_last_update_timestamp) > 86400 * 7 - for: 1h - labels: - severity: warning - annotations: - summary: "City data not updated in 7 days" - - # Cache health - - alert: ExternalCacheHitRateLow - expr: | - rate(external_cache_hit_total[5m]) / - (rate(external_cache_hit_total[5m]) + rate(external_cache_miss_total[5m])) < 0.7 - for: 15m - labels: - severity: warning - annotations: - summary: "Cache hit rate below 70%" - - # Ingestion failures - - alert: ExternalIngestionFailed - expr: | - external_ingestion_failures_total > 0 - for: 5m - labels: - severity: critical - annotations: - summary: "Data ingestion job failed" -``` - ---- - -## Conclusion - -This architecture redesign delivers: - -1. **✅ Centralized data management** - No more per-tenant redundant fetching -2. **✅ Multi-city scalability** - Easy to add Valencia, Barcelona, etc. -3. **✅ Sub-100ms training data access** - Redis + PostgreSQL cache -4. **✅ Automated 24-month windows** - Kubernetes CronJobs handle rotation -5. **✅ Zero downtime deployment** - Init job ensures data before service start -6. **✅ Observable & maintainable** - Metrics, logs, alerts built-in -7. **✅ Type-safe frontend integration** - Strict contract sync process - -**Next Steps:** -1. Review and approve architecture -2. Begin Phase 1 (Infrastructure) -3. Implement in phases with rollback capability -4. Monitor performance improvements -5. Plan Valencia/Barcelona adapter implementations - ---- - -**Document Version:** 1.0 -**Last Updated:** 2025-10-07 -**Approved By:** [Pending Review] diff --git a/FRONTEND_ALIGNMENT_STRATEGY.md b/FRONTEND_ALIGNMENT_STRATEGY.md deleted file mode 100644 index 8897b5ad..00000000 --- a/FRONTEND_ALIGNMENT_STRATEGY.md +++ /dev/null @@ -1,757 +0,0 @@ -# 🎯 Frontend-Backend Alignment Strategy - -**Status:** Ready for Execution -**Last Updated:** 2025-10-05 -**Backend Structure:** Fully analyzed (14 services, 3-tier architecture) - ---- - -## 📋 Executive Summary - -The backend has been successfully refactored to follow a **consistent 3-tier architecture**: -- **ATOMIC** endpoints = Direct CRUD on models (e.g., `ingredients.py`, `production_batches.py`) -- **OPERATIONS** endpoints = Business workflows (e.g., `inventory_operations.py`, `supplier_operations.py`) -- **ANALYTICS** endpoints = Reporting and insights (e.g., `analytics.py`) - -The frontend must now be updated to mirror this structure with **zero drift**. - ---- - -## 🏗️ Backend Service Structure - -### Complete Service Map - -| Service | ATOMIC Files | OPERATIONS Files | ANALYTICS Files | Other Files | -|---------|--------------|------------------|-----------------|-------------| -| **auth** | `users.py` | `auth_operations.py` | ❌ | `onboarding_progress.py` | -| **demo_session** | `demo_accounts.py`, `demo_sessions.py` | `demo_operations.py` | ❌ | `schemas.py` | -| **external** | `traffic_data.py`, `weather_data.py` | `external_operations.py` | ❌ | - | -| **forecasting** | `forecasts.py` | `forecasting_operations.py` | `analytics.py` | - | -| **inventory** | `ingredients.py`, `stock_entries.py`, `temperature_logs.py`, `transformations.py` | `inventory_operations.py`, `food_safety_operations.py` | `analytics.py`, `dashboard.py` | `food_safety_alerts.py`, `food_safety_compliance.py` | -| **notification** | `notifications.py` | `notification_operations.py` | `analytics.py` | - | -| **orders** | `orders.py`, `customers.py` | `order_operations.py`, `procurement_operations.py` | ❌ | - | -| **pos** | `configurations.py`, `transactions.py` | `pos_operations.py` | `analytics.py` | - | -| **production** | `production_batches.py`, `production_schedules.py` | `production_operations.py` | `analytics.py`, `production_dashboard.py` | - | -| **recipes** | `recipes.py`, `recipe_quality_configs.py` | `recipe_operations.py` | ❌ (in operations) | - | -| **sales** | `sales_records.py` | `sales_operations.py` | `analytics.py` | - | -| **suppliers** | `suppliers.py`, `deliveries.py`, `purchase_orders.py` | `supplier_operations.py` | `analytics.py` | - | -| **tenant** | `tenants.py`, `tenant_members.py` | `tenant_operations.py` | ❌ | `webhooks.py` | -| **training** | `models.py`, `training_jobs.py` | `training_operations.py` | ❌ | - | - ---- - -## 🎯 Frontend Refactoring Plan - -### Phase 1: Update TypeScript Types (`src/api/types/`) - -**Goal:** Ensure types match backend Pydantic schemas exactly. - -#### Priority Services (Start Here) -1. **inventory.ts** ✅ Already complex - verify alignment with: - - `ingredients.py` schemas - - `stock_entries.py` schemas - - `inventory_operations.py` request/response models - -2. **production.ts** - Map to: - - `ProductionBatchCreate`, `ProductionBatchUpdate`, `ProductionBatchResponse` - - `ProductionScheduleCreate`, `ProductionScheduleResponse` - - Operation-specific types from `production_operations.py` - -3. **sales.ts** - Map to: - - `SalesRecordCreate`, `SalesRecordUpdate`, `SalesRecordResponse` - - Import validation types from `sales_operations.py` - -4. **suppliers.ts** - Map to: - - `SupplierCreate`, `SupplierUpdate`, `SupplierResponse` - - `PurchaseOrderCreate`, `PurchaseOrderResponse` - - `DeliveryCreate`, `DeliveryUpdate`, `DeliveryResponse` - -5. **recipes.ts** - Map to: - - `RecipeCreate`, `RecipeUpdate`, `RecipeResponse` - - Quality config types - -#### Action Items -- [ ] Read backend `app/schemas/*.py` files for each service -- [ ] Compare with current `frontend/src/api/types/*.ts` -- [ ] Update/create types to match backend exactly -- [ ] Remove deprecated types for deleted endpoints -- [ ] Add JSDoc comments referencing backend schema files - ---- - -### Phase 2: Refactor Service Files (`src/api/services/`) - -**Goal:** Create clean service classes with ATOMIC, OPERATIONS, and ANALYTICS methods grouped logically. - -#### Current State -``` -frontend/src/api/services/ -├── inventory.ts ✅ Good structure, needs verification -├── production.ts ⚠️ Needs alignment check -├── sales.ts ⚠️ Needs alignment check -├── suppliers.ts ⚠️ Needs alignment check -├── recipes.ts ⚠️ Needs alignment check -├── forecasting.ts ⚠️ Needs alignment check -├── training.ts ⚠️ Needs alignment check -├── orders.ts ⚠️ Needs alignment check -├── foodSafety.ts ⚠️ May need merge with inventory -├── classification.ts ⚠️ Should be in inventory operations -├── transformations.ts ⚠️ Should be in inventory operations -├── inventoryDashboard.ts ⚠️ Should be in inventory analytics -└── ... (other services) -``` - -#### Target Structure (Example: Inventory Service) - -```typescript -// frontend/src/api/services/inventory.ts - -export class InventoryService { - private readonly baseUrl = '/tenants'; - - // ===== ATOMIC: Ingredients CRUD ===== - async createIngredient(tenantId: string, data: IngredientCreate): Promise - async getIngredient(tenantId: string, id: string): Promise - async listIngredients(tenantId: string, filters?: IngredientFilter): Promise - async updateIngredient(tenantId: string, id: string, data: IngredientUpdate): Promise - async softDeleteIngredient(tenantId: string, id: string): Promise - async hardDeleteIngredient(tenantId: string, id: string): Promise - - // ===== ATOMIC: Stock CRUD ===== - async createStock(tenantId: string, data: StockCreate): Promise - async getStock(tenantId: string, id: string): Promise - async listStock(tenantId: string, filters?: StockFilter): Promise> - async updateStock(tenantId: string, id: string, data: StockUpdate): Promise - async deleteStock(tenantId: string, id: string): Promise - - // ===== OPERATIONS: Stock Management ===== - async consumeStock(tenantId: string, data: StockConsumptionRequest): Promise - async getExpiringStock(tenantId: string, daysAhead: number): Promise - async getLowStock(tenantId: string): Promise - async getStockSummary(tenantId: string): Promise - - // ===== OPERATIONS: Classification ===== - async classifyProduct(tenantId: string, data: ProductClassificationRequest): Promise - async classifyBatch(tenantId: string, data: BatchClassificationRequest): Promise - - // ===== OPERATIONS: Food Safety ===== - async logTemperature(tenantId: string, data: TemperatureLogCreate): Promise - async getComplianceStatus(tenantId: string): Promise - async getFoodSafetyAlerts(tenantId: string): Promise - - // ===== ANALYTICS: Dashboard ===== - async getInventoryAnalytics(tenantId: string, dateRange?: DateRange): Promise - async getStockValueReport(tenantId: string): Promise - async getWasteAnalysis(tenantId: string, dateRange?: DateRange): Promise -} -``` - -#### Refactoring Rules - -1. **One Service = One Backend Service Domain** - - `inventoryService` → All `/tenants/{id}/inventory/*` endpoints - - `productionService` → All `/tenants/{id}/production/*` endpoints - -2. **Group Methods by Type** - - ATOMIC methods first (CRUD operations) - - OPERATIONS methods second (business logic) - - ANALYTICS methods last (reporting) - -3. **URL Construction Pattern** - ```typescript - // ATOMIC - `${this.baseUrl}/${tenantId}/inventory/ingredients` - - // OPERATIONS - `${this.baseUrl}/${tenantId}/inventory/operations/consume-stock` - - // ANALYTICS - `${this.baseUrl}/${tenantId}/inventory/analytics/waste-analysis` - ``` - -4. **No Inline API Calls** - - All `apiClient.get/post/put/delete` calls MUST be in service files - - Components/hooks should ONLY call service methods - -#### Service-by-Service Checklist - -- [ ] **inventory.ts** - Verify, add missing operations -- [ ] **production.ts** - Add batch/schedule operations, analytics -- [ ] **sales.ts** - Add import operations, analytics -- [ ] **suppliers.ts** - Split into supplier/PO/delivery methods -- [ ] **recipes.ts** - Add operations (duplicate, activate, feasibility) -- [ ] **forecasting.ts** - Add operations and analytics -- [ ] **training.ts** - Add training job operations -- [ ] **orders.ts** - Add order/procurement operations -- [ ] **auth.ts** - Add onboarding progress operations -- [ ] **tenant.ts** - Add tenant member operations -- [ ] **notification.ts** - Add notification operations -- [ ] **pos.ts** - Add POS configuration/transaction operations -- [ ] **external.ts** - Add traffic/weather data operations -- [ ] **demo.ts** - Add demo session operations - -#### Files to DELETE (Merge into main services) - -- [ ] ❌ `classification.ts` → Merge into `inventory.ts` (operations section) -- [ ] ❌ `transformations.ts` → Merge into `inventory.ts` (operations section) -- [ ] ❌ `inventoryDashboard.ts` → Merge into `inventory.ts` (analytics section) -- [ ] ❌ `foodSafety.ts` → Merge into `inventory.ts` (operations section) -- [ ] ❌ `dataImport.ts` → Merge into `sales.ts` (operations section) -- [ ] ❌ `qualityTemplates.ts` → Merge into `recipes.ts` (if still needed) -- [ ] ❌ `onboarding.ts` → Merge into `auth.ts` (operations section) -- [ ] ❌ `subscription.ts` → Merge into `tenant.ts` (operations section) - ---- - -### Phase 3: Update Hooks (`src/api/hooks/`) - -**Goal:** Create typed hooks that use updated service methods. - -#### Current State -``` -frontend/src/api/hooks/ -├── inventory.ts -├── production.ts -├── suppliers.ts -├── recipes.ts -├── forecasting.ts -├── training.ts -├── foodSafety.ts -├── inventoryDashboard.ts -├── qualityTemplates.ts -└── ... -``` - -#### Hook Naming Convention - -```typescript -// Query hooks (GET) -useIngredients(tenantId: string, filters?: IngredientFilter) -useIngredient(tenantId: string, ingredientId: string) -useLowStockIngredients(tenantId: string) - -// Mutation hooks (POST/PUT/DELETE) -useCreateIngredient() -useUpdateIngredient() -useDeleteIngredient() -useConsumeStock() -useClassifyProducts() -``` - -#### Hook Structure (React Query) - -```typescript -import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query'; -import { inventoryService } from '../services/inventory'; - -// Query Hook -export const useIngredients = (tenantId: string, filters?: IngredientFilter) => { - return useQuery({ - queryKey: ['ingredients', tenantId, filters], - queryFn: () => inventoryService.listIngredients(tenantId, filters), - enabled: !!tenantId, - }); -}; - -// Mutation Hook -export const useConsumeStock = () => { - const queryClient = useQueryClient(); - - return useMutation({ - mutationFn: ({ tenantId, data }: { tenantId: string; data: StockConsumptionRequest }) => - inventoryService.consumeStock(tenantId, data), - onSuccess: (_, { tenantId }) => { - queryClient.invalidateQueries({ queryKey: ['stock', tenantId] }); - queryClient.invalidateQueries({ queryKey: ['ingredients', tenantId] }); - }, - }); -}; -``` - -#### Action Items - -- [ ] Audit all hooks in `src/api/hooks/` -- [ ] Ensure each hook calls correct service method -- [ ] Update query keys to match new structure -- [ ] Add proper invalidation logic for mutations -- [ ] Remove hooks for deleted endpoints -- [ ] Merge duplicate hooks (e.g., `useFetchIngredients` + `useIngredients`) - -#### Files to DELETE (Merge into main hook files) - -- [ ] ❌ `foodSafety.ts` → Merge into `inventory.ts` -- [ ] ❌ `inventoryDashboard.ts` → Merge into `inventory.ts` -- [ ] ❌ `qualityTemplates.ts` → Merge into `recipes.ts` - ---- - -### Phase 4: Cross-Service Consistency - -**Goal:** Ensure naming and patterns are consistent across all services. - -#### Naming Conventions - -| Backend Pattern | Frontend Method | Hook Name | -|----------------|-----------------|-----------| -| `POST /ingredients` | `createIngredient()` | `useCreateIngredient()` | -| `GET /ingredients` | `listIngredients()` | `useIngredients()` | -| `GET /ingredients/{id}` | `getIngredient()` | `useIngredient()` | -| `PUT /ingredients/{id}` | `updateIngredient()` | `useUpdateIngredient()` | -| `DELETE /ingredients/{id}` | `deleteIngredient()` | `useDeleteIngredient()` | -| `POST /operations/consume-stock` | `consumeStock()` | `useConsumeStock()` | -| `GET /analytics/summary` | `getAnalyticsSummary()` | `useAnalyticsSummary()` | - -#### Query Parameter Mapping - -Backend query params should map to TypeScript filter objects: - -```typescript -// Backend: ?category=flour&is_low_stock=true&limit=50&offset=0 -// Frontend: -interface IngredientFilter { - category?: string; - product_type?: string; - is_active?: boolean; - is_low_stock?: boolean; - needs_reorder?: boolean; - search?: string; - limit?: number; - offset?: number; - order_by?: string; - order_direction?: 'asc' | 'desc'; -} -``` - ---- - -## 🧹 Cleanup & Verification - -### Step 1: Type Check - -```bash -cd frontend -npm run type-check -``` - -Fix all TypeScript errors related to: -- Missing types -- Incorrect method signatures -- Deprecated imports - -### Step 2: Search for Inline API Calls - -```bash -# Find direct axios/fetch calls in components -rg "apiClient\.(get|post|put|delete)" frontend/src/components --type ts -rg "axios\." frontend/src/components --type ts -rg "fetch\(" frontend/src/components --type ts -``` - -Move all found calls into appropriate service files. - -### Step 3: Delete Obsolete Files - -After verification, delete these files from git: - -```bash -# Service files to delete -git rm frontend/src/api/services/classification.ts -git rm frontend/src/api/services/transformations.ts -git rm frontend/src/api/services/inventoryDashboard.ts -git rm frontend/src/api/services/foodSafety.ts -git rm frontend/src/api/services/dataImport.ts -git rm frontend/src/api/services/qualityTemplates.ts -git rm frontend/src/api/services/onboarding.ts -git rm frontend/src/api/services/subscription.ts - -# Hook files to delete -git rm frontend/src/api/hooks/foodSafety.ts -git rm frontend/src/api/hooks/inventoryDashboard.ts -git rm frontend/src/api/hooks/qualityTemplates.ts - -# Types to verify (may need to merge, not delete) -# Check if still referenced before deleting -``` - -### Step 4: Update Imports - -Search for imports of deleted files: - -```bash -rg "from.*classification" frontend/src --type ts -rg "from.*transformations" frontend/src --type ts -rg "from.*foodSafety" frontend/src --type ts -rg "from.*inventoryDashboard" frontend/src --type ts -``` - -Update all found imports to use the consolidated service files. - -### Step 5: End-to-End Testing - -Test critical user flows: -- [ ] Create ingredient → Add stock → Consume stock -- [ ] Create recipe → Check feasibility → Start production batch -- [ ] Import sales data → View analytics -- [ ] Create purchase order → Receive delivery → Update stock -- [ ] View dashboard analytics for all services - -### Step 6: Network Inspection - -Open DevTools → Network tab and verify: -- [ ] All API calls use correct URLs matching backend structure -- [ ] No 404 errors from old endpoints -- [ ] Query parameters match backend expectations -- [ ] Response bodies match TypeScript types - ---- - -## 📊 Progress Tracking - -### Backend Analysis -- [x] Inventory service mapped -- [x] Production service mapped -- [x] Sales service mapped -- [x] Suppliers service mapped -- [x] Recipes service mapped -- [x] Forecasting service identified -- [x] Training service identified -- [x] All 14 services documented - -### Frontend Refactoring -- [x] **Phase 1: Types updated (14/14 services) - ✅ 100% COMPLETE** - - All TypeScript types now have zero drift with backend Pydantic schemas - - Comprehensive JSDoc documentation with backend file references - - All 14 services covered: inventory, production, sales, suppliers, recipes, forecasting, orders, training, tenant, auth, notification, pos, external, demo - -- [x] **Phase 2: Services refactored (14/14 services) - ✅ 100% COMPLETE** - - [x] **inventory.ts** - ✅ COMPLETE (2025-10-05) - - Organized using 3-tier architecture comments (ATOMIC, OPERATIONS, ANALYTICS, COMPLIANCE) - - Complete coverage: ingredients, stock, movements, transformations, temperature logs - - Operations: stock management, classification, food safety - - Analytics: dashboard summary, inventory analytics - - All endpoints aligned with backend API structure - - File: [frontend/src/api/services/inventory.ts](frontend/src/api/services/inventory.ts) - - - [x] **production.ts** - ✅ COMPLETE (2025-10-05) - - Organized using 3-tier architecture comments (ATOMIC, OPERATIONS, ANALYTICS) - - ATOMIC: Batches CRUD, Schedules CRUD - - OPERATIONS: Batch lifecycle (start, complete, status), schedule finalization, capacity management, quality checks - - ANALYTICS: Performance, yield trends, defects, equipment efficiency, capacity bottlenecks, dashboard - - 33 methods covering complete production workflow - - File: [frontend/src/api/services/production.ts](frontend/src/api/services/production.ts) - - - [x] **sales.ts** - ✅ COMPLETE (2025-10-05) - - Organized using 3-tier architecture comments (ATOMIC, OPERATIONS, ANALYTICS) - - ATOMIC: Sales Records CRUD, Categories - - OPERATIONS: Validation, cross-service product queries, data import (validate, execute, history, template), aggregation (by product, category, channel) - - ANALYTICS: Sales summary analytics - - 16 methods covering complete sales workflow including CSV import - - File: [frontend/src/api/services/sales.ts](frontend/src/api/services/sales.ts) - - - [x] **suppliers.ts** - ✅ COMPLETE (2025-10-05) - - Organized using 3-tier architecture comments (ATOMIC, OPERATIONS, ANALYTICS) - - ATOMIC: Suppliers CRUD, Purchase Orders CRUD, Deliveries CRUD - - OPERATIONS: Statistics, active suppliers, top suppliers, pending approvals, supplier approval - - ANALYTICS: Performance calculation, metrics, alerts evaluation - - UTILITIES: Order total calculation, supplier code formatting, tax ID validation, currency formatting - - 25 methods covering complete supplier lifecycle including performance tracking - - File: [frontend/src/api/services/suppliers.ts](frontend/src/api/services/suppliers.ts) - - - [x] **recipes.ts** - ✅ COMPLETE (2025-10-05) - - Organized using 3-tier architecture comments (ATOMIC, OPERATIONS) - - ATOMIC: Recipes CRUD, Quality Configuration CRUD - - OPERATIONS: Recipe Management (duplicate, activate, feasibility) - - 15 methods covering recipe lifecycle and quality management - - File: [frontend/src/api/services/recipes.ts](frontend/src/api/services/recipes.ts) - - - [x] **forecasting.ts** - ✅ COMPLETE (2025-10-05) - - Organized using 3-tier architecture comments (ATOMIC, OPERATIONS, ANALYTICS) - - ATOMIC: Forecast CRUD - - OPERATIONS: Single/Multi-day/Batch forecasts, Realtime predictions, Validation, Cache management - - ANALYTICS: Performance metrics - - 11 methods covering forecasting workflow - - File: [frontend/src/api/services/forecasting.ts](frontend/src/api/services/forecasting.ts) - - - [x] **orders.ts** - ✅ COMPLETE (2025-10-05) - - Organized using 3-tier architecture comments (ATOMIC, OPERATIONS) - - ATOMIC: Orders CRUD, Customers CRUD - - OPERATIONS: Dashboard & Analytics, Business Intelligence, Procurement Planning (21+ methods) - - 30+ methods covering order and procurement lifecycle - - File: [frontend/src/api/services/orders.ts](frontend/src/api/services/orders.ts) - - - [x] **training.ts** - ✅ COMPLETE (2025-10-05) - - Organized using 3-tier architecture comments (ATOMIC, OPERATIONS) - - ATOMIC: Training Job Status, Model Management - - OPERATIONS: Training Job Creation - - WebSocket Support for real-time training updates - - 9 methods covering ML training workflow - - File: [frontend/src/api/services/training.ts](frontend/src/api/services/training.ts) - - - [x] **tenant.ts** - ✅ COMPLETE (2025-10-05) - - Organized using 3-tier architecture comments (ATOMIC, OPERATIONS) - - ATOMIC: Tenant CRUD, Team Member Management - - OPERATIONS: Access Control, Search & Discovery, Model Status, Statistics & Admin - - Frontend Context Management utilities - - 17 methods covering tenant and team management - - File: [frontend/src/api/services/tenant.ts](frontend/src/api/services/tenant.ts) - - - [x] **auth.ts** - ✅ COMPLETE (2025-10-05) - - Organized using 3-tier architecture comments (ATOMIC, OPERATIONS) - - ATOMIC: User Profile - - OPERATIONS: Authentication (register, login, tokens, password), Email Verification - - 10 methods covering authentication workflow - - File: [frontend/src/api/services/auth.ts](frontend/src/api/services/auth.ts) - - - [x] **pos.ts** - ✅ COMPLETE (2025-10-05) - - Organized using 3-tier architecture comments (ATOMIC, OPERATIONS, ANALYTICS) - - ATOMIC: POS Configuration CRUD, Transactions - - OPERATIONS: Supported Systems, Sync Operations, Webhook Management - - Frontend Utility Methods for UI helpers - - 20+ methods covering POS integration lifecycle - - File: [frontend/src/api/services/pos.ts](frontend/src/api/services/pos.ts) - - - [x] **demo.ts** - ✅ COMPLETE (2025-10-05) - - Organized using 3-tier architecture comments (ATOMIC, OPERATIONS) - - ATOMIC: Demo Accounts, Demo Sessions - - OPERATIONS: Demo Session Management (extend, destroy, stats, cleanup) - - 6 functions covering demo session lifecycle - - File: [frontend/src/api/services/demo.ts](frontend/src/api/services/demo.ts) - - - Note: notification.ts and external.ts services do not exist as separate files - endpoints likely integrated into other services - -- [x] **Phase 3: Hooks updated (14/14 services) - ✅ 100% COMPLETE** - - All React Query hooks updated to match Phase 1 type changes - - Fixed type imports, method signatures, and enum values - - Updated infinite query hooks with initialPageParam - - Resolved all service method signature mismatches - - **Type Check Status: ✅ ZERO ERRORS** - -- [ ] Phase 4: Cross-service consistency verified -- [ ] Cleanup: Obsolete files deleted -- [x] **Verification: Type checks passing - ✅ COMPLETE** - - TypeScript compilation: ✅ 0 errors - - All hooks properly typed - - All service methods aligned -- [ ] Verification: E2E tests passing - -#### Detailed Progress (Last Updated: 2025-10-05) - -**Phase 1 - TypeScript Types:** -- [x] **inventory.ts** - ✅ COMPLETE (2025-10-05) - - Added comprehensive JSDoc references to backend schema files - - All 3 schema categories covered: inventory.py, food_safety.py, dashboard.py - - Includes: Ingredients, Stock, Movements, Transformations, Classification, Food Safety, Dashboard - - Type check: ✅ PASSING (no errors) - - File: [frontend/src/api/types/inventory.ts](frontend/src/api/types/inventory.ts) - -- [x] **production.ts** - ✅ COMPLETE (2025-10-05) - - Mirrored 2 backend schema files: production.py, quality_templates.py - - Includes: Batches, Schedules, Quality Checks, Quality Templates, Process Stages - - Added all operations and analytics types - - Type check: ✅ PASSING (no errors) - - File: [frontend/src/api/types/production.ts](frontend/src/api/types/production.ts) - -- [x] **sales.ts** - ✅ COMPLETE (2025-10-05) - - Mirrored backend schema: sales.py - - **BREAKING CHANGE**: Product references now use inventory_product_id (inventory service integration) - - Includes: Sales Data CRUD, Analytics, Import/Validation operations - - Type check: ✅ PASSING (no errors) - - File: [frontend/src/api/types/sales.ts](frontend/src/api/types/sales.ts) - -- [x] **suppliers.ts** - ✅ COMPLETE (2025-10-05) - - Mirrored 2 backend schema files: suppliers.py, performance.py - - Most comprehensive service: Suppliers, Purchase Orders, Deliveries, Performance, Alerts, Scorecards - - Includes: 13 enums, 60+ interfaces covering full supplier lifecycle - - Business model detection and performance analytics included - - Type check: ✅ PASSING (no errors) - - File: [frontend/src/api/types/suppliers.ts](frontend/src/api/types/suppliers.ts) - -- [x] **recipes.ts** - ✅ COMPLETE (2025-10-05) - - Mirrored backend schema: recipes.py - - Includes: Recipe CRUD, Recipe Ingredients, Quality Configuration (stage-based), Operations (duplicate, activate, feasibility) - - 3 enums, 20+ interfaces covering recipe lifecycle, quality checks, production batches - - Quality templates integration for production workflow - - Type check: ✅ PASSING (no type errors specific to recipes) - - File: [frontend/src/api/types/recipes.ts](frontend/src/api/types/recipes.ts) - -- [x] **forecasting.ts** - ✅ COMPLETE (2025-10-05) - - Mirrored backend schema: forecasts.py - - Includes: Forecast CRUD, Operations (single, multi-day, batch, realtime predictions), Analytics, Validation - - 1 enum, 15+ interfaces covering forecast generation, batch processing, predictions, performance metrics - - Integration with inventory service via inventory_product_id references - - Type check: ✅ PASSING (no type errors specific to forecasting) - - File: [frontend/src/api/types/forecasting.ts](frontend/src/api/types/forecasting.ts) - -- [x] **orders.ts** - ✅ COMPLETE (2025-10-05) - - Mirrored 2 backend schema files: order_schemas.py, procurement_schemas.py - - Includes: Customer CRUD, Order CRUD (items, workflow), Procurement Plans (MRP-style), Requirements, Dashboard - - 17 enums, 50+ interfaces covering full order and procurement lifecycle - - Advanced features: Business model detection, procurement planning, demand requirements - - Type check: ✅ PASSING (no type errors specific to orders) - - File: [frontend/src/api/types/orders.ts](frontend/src/api/types/orders.ts) - -- [x] **training.ts** - ✅ COMPLETE (2025-10-05) - - Mirrored backend schema: training.py - - Includes: Training Jobs, Model Management, Data Validation, Real-time Progress (WebSocket), Bulk Operations - - 1 enum, 25+ interfaces covering ML training workflow, Prophet model configuration, metrics, scheduling - - Advanced features: WebSocket progress updates, external data integration (weather/traffic), model versioning - - Type check: ✅ PASSING (no type errors specific to training) - - File: [frontend/src/api/types/training.ts](frontend/src/api/types/training.ts) - -- [x] **tenant.ts** - ✅ COMPLETE (2025-10-05) - **CRITICAL FIX** - - Mirrored backend schema: tenants.py - - **FIXED**: Added required `owner_id` field to TenantResponse - resolves type error - - Includes: Bakery Registration, Tenant CRUD, Members, Subscriptions, Access Control, Analytics - - 10+ interfaces covering tenant lifecycle, team management, subscription plans (basic/professional/enterprise) - - Type check: ✅ PASSING - owner_id error RESOLVED - - File: [frontend/src/api/types/tenant.ts](frontend/src/api/types/tenant.ts) - -- [x] **auth.ts** - ✅ COMPLETE (2025-10-05) - - Mirrored 2 backend schema files: auth.py, users.py - - Includes: Registration, Login, Token Management, Password Reset, Email Verification, User Management - - 14+ interfaces covering authentication workflow, JWT tokens, error handling, internal service communication - - Token response follows industry standards (Firebase, AWS Cognito) - - Type check: ⚠️ Hook errors remain (Phase 3) - types complete - - File: [frontend/src/api/types/auth.ts](frontend/src/api/types/auth.ts) - -- [x] **notification.ts** - ✅ COMPLETE (2025-10-05) - - Mirrored backend schema: notifications.py - - Includes: Notifications CRUD, Bulk Send, Preferences, Templates, Webhooks, Statistics - - 3 enums, 14+ interfaces covering notification lifecycle, delivery tracking, user preferences - - Multi-channel support: Email, WhatsApp, Push, SMS - - Advanced features: Quiet hours, digest frequency, template system, delivery webhooks - - Type check: ⚠️ Hook errors remain (Phase 3) - types complete - - File: [frontend/src/api/types/notification.ts](frontend/src/api/types/notification.ts) - -- [x] **pos.ts** - ✅ ALREADY COMPLETE (2025-09-11) - - Mirrored backend models: pos_config.py, pos_transaction.py - - Includes: Configurations, Transactions, Transaction Items, Webhooks, Sync Logs, Analytics - - 13 type aliases, 40+ interfaces covering POS integration lifecycle - - Multi-POS support: Square, Toast, Lightspeed - - Advanced features: Sync management, webhook handling, duplicate detection, sync analytics - - Type check: ⚠️ Hook errors remain (Phase 3) - types complete - - File: [frontend/src/api/types/pos.ts](frontend/src/api/types/pos.ts) - -- [x] **external.ts** - ✅ COMPLETE (2025-10-05) - - Mirrored 2 backend schema files: weather.py, traffic.py - - Includes: Weather Data, Weather Forecasts, Traffic Data, Analytics, Hourly Forecasts - - 20+ interfaces covering external data lifecycle, historical data, forecasting - - Data sources: AEMET (weather), Madrid OpenData (traffic) - - Advanced features: Location-based queries, date range filtering, analytics aggregation - - Type check: ⚠️ Hook errors remain (Phase 3) - types complete - - File: [frontend/src/api/types/external.ts](frontend/src/api/types/external.ts) - -- [x] **demo.ts** - ✅ COMPLETE (2025-10-05) - - Mirrored backend schema: schemas.py - - Includes: Demo Sessions, Account Info, Data Cloning, Statistics - - 8 interfaces covering demo session lifecycle, tenant data cloning - - Demo account types: individual_bakery, central_baker - - Advanced features: Session extension, virtual tenant management, data cloning - - Type check: ⚠️ Hook errors remain (Phase 3) - types complete - - File: [frontend/src/api/types/demo.ts](frontend/src/api/types/demo.ts) - ---- - -## 🚨 Critical Reminders - -### ✅ Must Follow -1. **Read backend schemas first** - Don't guess types -2. **Test after each service** - Don't batch all changes -3. **Update one service fully** - Types → Service → Hooks → Test -4. **Delete old files immediately** - Prevents confusion -5. **Document breaking changes** - Help other developers - -### ❌ Absolutely Avoid -1. ❌ Creating new service files without backend equivalent -2. ❌ Keeping "temporary" hybrid files -3. ❌ Skipping type updates -4. ❌ Direct API calls in components -5. ❌ Mixing ATOMIC and OPERATIONS in unclear ways - ---- - -## 🎯 Success Criteria - -The refactoring is complete when: - -- [x] All TypeScript types match backend Pydantic schemas ✅ -- [x] All service methods map 1:1 to backend endpoints ✅ -- [x] All hooks use service methods (no direct API calls) ✅ -- [x] `npm run type-check` passes with zero errors ✅ -- [x] Production build succeeds ✅ -- [x] Code is documented with JSDoc comments ✅ -- [x] This document is marked as [COMPLETED] ✅ - -**Note:** Legacy service files (classification, foodSafety, etc.) preserved to maintain backward compatibility with existing components. Future migration recommended but not required. - ---- - -**PROJECT STATUS: ✅ [COMPLETED] - 100%** - -- ✅ Phase 1 Complete (14/14 core services - TypeScript types) -- ✅ Phase 2 Complete (14/14 core services - Service files) -- ✅ Phase 3 Complete (14/14 core services - Hooks) -- ✅ Phase 4 Complete (Cross-service consistency verified) -- ✅ Phase 5 Complete (Legacy file cleanup and consolidation) - -**Architecture:** -- 14 Core consolidated services (inventory, sales, production, recipes, etc.) -- 3 Specialized domain modules (qualityTemplates, onboarding, subscription) -- Total: 17 production services (down from 22 - **23% reduction**) - -**Final Verification:** -- ✅ TypeScript compilation: 0 errors -- ✅ Production build: Success (built in 3.03s) -- ✅ Zero drift with backend Pydantic schemas -- ✅ All 14 services fully aligned - -**Achievements:** -- Complete frontend-backend type alignment across 14 microservices -- Consistent 3-tier architecture (ATOMIC, OPERATIONS, ANALYTICS) -- All React Query hooks properly typed with zero errors -- Comprehensive JSDoc documentation referencing backend schemas -- Production-ready build verified - -**Cleanup Progress (2025-10-05):** -- ✅ Deleted unused services: `transformations.ts`, `foodSafety.ts`, `inventoryDashboard.ts` -- ✅ Deleted unused hooks: `foodSafety.ts`, `inventoryDashboard.ts` -- ✅ Updated `index.ts` exports to remove deleted modules -- ✅ Fixed `inventory.ts` hooks to use consolidated `inventoryService` -- ✅ Production build: **Success (3.06s)** - -**Additional Cleanup (2025-10-05 - Session 2):** -- ✅ Migrated `classification.ts` → `inventory.ts` hooks (useClassifyBatch) -- ✅ Migrated `dataImport.ts` → `sales.ts` hooks (useValidateImportFile, useImportSalesData) -- ✅ Updated UploadSalesDataStep component to use consolidated hooks -- ✅ Deleted `classification.ts` service and hooks -- ✅ Deleted `dataImport.ts` service and hooks -- ✅ Production build: **Success (2.96s)** - -**Total Files Deleted: 9** -- Services: `transformations.ts`, `foodSafety.ts`, `inventoryDashboard.ts`, `classification.ts`, `dataImport.ts` -- Hooks: `foodSafety.ts`, `inventoryDashboard.ts`, `classification.ts`, `dataImport.ts` - -**Specialized Service Modules (Intentionally Preserved):** - -These 3 files are **NOT legacy** - they are specialized, domain-specific modules that complement the core consolidated services: - -| Module | Purpose | Justification | Components | -|--------|---------|---------------|------------| -| **qualityTemplates.ts** | Production quality check template management | 12 specialized methods for template CRUD, validation, and execution. Domain-specific to quality assurance workflow. | 4 (recipes/production) | -| **onboarding.ts** | User onboarding progress tracking | Manages multi-step onboarding state, progress persistence, and step completion. User journey management. | 1 (OnboardingWizard) | -| **subscription.ts** | Subscription tier access control | Feature gating based on subscription plans (STARTER/PROFESSIONAL/ENTERPRISE). Business logic layer. | 2 (analytics pages) | - -**Architecture Decision:** -These modules follow **Domain-Driven Design** principles - they encapsulate complex domain logic that would clutter the main services. They are: -- ✅ Well-tested and production-proven -- ✅ Single Responsibility Principle compliant -- ✅ Zero duplication with consolidated services -- ✅ Clear boundaries and interfaces -- ✅ Actively maintained - -**Status:** These are **permanent architecture components**, not technical debt. - -**Next Steps (Optional - Future Enhancements):** -1. Add E2E tests to verify all workflows -2. Performance optimization and bundle size analysis -3. Document these specialized modules in architecture diagrams diff --git a/FRONTEND_INTEGRATION_GUIDE.md b/FRONTEND_INTEGRATION_GUIDE.md deleted file mode 100644 index 659f1c9a..00000000 --- a/FRONTEND_INTEGRATION_GUIDE.md +++ /dev/null @@ -1,748 +0,0 @@ -# FRONTEND INTEGRATION GUIDE - Procurement Features - -## ✅ COMPLETED FRONTEND CHANGES - -All TypeScript types, API service methods, and React hooks have been implemented. This guide shows how to use them in your components. - ---- - -## 📦 WHAT'S BEEN ADDED - -### 1. **New Types** (`frontend/src/api/types/orders.ts`) - -```typescript -// Approval workflow tracking -export interface ApprovalWorkflowEntry { - timestamp: string; - from_status: string; - to_status: string; - user_id?: string; - notes?: string; -} - -// Purchase order creation result -export interface CreatePOsResult { - success: boolean; - created_pos: Array<{ - po_id: string; - po_number: string; - supplier_id: string; - items_count: number; - total_amount: number; - }>; - failed_pos: Array<{ - supplier_id: string; - error: string; - }>; - total_created: number; - total_failed: number; -} - -// Request types -export interface LinkRequirementToPORequest { - purchase_order_id: string; - purchase_order_number: string; - ordered_quantity: number; - expected_delivery_date?: string; -} - -export interface UpdateDeliveryStatusRequest { - delivery_status: string; - received_quantity?: number; - actual_delivery_date?: string; - quality_rating?: number; -} - -export interface ApprovalRequest { - approval_notes?: string; -} - -export interface RejectionRequest { - rejection_notes?: string; -} -``` - -**Updated ProcurementPlanResponse:** -- Added `approval_workflow?: ApprovalWorkflowEntry[]` - tracks all approval actions - ---- - -### 2. **New API Methods** (`frontend/src/api/services/orders.ts`) - -```typescript -class OrdersService { - // Recalculate plan with current inventory - static async recalculateProcurementPlan(tenantId: string, planId: string): Promise - - // Approve plan with notes - static async approveProcurementPlan(tenantId: string, planId: string, request?: ApprovalRequest): Promise - - // Reject plan with notes - static async rejectProcurementPlan(tenantId: string, planId: string, request?: RejectionRequest): Promise - - // Auto-create POs from plan - static async createPurchaseOrdersFromPlan(tenantId: string, planId: string, autoApprove?: boolean): Promise - - // Link requirement to PO - static async linkRequirementToPurchaseOrder(tenantId: string, requirementId: string, request: LinkRequirementToPORequest): Promise<{...}> - - // Update delivery status - static async updateRequirementDeliveryStatus(tenantId: string, requirementId: string, request: UpdateDeliveryStatusRequest): Promise<{...}> -} -``` - ---- - -### 3. **New React Hooks** (`frontend/src/api/hooks/orders.ts`) - -```typescript -// Recalculate plan -useRecalculateProcurementPlan(options?) - -// Approve plan -useApproveProcurementPlan(options?) - -// Reject plan -useRejectProcurementPlan(options?) - -// Create POs from plan -useCreatePurchaseOrdersFromPlan(options?) - -// Link requirement to PO -useLinkRequirementToPurchaseOrder(options?) - -// Update delivery status -useUpdateRequirementDeliveryStatus(options?) -``` - ---- - -## 🎨 HOW TO USE IN COMPONENTS - -### Example 1: Recalculate Plan Button - -```typescript -import { useRecalculateProcurementPlan } from '@/api/hooks/orders'; -import { useToast } from '@/hooks/useToast'; - -function ProcurementPlanActions({ plan, tenantId }) { - const { toast } = useToast(); - const recalculateMutation = useRecalculateProcurementPlan({ - onSuccess: (data) => { - if (data.success && data.plan) { - toast({ - title: 'Plan recalculado', - description: `Plan actualizado con ${data.plan.total_requirements} requerimientos`, - variant: 'success', - }); - } - }, - onError: (error) => { - toast({ - title: 'Error al recalcular', - description: error.message, - variant: 'destructive', - }); - }, - }); - - const handleRecalculate = () => { - if (confirm('¿Recalcular el plan con el inventario actual?')) { - recalculateMutation.mutate({ tenantId, planId: plan.id }); - } - }; - - // Show warning if plan is old - const planAgeHours = (new Date().getTime() - new Date(plan.created_at).getTime()) / (1000 * 60 * 60); - const isStale = planAgeHours > 24; - - return ( -
- {isStale && ( - - - Plan desactualizado - - Este plan tiene más de 24 horas. El inventario puede haber cambiado. - - - )} - - -
- ); -} -``` - ---- - -### Example 2: Approve/Reject Plan with Notes - -```typescript -import { useApproveProcurementPlan, useRejectProcurementPlan } from '@/api/hooks/orders'; -import { Dialog, DialogContent, DialogHeader, DialogTitle } from '@/components/ui/dialog'; -import { Textarea } from '@/components/ui/textarea'; - -function ApprovalDialog({ plan, tenantId, open, onClose }) { - const [notes, setNotes] = useState(''); - const [action, setAction] = useState<'approve' | 'reject'>('approve'); - - const approveMutation = useApproveProcurementPlan({ - onSuccess: () => { - toast({ title: 'Plan aprobado', variant: 'success' }); - onClose(); - }, - }); - - const rejectMutation = useRejectProcurementPlan({ - onSuccess: () => { - toast({ title: 'Plan rechazado', variant: 'success' }); - onClose(); - }, - }); - - const handleSubmit = () => { - if (action === 'approve') { - approveMutation.mutate({ - tenantId, - planId: plan.id, - approval_notes: notes || undefined, - }); - } else { - rejectMutation.mutate({ - tenantId, - planId: plan.id, - rejection_notes: notes || undefined, - }); - } - }; - - return ( - - - - - {action === 'approve' ? 'Aprobar' : 'Rechazar'} Plan de Compras - - - -
-
- -