Add new infra architecture

infrastructure/services/microservices/external/cronjobs/external-data-rotation-cronjob.yaml

@@ -0,0 +1,68 @@
+# infrastructure/kubernetes/base/cronjobs/external-data-rotation-cronjob.yaml
+# Monthly CronJob to rotate 24-month sliding window (runs 1st of month at 2am UTC)
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: external-data-rotation
+  namespace: bakery-ia
+  labels:
+    app: external-service
+    component: data-rotation
+spec:
+  schedule: "0 2 1 * *"
+
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 3
+
+  concurrencyPolicy: Forbid
+
+  jobTemplate:
+    metadata:
+      labels:
+        app: external-service
+        job: data-rotation
+    spec:
+      imagePullSecrets:
+      - name: dockerhub-creds
+      ttlSecondsAfterFinished: 172800
+      backoffLimit: 2
+
+      template:
+        metadata:
+          labels:
+            app: external-service
+            cronjob: data-rotation
+        spec:
+          restartPolicy: OnFailure
+
+          containers:
+          - name: data-rotator
+            image: bakery/external-service:latest
+            imagePullPolicy: Always
+
+            command:
+              - python
+              - -m
+              - app.jobs.rotate_data
+
+            args:
+              - "--log-level=INFO"
+              - "--notify-slack=true"
+
+            envFrom:
+            - configMapRef:
+                name: bakery-config
+            - secretRef:
+                name: database-secrets
+            - secretRef:
+                name: external-api-secrets
+            - secretRef:
+                name: monitoring-secrets
+
+            resources:
+              requests:
+                memory: "512Mi"
+                cpu: "250m"
+              limits:
+                memory: "1Gi"
+                cpu: "500m"

infrastructure/services/microservices/external/external-service.yaml

@@ -0,0 +1,216 @@
+# infrastructure/kubernetes/base/components/external/external-service.yaml
+# External Data Service v2.0 - Optimized city-based architecture
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: external-service
+  namespace: bakery-ia
+  labels:
+    app.kubernetes.io/name: external-service
+    app.kubernetes.io/component: microservice
+    app.kubernetes.io/part-of: bakery-ia
+    version: "2.0"
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: external-service
+      app.kubernetes.io/component: microservice
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: external-service
+        app.kubernetes.io/component: microservice
+        version: "2.0"
+    spec:
+      imagePullSecrets:
+      - name: dockerhub-creds
+      initContainers:
+      # Wait for Redis to be ready
+      - name: wait-for-redis
+        image: redis:7.4-alpine
+        command:
+        - sh
+        - -c
+        - |
+          echo "Waiting for Redis to be ready..."
+          until redis-cli -h $REDIS_HOST -p $REDIS_PORT --tls --cert /tls/redis-cert.pem --key /tls/redis-key.pem --cacert /tls/ca-cert.pem -a "$REDIS_PASSWORD" ping | grep -q PONG; do
+            echo "Redis not ready yet, waiting..."
+            sleep 2
+          done
+          echo "Redis is ready!"
+        env:
+        - name: REDIS_HOST
+          valueFrom:
+            configMapKeyRef:
+              name: bakery-config
+              key: REDIS_HOST
+        - name: REDIS_PORT
+          valueFrom:
+            configMapKeyRef:
+              name: bakery-config
+              key: REDIS_PORT
+        - name: REDIS_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: redis-secrets
+              key: REDIS_PASSWORD
+        volumeMounts:
+        - name: redis-tls
+          mountPath: /tls
+          readOnly: true
+      # Wait for database migration to complete
+      - name: wait-for-migration
+        image: postgres:17-alpine
+        command:
+        - sh
+        - -c
+        - |
+          echo "Waiting for external database and migrations to be ready..."
+          # Wait for database to be accessible
+          until pg_isready -h $EXTERNAL_DB_HOST -p $EXTERNAL_DB_PORT -U $EXTERNAL_DB_USER; do
+            echo "Database not ready yet, waiting..."
+            sleep 2
+          done
+          echo "Database is ready!"
+
+          # Verify that migrations have completed by checking for the alembic_version table
+          ATTEMPTS=30
+          COUNT=0
+          until [ $COUNT -ge $ATTEMPTS ]; do
+            if PGPASSWORD="$EXTERNAL_DB_PASSWORD" psql -h "$EXTERNAL_DB_HOST" -p "$EXTERNAL_DB_PORT" -U "$EXTERNAL_DB_USER" -d "$EXTERNAL_DB_NAME" -c "\dt alembic_version" > /dev/null 2>&1; then
+              echo "Migrations are complete - alembic_version table exists"
+              exit 0
+            else
+              echo "Migrations not complete yet, waiting... ($((COUNT + 1))/$ATTEMPTS)"
+              sleep 10
+            fi
+            COUNT=$((COUNT + 1))
+          done
+
+          echo "Timeout waiting for migrations to complete"
+          exit 1
+        env:
+        - name: EXTERNAL_DB_HOST
+          valueFrom:
+            configMapKeyRef:
+              name: bakery-config
+              key: EXTERNAL_DB_HOST
+        - name: EXTERNAL_DB_PORT
+          valueFrom:
+            configMapKeyRef:
+              name: bakery-config
+              key: DB_PORT
+        - name: EXTERNAL_DB_USER
+          valueFrom:
+            secretKeyRef:
+              name: database-secrets
+              key: EXTERNAL_DB_USER
+        - name: EXTERNAL_DB_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: database-secrets
+              key: EXTERNAL_DB_PASSWORD
+        - name: EXTERNAL_DB_NAME
+          value: "external_db"
+
+      containers:
+      - name: external-service
+        image: bakery/external-service:latest
+        ports:
+        - containerPort: 8000
+          name: http
+        env:
+        # OpenTelemetry Configuration
+        - name: OTEL_COLLECTOR_ENDPOINT
+          value: "http://signoz-otel-collector.bakery-ia.svc.cluster.local:4318"
+        - name: OTEL_EXPORTER_OTLP_ENDPOINT
+          valueFrom:
+            configMapKeyRef:
+              name: bakery-config
+              key: OTEL_EXPORTER_OTLP_ENDPOINT
+        - name: OTEL_SERVICE_NAME
+          value: "external-service"
+        - name: ENABLE_TRACING
+          value: "true"
+        # Logging Configuration
+        - name: OTEL_LOGS_EXPORTER
+          value: "otlp"
+        - name: OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED
+          value: "true"
+        # Metrics Configuration
+        - name: ENABLE_OTEL_METRICS
+          value: "true"
+        - name: ENABLE_SYSTEM_METRICS
+          value: "true"
+        envFrom:
+        - configMapRef:
+            name: bakery-config
+        - secretRef:
+            name: database-secrets
+        - secretRef:
+            name: redis-secrets
+        - secretRef:
+            name: rabbitmq-secrets
+        - secretRef:
+            name: jwt-secrets
+        - secretRef:
+            name: external-api-secrets
+        - secretRef:
+            name: payment-secrets
+        - secretRef:
+            name: email-secrets
+        - secretRef:
+            name: monitoring-secrets
+        - secretRef:
+            name: pos-integration-secrets
+        - secretRef:
+            name: whatsapp-secrets
+        resources:
+          requests:
+            memory: "256Mi"
+            cpu: "100m"
+          limits:
+            memory: "512Mi"
+            cpu: "500m"
+        livenessProbe:
+          httpGet:
+            path: /health/live
+            port: 8000
+          initialDelaySeconds: 30
+          timeoutSeconds: 5
+          periodSeconds: 10
+          failureThreshold: 3
+        readinessProbe:
+          httpGet:
+            path: /health/ready
+            port: 8000
+          initialDelaySeconds: 15
+          timeoutSeconds: 3
+          periodSeconds: 5
+          failureThreshold: 5
+      volumes:
+      - name: redis-tls
+        secret:
+          secretName: redis-tls-secret
+          defaultMode: 0400
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: external-service
+  namespace: bakery-ia
+  labels:
+    app.kubernetes.io/name: external-service
+    app.kubernetes.io/component: microservice
+spec:
+  type: ClusterIP
+  ports:
+  - port: 8000
+    targetPort: 8000
+    protocol: TCP
+    name: http
+  selector:
+    app.kubernetes.io/name: external-service
+    app.kubernetes.io/component: microservice

infrastructure/services/microservices/external/migrations/external-data-init-job.yaml

@@ -0,0 +1,82 @@
+# infrastructure/kubernetes/base/jobs/external-data-init-job.yaml
+# One-time job to initialize 24 months of historical data for all enabled cities
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: external-data-init
+  namespace: bakery-ia
+  labels:
+    app: external-service
+    component: data-initialization
+spec:
+  ttlSecondsAfterFinished: 86400
+  backoffLimit: 3
+  template:
+    metadata:
+      labels:
+        app: external-service
+        job: data-init
+    spec:
+      imagePullSecrets:
+      - name: dockerhub-creds
+      restartPolicy: OnFailure
+
+      initContainers:
+      - name: wait-for-db
+        image: postgres:17-alpine
+        command:
+          - sh
+          - -c
+          - |
+            until pg_isready -h $EXTERNAL_DB_HOST -p $DB_PORT -U $EXTERNAL_DB_USER; do
+              echo "Waiting for database..."
+              sleep 2
+            done
+            echo "Database is ready"
+        envFrom:
+        - configMapRef:
+            name: bakery-config
+        - secretRef:
+            name: database-secrets
+      - name: wait-for-migration
+        image: postgres:17-alpine
+        command:
+          - sh
+          - -c
+          - |
+            echo "Waiting for external-service migration to complete..."
+            sleep 15
+            echo "Migration should be complete"
+        envFrom:
+        - configMapRef:
+            name: bakery-config
+
+      containers:
+      - name: data-loader
+        image: bakery/external-service:latest
+        imagePullPolicy: Always
+
+        command:
+          - python
+          - -m
+          - app.jobs.initialize_data
+
+        args:
+          - "--months=6"  # Reduced from 24 to avoid memory/rate limit issues
+          - "--log-level=INFO"
+
+        envFrom:
+        - configMapRef:
+            name: bakery-config
+        - secretRef:
+            name: database-secrets
+        - secretRef:
+            name: external-api-secrets
+
+        resources:
+          requests:
+            memory: "2Gi"  # Increased from 1Gi
+            cpu: "500m"
+          limits:
+            memory: "4Gi"  # Increased from 2Gi
+            cpu: "1000m"

infrastructure/services/microservices/external/migrations/external-migration-job.yaml

@@ -0,0 +1,58 @@
+# Enhanced migration job for external service with automatic table creation
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: external-migration
+  namespace: bakery-ia
+  labels:
+    app.kubernetes.io/name: external-migration
+    app.kubernetes.io/component: migration
+    app.kubernetes.io/part-of: bakery-ia
+spec:
+  backoffLimit: 3
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: external-migration
+        app.kubernetes.io/component: migration
+    spec:
+      imagePullSecrets:
+      - name: dockerhub-creds
+      - name: ghcr-creds
+      initContainers:
+      - name: wait-for-db
+        image: postgres:17-alpine
+        command: ["sh", "-c", "until pg_isready -h external-db-service -p 5432; do sleep 2; done"]
+        resources:
+          requests:
+            memory: "64Mi"
+            cpu: "50m"
+          limits:
+            memory: "128Mi"
+            cpu: "100m"
+      containers:
+      - name: migrate
+        image: bakery/external-service
+        command: ["python", "/app/shared/scripts/run_migrations.py", "external"]
+        env:
+        - name: EXTERNAL_DATABASE_URL
+          valueFrom:
+            secretKeyRef:
+              name: database-secrets
+              key: EXTERNAL_DATABASE_URL
+        - name: DB_FORCE_RECREATE
+          valueFrom:
+            configMapKeyRef:
+              name: bakery-config
+              key: DB_FORCE_RECREATE
+              optional: true
+        - name: LOG_LEVEL
+          value: "INFO"
+        resources:
+          requests:
+            memory: "256Mi"
+            cpu: "100m"
+          limits:
+            memory: "512Mi"
+            cpu: "500m"
+      restartPolicy: OnFailure