Improve AI logic

2025-11-05 13:34:56 +01:00
parent 5c87fbcf48
commit 394ad3aea4
218 changed files with 30627 additions and 7658 deletions
--- a/infrastructure/kubernetes/base/components/ai-insights/ai-insights-service.yaml
+++ b/infrastructure/kubernetes/base/components/ai-insights/ai-insights-service.yaml
@@ -0,0 +1,127 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ai-insights-service
+  namespace: bakery-ia
+  labels:
+    app.kubernetes.io/name: ai-insights-service
+    app.kubernetes.io/component: microservice
+    app.kubernetes.io/part-of: bakery-ia
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: ai-insights-service
+      app.kubernetes.io/component: microservice
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: ai-insights-service
+        app.kubernetes.io/component: microservice
+    spec:
+      initContainers:
+      - name: wait-for-migration
+        image: postgres:17-alpine
+        command:
+        - sh
+        - -c
+        - |
+          echo "Waiting for ai-insights database and migrations to be ready..."
+          # Wait for database to be accessible
+          until pg_isready -h $AI_INSIGHTS_DB_HOST -p $AI_INSIGHTS_DB_PORT -U $AI_INSIGHTS_DB_USER; do
+            echo "Database not ready yet, waiting..."
+            sleep 2
+          done
+          echo "Database is ready!"
+          # Give migrations extra time to complete after DB is ready
+          echo "Waiting for migrations to complete..."
+          sleep 10
+          echo "Ready to start service"
+        env:
+        - name: AI_INSIGHTS_DB_HOST
+          valueFrom:
+            configMapKeyRef:
+              name: bakery-config
+              key: AI_INSIGHTS_DB_HOST
+        - name: AI_INSIGHTS_DB_PORT
+          valueFrom:
+            configMapKeyRef:
+              name: bakery-config
+              key: DB_PORT
+        - name: AI_INSIGHTS_DB_USER
+          valueFrom:
+            secretKeyRef:
+              name: database-secrets
+              key: AI_INSIGHTS_DB_USER
+      containers:
+      - name: ai-insights-service
+        image: bakery/ai-insights-service:dev
+        ports:
+        - containerPort: 8000
+          name: http
+        envFrom:
+        - configMapRef:
+            name: bakery-config
+        - secretRef:
+            name: database-secrets
+        - secretRef:
+            name: redis-secrets
+        - secretRef:
+            name: rabbitmq-secrets
+        - secretRef:
+            name: jwt-secrets
+        - secretRef:
+            name: external-api-secrets
+        - secretRef:
+            name: payment-secrets
+        - secretRef:
+            name: email-secrets
+        - secretRef:
+            name: monitoring-secrets
+        - secretRef:
+            name: pos-integration-secrets
+        - secretRef:
+            name: whatsapp-secrets
+        resources:
+          requests:
+            memory: "512Mi"
+            cpu: "200m"
+          limits:
+            memory: "1Gi"
+            cpu: "1000m"
+        livenessProbe:
+          httpGet:
+            path: /health
+            port: 8000
+          initialDelaySeconds: 30
+          timeoutSeconds: 5
+          periodSeconds: 10
+          failureThreshold: 3
+        readinessProbe:
+          httpGet:
+            path: /health
+            port: 8000
+          initialDelaySeconds: 15
+          timeoutSeconds: 3
+          periodSeconds: 5
+          failureThreshold: 5
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: ai-insights-service
+  namespace: bakery-ia
+  labels:
+    app.kubernetes.io/name: ai-insights-service
+    app.kubernetes.io/component: microservice
+spec:
+  type: ClusterIP
+  ports:
+  - port: 8000
+    targetPort: 8000
+    protocol: TCP
+    name: http
+  selector:
+    app.kubernetes.io/name: ai-insights-service
+    app.kubernetes.io/component: microservice
--- a/infrastructure/kubernetes/base/components/databases/ai-insights-db.yaml
+++ b/infrastructure/kubernetes/base/components/databases/ai-insights-db.yaml
@@ -0,0 +1,169 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ai-insights-db
+  namespace: bakery-ia
+  labels:
+    app.kubernetes.io/name: ai-insights-db
+    app.kubernetes.io/component: database
+    app.kubernetes.io/part-of: bakery-ia
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: ai-insights-db
+      app.kubernetes.io/component: database
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: ai-insights-db
+        app.kubernetes.io/component: database
+    spec:
+      securityContext:
+        fsGroup: 70
+      initContainers:
+      - name: fix-tls-permissions
+        image: busybox:latest
+        securityContext:
+          runAsUser: 0
+        command: ['sh', '-c']
+        args:
+        - |
+          cp /tls-source/* /tls/
+          chmod 600 /tls/server-key.pem
+          chmod 644 /tls/server-cert.pem /tls/ca-cert.pem
+          chown 70:70 /tls/*
+          ls -la /tls/
+        volumeMounts:
+        - name: tls-certs-source
+          mountPath: /tls-source
+          readOnly: true
+        - name: tls-certs-writable
+          mountPath: /tls
+      containers:
+      - name: postgres
+        image: postgres:17-alpine
+        command: ["docker-entrypoint.sh", "-c", "config_file=/etc/postgresql/postgresql.conf"]
+        ports:
+        - containerPort: 5432
+          name: postgres
+        env:
+        - name: POSTGRES_DB
+          valueFrom:
+            configMapKeyRef:
+              name: bakery-config
+              key: AI_INSIGHTS_DB_NAME
+        - name: POSTGRES_USER
+          valueFrom:
+            secretKeyRef:
+              name: database-secrets
+              key: AI_INSIGHTS_DB_USER
+        - name: POSTGRES_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: database-secrets
+              key: AI_INSIGHTS_DB_PASSWORD
+        - name: POSTGRES_INITDB_ARGS
+          valueFrom:
+            configMapKeyRef:
+              name: bakery-config
+              key: POSTGRES_INITDB_ARGS
+        - name: PGDATA
+          value: /var/lib/postgresql/data/pgdata
+        - name: POSTGRES_HOST_SSL
+          value: "on"
+        - name: PGSSLCERT
+          value: /tls/server-cert.pem
+        - name: PGSSLKEY
+          value: /tls/server-key.pem
+        - name: PGSSLROOTCERT
+          value: /tls/ca-cert.pem
+        volumeMounts:
+        - name: postgres-data
+          mountPath: /var/lib/postgresql/data
+        - name: init-scripts
+          mountPath: /docker-entrypoint-initdb.d
+        - name: tls-certs-writable
+          mountPath: /tls
+        - name: postgres-config
+          mountPath: /etc/postgresql
+          readOnly: true
+        resources:
+          requests:
+            memory: "256Mi"
+            cpu: "100m"
+          limits:
+            memory: "512Mi"
+            cpu: "500m"
+        livenessProbe:
+          exec:
+            command:
+            - sh
+            - -c
+            - pg_isready -U $POSTGRES_USER -d $POSTGRES_DB
+          initialDelaySeconds: 30
+          timeoutSeconds: 5
+          periodSeconds: 10
+          failureThreshold: 3
+        readinessProbe:
+          exec:
+            command:
+            - sh
+            - -c
+            - pg_isready -U $POSTGRES_USER -d $POSTGRES_DB
+          initialDelaySeconds: 5
+          timeoutSeconds: 1
+          periodSeconds: 5
+          failureThreshold: 3
+      volumes:
+      - name: postgres-data
+        persistentVolumeClaim:
+          claimName: ai-insights-db-pvc
+      - name: init-scripts
+        configMap:
+          name: postgres-init-config
+      - name: tls-certs-source
+        secret:
+          secretName: postgres-tls
+      - name: tls-certs-writable
+        emptyDir: {}
+      - name: postgres-config
+        configMap:
+          name: postgres-logging-config
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: ai-insights-db-service
+  namespace: bakery-ia
+  labels:
+    app.kubernetes.io/name: ai-insights-db
+    app.kubernetes.io/component: database
+spec:
+  type: ClusterIP
+  ports:
+  - port: 5432
+    targetPort: 5432
+    protocol: TCP
+    name: postgres
+  selector:
+    app.kubernetes.io/name: ai-insights-db
+    app.kubernetes.io/component: database
+
+
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: ai-insights-db-pvc
+  namespace: bakery-ia
+  labels:
+    app.kubernetes.io/name: ai-insights-db
+    app.kubernetes.io/component: database
+spec:
+  accessModes:
+  - ReadWriteOnce
+  resources:
+    requests:
+      storage: 2Gi
--- a/infrastructure/kubernetes/base/components/training/training-service.yaml
+++ b/infrastructure/kubernetes/base/components/training/training-service.yaml
@@ -113,7 +113,7 @@ spec:
      volumes:
      - name: tmp-storage
        emptyDir:
-          sizeLimit: 2Gi
+          sizeLimit: 4Gi  # Increased from 2Gi to handle cmdstan temp files during optimization
      - name: model-storage
        persistentVolumeClaim:
          claimName: model-storage
--- a/infrastructure/kubernetes/base/configmap.yaml
+++ b/infrastructure/kubernetes/base/configmap.yaml
@@ -59,6 +59,7 @@ data:
  PROCUREMENT_DB_HOST: "procurement-db-service"
  ORCHESTRATOR_DB_HOST: "orchestrator-db-service"
  ALERT_PROCESSOR_DB_HOST: "alert-processor-db-service"
+  AI_INSIGHTS_DB_HOST: "ai-insights-db-service"

  # Database Configuration
  DB_PORT: "5432"
@@ -78,6 +79,7 @@ data:
  PROCUREMENT_DB_NAME: "procurement_db"
  ORCHESTRATOR_DB_NAME: "orchestrator_db"
  ALERT_PROCESSOR_DB_NAME: "alert_processor_db"
+  AI_INSIGHTS_DB_NAME: "ai_insights_db"
  POSTGRES_INITDB_ARGS: "--encoding=UTF-8 --lc-collate=C --lc-ctype=C"

  # ================================================================
@@ -99,6 +101,7 @@ data:
  PRODUCTION_SERVICE_URL: "http://production-service:8000"
  ALERT_PROCESSOR_SERVICE_URL: "http://alert-processor-api:8010"
  ORCHESTRATOR_SERVICE_URL: "http://orchestrator-service:8000"
+  AI_INSIGHTS_SERVICE_URL: "http://ai-insights-service:8000"

  # ================================================================
  # AUTHENTICATION & SECURITY SETTINGS
--- a/infrastructure/kubernetes/base/kustomization.yaml
+++ b/infrastructure/kubernetes/base/kustomization.yaml
@@ -38,6 +38,7 @@ resources:
  - migrations/demo-session-migration-job.yaml
  - migrations/procurement-migration-job.yaml
  - migrations/orchestrator-migration-job.yaml
+  - migrations/ai-insights-migration-job.yaml

  # Demo initialization jobs (in Helm hook weight order)
  - jobs/demo-seed-rbac.yaml
@@ -98,6 +99,7 @@ resources:
  - components/databases/procurement-db.yaml
  - components/databases/orchestrator-db.yaml
  - components/databases/alert-processor-db.yaml
+  - components/databases/ai-insights-db.yaml

  # Demo session components
  - components/demo-session/database.yaml
@@ -123,6 +125,7 @@ resources:
  - components/orchestrator/orchestrator-service.yaml
  - components/alert-processor/alert-processor-service.yaml
  - components/alert-processor/alert-processor-api.yaml
+  - components/ai-insights/ai-insights-service.yaml

  # Frontend
  - components/frontend/frontend-service.yaml
@@ -166,6 +169,8 @@ images:
    newTag: latest
  - name: bakery/alert-processor
    newTag: latest
+  - name: bakery/ai-insights-service
+    newTag: latest
  - name: bakery/demo-session-service
    newTag: latest
  - name: bakery/gateway
--- a/infrastructure/kubernetes/base/migrations/ai-insights-migration-job.yaml
+++ b/infrastructure/kubernetes/base/migrations/ai-insights-migration-job.yaml
@@ -0,0 +1,65 @@
+# Enhanced migration job for ai-insights service with automatic table creation
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: ai-insights-migration
+  namespace: bakery-ia
+  labels:
+    app.kubernetes.io/name: ai-insights-migration
+    app.kubernetes.io/component: migration
+    app.kubernetes.io/part-of: bakery-ia
+spec:
+  backoffLimit: 3
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: ai-insights-migration
+        app.kubernetes.io/component: migration
+    spec:
+      initContainers:
+      - name: wait-for-db
+        image: postgres:17-alpine
+        command: ["sh", "-c", "until pg_isready -h ai-insights-db-service -p 5432; do sleep 2; done"]
+        resources:
+          requests:
+            memory: "32Mi"
+            cpu: "10m"
+          limits:
+            memory: "128Mi"
+            cpu: "100m"
+      containers:
+      - name: migrate
+        image: bakery/ai-insights-service:dev
+        command: ["python", "/app/shared/scripts/run_migrations.py", "ai_insights"]
+        env:
+        - name: AI_INSIGHTS_DATABASE_URL
+          valueFrom:
+            secretKeyRef:
+              name: database-secrets
+              key: AI_INSIGHTS_DATABASE_URL
+        - name: DATABASE_URL
+          valueFrom:
+            secretKeyRef:
+              name: database-secrets
+              key: AI_INSIGHTS_DATABASE_URL
+        - name: REDIS_URL
+          valueFrom:
+            secretKeyRef:
+              name: database-secrets
+              key: REDIS_URL
+        - name: DB_FORCE_RECREATE
+          valueFrom:
+            configMapKeyRef:
+              name: bakery-config
+              key: DB_FORCE_RECREATE
+              optional: true
+        - name: LOG_LEVEL
+          value: "INFO"
+        resources:
+          requests:
+            memory: "128Mi"
+            cpu: "50m"
+          limits:
+            memory: "512Mi"
+            cpu: "500m"
+      restartPolicy: OnFailure
--- a/infrastructure/kubernetes/base/secrets.yaml
+++ b/infrastructure/kubernetes/base/secrets.yaml
@@ -26,6 +26,7 @@ data:
  DEMO_SESSION_DB_USER: ZGVtb19zZXNzaW9uX3VzZXI=  # demo_session_user
  ORCHESTRATOR_DB_USER: b3JjaGVzdHJhdG9yX3VzZXI=  # orchestrator_user
  PROCUREMENT_DB_USER: cHJvY3VyZW1lbnRfdXNlcg==  # procurement_user
+  AI_INSIGHTS_DB_USER: YWlfaW5zaWdodHNfdXNlcg==  # ai_insights_user

  # Database Passwords (base64 encoded from .env)
  AUTH_DB_PASSWORD: djJvOHBqVWRSUVprR1JsbDlOV2JXdGt4WUFGcVBmOWw= # v2o8pjUdRQZkGRll...
@@ -45,6 +46,7 @@ data:
  DEMO_SESSION_DB_PASSWORD: ZGVtb19zZXNzaW9uX3Bhc3MxMjM=  # demo_session_pass123
  ORCHESTRATOR_DB_PASSWORD: b3JjaGVzdHJhdG9yX3Bhc3MxMjM=  # orchestrator_pass123
  PROCUREMENT_DB_PASSWORD: cHJvY3VyZW1lbnRfcGFzczEyMw==  # procurement_pass123
+  AI_INSIGHTS_DB_PASSWORD: YWlfaW5zaWdodHNfcGFzczEyMw==  # ai_insights_pass123

  # Database URLs (base64 encoded)
  AUTH_DATABASE_URL: cG9zdGdyZXNxbCthc3luY3BnOi8vYXV0aF91c2VyOnYybzhwalVkUlFaa0dSbGw5TldiV3RreFlBRnFQZjlsQGF1dGgtZGItc2VydmljZTo1NDMyL2F1dGhfZGI= # Updated with new password
@@ -64,6 +66,10 @@ data:
  DEMO_SESSION_DATABASE_URL: cG9zdGdyZXNxbCthc3luY3BnOi8vZGVtb19zZXNzaW9uX3VzZXI6ZGVtb19zZXNzaW9uX3Bhc3MxMjNAZGVtby1zZXNzaW9uLWRiLXNlcnZpY2U6NTQzMi9kZW1vX3Nlc3Npb25fZGI=  # postgresql+asyncpg://demo_session_user:demo_session_pass123@demo-session-db-service:5432/demo_session_db
  ORCHESTRATOR_DATABASE_URL: cG9zdGdyZXNxbCthc3luY3BnOi8vb3JjaGVzdHJhdG9yX3VzZXI6b3JjaGVzdHJhdG9yX3Bhc3MxMjNAb3JjaGVzdHJhdG9yLWRiLXNlcnZpY2U6NTQzMi9vcmNoZXN0cmF0b3JfZGI=  # postgresql+asyncpg://orchestrator_user:orchestrator_pass123@orchestrator-db-service:5432/orchestrator_db
  PROCUREMENT_DATABASE_URL: cG9zdGdyZXNxbCthc3luY3BnOi8vcHJvY3VyZW1lbnRfdXNlcjpwcm9jdXJlbWVudF9wYXNzMTIzQHByb2N1cmVtZW50LWRiLXNlcnZpY2U6NTQzMi9wcm9jdXJlbWVudF9kYg==  # postgresql+asyncpg://procurement_user:procurement_pass123@procurement-db-service:5432/procurement_db
+  AI_INSIGHTS_DATABASE_URL: cG9zdGdyZXNxbCthc3luY3BnOi8vYWlfaW5zaWdodHNfdXNlcjphaV9pbnNpZ2h0c19wYXNzMTIzQGFpLWluc2lnaHRzLWRiLXNlcnZpY2U6NTQzMi9haV9pbnNpZ2h0c19kYg==  # postgresql+asyncpg://ai_insights_user:ai_insights_pass123@ai-insights-db-service:5432/ai_insights_db
+
+  # Redis URL
+  REDIS_URL: cmVkaXM6Ly86T3hkbWRKamRWTlhwMzdNTkMySUZvTW5UcGZHR0Z2MWtAcmVkaXMtc2VydmljZTo2Mzc5LzA= # redis://:OxdmdJjdVNXp37MNC2IFoMnTpfGGFv1k@redis-service:6379/0

 ---
 apiVersion: v1
--- a/infrastructure/kubernetes/overlays/dev/kustomization.yaml
+++ b/infrastructure/kubernetes/overlays/dev/kustomization.yaml
@@ -391,6 +391,21 @@ patches:
          limits:
            memory: "256Mi"
            cpu: "200m"
+  - target:
+      group: apps
+      version: v1
+      kind: Deployment
+      name: ai-insights-db
+    patch: |-
+      - op: replace
+        path: /spec/template/spec/containers/0/resources
+        value:
+          requests:
+            memory: "64Mi"
+            cpu: "25m"
+          limits:
+            memory: "256Mi"
+            cpu: "200m"
  # Service patches
  - target:
      group: apps
@@ -572,6 +587,21 @@ patches:
          limits:
            memory: "1Gi"
            cpu: "500m"
+  - target:
+      group: apps
+      version: v1
+      kind: Deployment
+      name: ai-insights-service
+    patch: |-
+      - op: replace
+        path: /spec/template/spec/containers/0/resources
+        value:
+          requests:
+            memory: "128Mi"
+            cpu: "50m"
+          limits:
+            memory: "512Mi"
+            cpu: "300m"

 secretGenerator:
  - name: dev-secrets
@@ -613,6 +643,8 @@ images:
    newTag: dev
  - name: bakery/alert-processor
    newTag: dev
+  - name: bakery/ai-insights-service
+    newTag: dev
  - name: bakery/demo-session-service
    newTag: dev
  - name: bakery/gateway
@@ -649,6 +681,8 @@ replicas:
    count: 1
  - name: alert-processor-service
    count: 1
+  - name: ai-insights-service
+    count: 1
  - name: demo-session-service
    count: 1
  - name: gateway