Imporve monitoring 3

2026-01-09 11:18:20 +01:00
parent 8ca5d9c100
commit 43a3f35bd1
27 changed files with 1279 additions and 32 deletions
--- a/infrastructure/kubernetes/base/components/ai-insights/ai-insights-service.yaml
+++ b/infrastructure/kubernetes/base/components/ai-insights/ai-insights-service.yaml
@@ -99,7 +99,10 @@ spec:
        - name: OTEL_COLLECTOR_ENDPOINT
          value: "http://signoz-otel-collector.bakery-ia.svc.cluster.local:4318"
        - name: OTEL_EXPORTER_OTLP_ENDPOINT
-          value: "http://signoz-otel-collector.bakery-ia.svc.cluster.local:4318"
+          valueFrom:
+            configMapKeyRef:
+              name: bakery-config
+              key: OTEL_EXPORTER_OTLP_ENDPOINT
        - name: OTEL_SERVICE_NAME
          value: "ai-insights-service"
        - name: ENABLE_TRACING
--- a/infrastructure/kubernetes/base/components/auth/auth-service.yaml
+++ b/infrastructure/kubernetes/base/components/auth/auth-service.yaml
@@ -100,7 +100,10 @@ spec:
        - name: OTEL_COLLECTOR_ENDPOINT
          value: "http://signoz-otel-collector.bakery-ia.svc.cluster.local:4318"
        - name: OTEL_EXPORTER_OTLP_ENDPOINT
-          value: "http://signoz-otel-collector.bakery-ia.svc.cluster.local:4318"
+          valueFrom:
+            configMapKeyRef:
+              name: bakery-config
+              key: OTEL_EXPORTER_OTLP_ENDPOINT
        - name: OTEL_SERVICE_NAME
          value: "auth-service"
        - name: ENABLE_TRACING
--- a/infrastructure/kubernetes/base/components/distribution/distribution-service.yaml
+++ b/infrastructure/kubernetes/base/components/distribution/distribution-service.yaml
@@ -64,7 +64,10 @@ spec:
        - name: OTEL_COLLECTOR_ENDPOINT
          value: "http://signoz-otel-collector.bakery-ia.svc.cluster.local:4318"
        - name: OTEL_EXPORTER_OTLP_ENDPOINT
-          value: "http://signoz-otel-collector.bakery-ia.svc.cluster.local:4318"
+          valueFrom:
+            configMapKeyRef:
+              name: bakery-config
+              key: OTEL_EXPORTER_OTLP_ENDPOINT
        - name: OTEL_SERVICE_NAME
          value: "distribution-service"
        - name: ENABLE_TRACING
--- a/infrastructure/kubernetes/base/components/external/external-service.yaml
+++ b/infrastructure/kubernetes/base/components/external/external-service.yaml
@@ -92,7 +92,10 @@ spec:
        - name: OTEL_COLLECTOR_ENDPOINT
          value: "http://signoz-otel-collector.bakery-ia.svc.cluster.local:4318"
        - name: OTEL_EXPORTER_OTLP_ENDPOINT
-          value: "http://signoz-otel-collector.bakery-ia.svc.cluster.local:4318"
+          valueFrom:
+            configMapKeyRef:
+              name: bakery-config
+              key: OTEL_EXPORTER_OTLP_ENDPOINT
        - name: OTEL_SERVICE_NAME
          value: "external-service"
        - name: ENABLE_TRACING
--- a/infrastructure/kubernetes/base/components/forecasting/forecasting-service.yaml
+++ b/infrastructure/kubernetes/base/components/forecasting/forecasting-service.yaml
@@ -99,7 +99,10 @@ spec:
        - name: OTEL_COLLECTOR_ENDPOINT
          value: "http://signoz-otel-collector.bakery-ia.svc.cluster.local:4318"
        - name: OTEL_EXPORTER_OTLP_ENDPOINT
-          value: "http://signoz-otel-collector.bakery-ia.svc.cluster.local:4318"
+          valueFrom:
+            configMapKeyRef:
+              name: bakery-config
+              key: OTEL_EXPORTER_OTLP_ENDPOINT
        - name: OTEL_SERVICE_NAME
          value: "forecasting-service"
        - name: ENABLE_TRACING
--- a/infrastructure/kubernetes/base/components/infrastructure/gateway-service.yaml
+++ b/infrastructure/kubernetes/base/components/infrastructure/gateway-service.yaml
@@ -52,7 +52,10 @@ spec:
            name: whatsapp-secrets
        env:
        - name: OTEL_EXPORTER_OTLP_ENDPOINT
-          value: "http://signoz-otel-collector.bakery-ia.svc.cluster.local:4317"
+          valueFrom:
+            configMapKeyRef:
+              name: bakery-config
+              key: OTEL_EXPORTER_OTLP_ENDPOINT
        resources:
          requests:
            memory: "256Mi"
--- a/infrastructure/kubernetes/base/components/inventory/inventory-service.yaml
+++ b/infrastructure/kubernetes/base/components/inventory/inventory-service.yaml
@@ -99,7 +99,10 @@ spec:
        - name: OTEL_COLLECTOR_ENDPOINT
          value: "http://signoz-otel-collector.bakery-ia.svc.cluster.local:4318"
        - name: OTEL_EXPORTER_OTLP_ENDPOINT
-          value: "http://signoz-otel-collector.bakery-ia.svc.cluster.local:4318"
+          valueFrom:
+            configMapKeyRef:
+              name: bakery-config
+              key: OTEL_EXPORTER_OTLP_ENDPOINT
        - name: OTEL_SERVICE_NAME
          value: "inventory-service"
        - name: ENABLE_TRACING
--- a/infrastructure/kubernetes/base/components/notification/notification-service.yaml
+++ b/infrastructure/kubernetes/base/components/notification/notification-service.yaml
@@ -99,7 +99,10 @@ spec:
        - name: OTEL_COLLECTOR_ENDPOINT
          value: "http://signoz-otel-collector.bakery-ia.svc.cluster.local:4318"
        - name: OTEL_EXPORTER_OTLP_ENDPOINT
-          value: "http://signoz-otel-collector.bakery-ia.svc.cluster.local:4318"
+          valueFrom:
+            configMapKeyRef:
+              name: bakery-config
+              key: OTEL_EXPORTER_OTLP_ENDPOINT
        - name: OTEL_SERVICE_NAME
          value: "notification-service"
        - name: ENABLE_TRACING
--- a/infrastructure/kubernetes/base/components/orchestrator/orchestrator-service.yaml
+++ b/infrastructure/kubernetes/base/components/orchestrator/orchestrator-service.yaml
@@ -99,7 +99,10 @@ spec:
        - name: OTEL_COLLECTOR_ENDPOINT
          value: "http://signoz-otel-collector.bakery-ia.svc.cluster.local:4318"
        - name: OTEL_EXPORTER_OTLP_ENDPOINT
-          value: "http://signoz-otel-collector.bakery-ia.svc.cluster.local:4318"
+          valueFrom:
+            configMapKeyRef:
+              name: bakery-config
+              key: OTEL_EXPORTER_OTLP_ENDPOINT
        - name: OTEL_SERVICE_NAME
          value: "orchestrator-service"
        - name: ENABLE_TRACING
--- a/infrastructure/kubernetes/base/components/orders/orders-service.yaml
+++ b/infrastructure/kubernetes/base/components/orders/orders-service.yaml
@@ -99,7 +99,10 @@ spec:
        - name: OTEL_COLLECTOR_ENDPOINT
          value: "http://signoz-otel-collector.bakery-ia.svc.cluster.local:4318"
        - name: OTEL_EXPORTER_OTLP_ENDPOINT
-          value: "http://signoz-otel-collector.bakery-ia.svc.cluster.local:4318"
+          valueFrom:
+            configMapKeyRef:
+              name: bakery-config
+              key: OTEL_EXPORTER_OTLP_ENDPOINT
        - name: OTEL_SERVICE_NAME
          value: "orders-service"
        - name: ENABLE_TRACING
--- a/infrastructure/kubernetes/base/components/pos/pos-service.yaml
+++ b/infrastructure/kubernetes/base/components/pos/pos-service.yaml
@@ -99,7 +99,10 @@ spec:
        - name: OTEL_COLLECTOR_ENDPOINT
          value: "http://signoz-otel-collector.bakery-ia.svc.cluster.local:4318"
        - name: OTEL_EXPORTER_OTLP_ENDPOINT
-          value: "http://signoz-otel-collector.bakery-ia.svc.cluster.local:4318"
+          valueFrom:
+            configMapKeyRef:
+              name: bakery-config
+              key: OTEL_EXPORTER_OTLP_ENDPOINT
        - name: OTEL_SERVICE_NAME
          value: "pos-service"
        - name: ENABLE_TRACING
--- a/infrastructure/kubernetes/base/components/procurement/procurement-service.yaml
+++ b/infrastructure/kubernetes/base/components/procurement/procurement-service.yaml
@@ -99,7 +99,10 @@ spec:
        - name: OTEL_COLLECTOR_ENDPOINT
          value: "http://signoz-otel-collector.bakery-ia.svc.cluster.local:4318"
        - name: OTEL_EXPORTER_OTLP_ENDPOINT
-          value: "http://signoz-otel-collector.bakery-ia.svc.cluster.local:4318"
+          valueFrom:
+            configMapKeyRef:
+              name: bakery-config
+              key: OTEL_EXPORTER_OTLP_ENDPOINT
        - name: OTEL_SERVICE_NAME
          value: "procurement-service"
        - name: ENABLE_TRACING
--- a/infrastructure/kubernetes/base/components/production/production-service.yaml
+++ b/infrastructure/kubernetes/base/components/production/production-service.yaml
@@ -99,7 +99,10 @@ spec:
        - name: OTEL_COLLECTOR_ENDPOINT
          value: "http://signoz-otel-collector.bakery-ia.svc.cluster.local:4318"
        - name: OTEL_EXPORTER_OTLP_ENDPOINT
-          value: "http://signoz-otel-collector.bakery-ia.svc.cluster.local:4318"
+          valueFrom:
+            configMapKeyRef:
+              name: bakery-config
+              key: OTEL_EXPORTER_OTLP_ENDPOINT
        - name: OTEL_SERVICE_NAME
          value: "production-service"
        - name: ENABLE_TRACING
--- a/infrastructure/kubernetes/base/components/recipes/recipes-service.yaml
+++ b/infrastructure/kubernetes/base/components/recipes/recipes-service.yaml
@@ -99,7 +99,10 @@ spec:
        - name: OTEL_COLLECTOR_ENDPOINT
          value: "http://signoz-otel-collector.bakery-ia.svc.cluster.local:4318"
        - name: OTEL_EXPORTER_OTLP_ENDPOINT
-          value: "http://signoz-otel-collector.bakery-ia.svc.cluster.local:4318"
+          valueFrom:
+            configMapKeyRef:
+              name: bakery-config
+              key: OTEL_EXPORTER_OTLP_ENDPOINT
        - name: OTEL_SERVICE_NAME
          value: "recipes-service"
        - name: ENABLE_TRACING
--- a/infrastructure/kubernetes/base/components/sales/sales-service.yaml
+++ b/infrastructure/kubernetes/base/components/sales/sales-service.yaml
@@ -99,7 +99,10 @@ spec:
        - name: OTEL_COLLECTOR_ENDPOINT
          value: "http://signoz-otel-collector.bakery-ia.svc.cluster.local:4318"
        - name: OTEL_EXPORTER_OTLP_ENDPOINT
-          value: "http://signoz-otel-collector.bakery-ia.svc.cluster.local:4318"
+          valueFrom:
+            configMapKeyRef:
+              name: bakery-config
+              key: OTEL_EXPORTER_OTLP_ENDPOINT
        - name: OTEL_SERVICE_NAME
          value: "sales-service"
        - name: ENABLE_TRACING
--- a/infrastructure/kubernetes/base/components/suppliers/suppliers-service.yaml
+++ b/infrastructure/kubernetes/base/components/suppliers/suppliers-service.yaml
@@ -99,7 +99,10 @@ spec:
        - name: OTEL_COLLECTOR_ENDPOINT
          value: "http://signoz-otel-collector.bakery-ia.svc.cluster.local:4318"
        - name: OTEL_EXPORTER_OTLP_ENDPOINT
-          value: "http://signoz-otel-collector.bakery-ia.svc.cluster.local:4318"
+          valueFrom:
+            configMapKeyRef:
+              name: bakery-config
+              key: OTEL_EXPORTER_OTLP_ENDPOINT
        - name: OTEL_SERVICE_NAME
          value: "suppliers-service"
        - name: ENABLE_TRACING
--- a/infrastructure/kubernetes/base/components/tenant/tenant-service.yaml
+++ b/infrastructure/kubernetes/base/components/tenant/tenant-service.yaml
@@ -99,7 +99,10 @@ spec:
        - name: OTEL_COLLECTOR_ENDPOINT
          value: "http://signoz-otel-collector.bakery-ia.svc.cluster.local:4318"
        - name: OTEL_EXPORTER_OTLP_ENDPOINT
-          value: "http://signoz-otel-collector.bakery-ia.svc.cluster.local:4318"
+          valueFrom:
+            configMapKeyRef:
+              name: bakery-config
+              key: OTEL_EXPORTER_OTLP_ENDPOINT
        - name: OTEL_SERVICE_NAME
          value: "tenant-service"
        - name: ENABLE_TRACING
--- a/infrastructure/kubernetes/base/components/training/training-service.yaml
+++ b/infrastructure/kubernetes/base/components/training/training-service.yaml
@@ -99,7 +99,10 @@ spec:
        - name: OTEL_COLLECTOR_ENDPOINT
          value: "http://signoz-otel-collector.bakery-ia.svc.cluster.local:4318"
        - name: OTEL_EXPORTER_OTLP_ENDPOINT
-          value: "http://signoz-otel-collector.bakery-ia.svc.cluster.local:4318"
+          valueFrom:
+            configMapKeyRef:
+              name: bakery-config
+              key: OTEL_EXPORTER_OTLP_ENDPOINT
        - name: OTEL_SERVICE_NAME
          value: "training-service"
        - name: ENABLE_TRACING
--- a/infrastructure/kubernetes/base/configmap.yaml
+++ b/infrastructure/kubernetes/base/configmap.yaml
@@ -385,7 +385,8 @@ data:
  # OBSERVABILITY - SigNoz (Unified Monitoring)
  # ================================================================
  # OpenTelemetry Configuration - Direct to SigNoz
-  OTEL_EXPORTER_OTLP_ENDPOINT: "http://signoz-otel-collector.bakery-ia.svc.cluster.local:4317"
+  # IMPORTANT: gRPC endpoints should NOT include http:// prefix
+  OTEL_EXPORTER_OTLP_ENDPOINT: "signoz-otel-collector.bakery-ia.svc.cluster.local:4317"
  OTEL_EXPORTER_OTLP_PROTOCOL: "grpc"
  OTEL_SERVICE_NAME: "bakery-ia"
  OTEL_RESOURCE_ATTRIBUTES: "deployment.environment=development"
--- a/infrastructure/kubernetes/fix-otel-endpoints.sh
+++ b/infrastructure/kubernetes/fix-otel-endpoints.sh
@@ -0,0 +1,60 @@
+#!/bin/bash
+
+# Fix OTEL endpoint configuration in all service manifests
+# This script replaces hardcoded OTEL_EXPORTER_OTLP_ENDPOINT values
+# with references to the central bakery-config ConfigMap
+
+set -e
+
+GREEN='\033[0;32m'
+BLUE='\033[0;34m'
+NC='\033[0m'
+
+echo -e "${BLUE}Fixing OTEL endpoint configuration in all services...${NC}"
+echo ""
+
+# Find all service YAML files
+SERVICE_FILES=$(find infrastructure/kubernetes/base/components -name "*-service.yaml")
+
+for file in $SERVICE_FILES; do
+    # Check if file contains hardcoded OTEL_EXPORTER_OTLP_ENDPOINT
+    if grep -q "name: OTEL_EXPORTER_OTLP_ENDPOINT" "$file"; then
+        # Check if it's already using configMapKeyRef
+        if grep -A 3 "name: OTEL_EXPORTER_OTLP_ENDPOINT" "$file" | grep -q "configMapKeyRef"; then
+            echo -e "${GREEN}✓ $file already using ConfigMap${NC}"
+        else
+            echo -e "${BLUE}→ Fixing $file${NC}"
+
+            # Create a temporary file
+            tmp_file=$(mktemp)
+
+            # Process the file
+            awk '
+            /name: OTEL_EXPORTER_OTLP_ENDPOINT/ {
+                print $0
+                # Read and skip the next line (value line)
+                getline
+                # Output the configMapKeyRef instead
+                print "          valueFrom:"
+                print "            configMapKeyRef:"
+                print "              name: bakery-config"
+                print "              key: OTEL_EXPORTER_OTLP_ENDPOINT"
+                next
+            }
+            { print }
+            ' "$file" > "$tmp_file"
+
+            # Replace original file
+            mv "$tmp_file" "$file"
+            echo -e "${GREEN}  ✓ Fixed${NC}"
+        fi
+    fi
+done
+
+echo ""
+echo -e "${GREEN}✓ All service files processed!${NC}"
+echo ""
+echo "Next steps:"
+echo "1. Review changes: git diff infrastructure/kubernetes/base/components"
+echo "2. Apply changes: kubectl apply -k infrastructure/kubernetes/overlays/dev"
+echo "3. Restart services: kubectl rollout restart deployment -n bakery-ia --all"
--- a/infrastructure/kubernetes/overlays/prod/prod-configmap.yaml
+++ b/infrastructure/kubernetes/overlays/prod/prod-configmap.yaml
@@ -23,7 +23,8 @@ data:
  ENABLE_LOGS: "true"

  # OpenTelemetry Configuration - Direct to SigNoz
-  OTEL_EXPORTER_OTLP_ENDPOINT: "http://signoz-otel-collector.bakery-ia.svc.cluster.local:4317"
+  # IMPORTANT: gRPC endpoints should NOT include http:// prefix
+  OTEL_EXPORTER_OTLP_ENDPOINT: "signoz-otel-collector.bakery-ia.svc.cluster.local:4317"
  OTEL_EXPORTER_OTLP_PROTOCOL: "grpc"
  OTEL_SERVICE_NAME: "bakery-ia"
  OTEL_RESOURCE_ATTRIBUTES: "deployment.environment=production,cluster.name=bakery-ia-prod"