bakery-ia/infrastructure/helm/signoz-values-dev.yaml

# SigNoz Helm Chart Values - Development Environment
# Optimized for local development with minimal resource usage
# DEPLOYED IN bakery-ia NAMESPACE - Ingress managed by bakery-ingress
#
# Official Chart: https://github.com/SigNoz/charts
# Install Command: helm install signoz signoz/signoz -n bakery-ia -f signoz-values-dev.yaml

global:
  storageClass: "standard"
  clusterName: "bakery-ia-dev"
  domain: "monitoring.bakery-ia.local"
  # Docker Hub credentials - applied to all sub-charts (including Zookeeper, ClickHouse, etc)
  imagePullSecrets:
    - dockerhub-creds

# Docker Hub credentials for pulling images (root level for SigNoz components)
imagePullSecrets:
  - dockerhub-creds

# SignOz Main Component (includes frontend and query service)
signoz:
  replicaCount: 1

  service:
    type: ClusterIP
    port: 8080

  # DISABLE built-in ingress - using unified bakery-ingress instead
  # Route configured in infrastructure/kubernetes/overlays/dev/dev-ingress.yaml
  ingress:
    enabled: false

  resources:
    requests:
      cpu: 100m  # Combined frontend + query service
      memory: 256Mi
    limits:
      cpu: 1000m
      memory: 1Gi

  # Environment variables (new format - replaces configVars)
  env:
    signoz_telemetrystore_provider: "clickhouse"
    dot_metrics_enabled: "true"
    signoz_emailing_enabled: "false"
    signoz_alertmanager_provider: "signoz"
    # Retention for dev (7 days)
    signoz_traces_ttl_duration_hrs: "168"
    signoz_metrics_ttl_duration_hrs: "168"
    signoz_logs_ttl_duration_hrs: "168"
    # OpAMP Server Configuration
    signoz_opamp_server_enabled: "true"
    signoz_opamp_server_endpoint: "0.0.0.0:4320"

  persistence:
    enabled: true
    size: 5Gi
    storageClass: "standard"

# AlertManager Configuration
alertmanager:
  replicaCount: 1
  image:
    repository: signoz/alertmanager
    tag: 0.23.5
    pullPolicy: IfNotPresent

  service:
    type: ClusterIP
    port: 9093

  resources:
    requests:
      cpu: 25m  # Reduced for local dev
      memory: 64Mi  # Reduced for local dev
    limits:
      cpu: 200m
      memory: 256Mi

  persistence:
    enabled: true
    size: 2Gi
    storageClass: "standard"

  config:
    global:
      resolve_timeout: 5m
    route:
      group_by: ['alertname', 'cluster', 'service']
      group_wait: 10s
      group_interval: 10s
      repeat_interval: 12h
      receiver: 'default'
    receivers:
      - name: 'default'
        # Add email, slack, webhook configs here

# ClickHouse Configuration - Time Series Database
# Minimal resources for local development on constrained Kind cluster
clickhouse:
  enabled: true
  installCustomStorageClass: false

  image:
    registry: docker.io
    repository: clickhouse/clickhouse-server
    tag: 25.5.6  # Official recommended version

  # Reduce ClickHouse resource requests for local dev
  clickhouse:
    resources:
      requests:
        cpu: 200m  # Reduced from default 500m
        memory: 512Mi
      limits:
        cpu: 1000m
        memory: 1Gi

  persistence:
    enabled: true
    size: 20Gi

# Zookeeper Configuration (required by ClickHouse)
zookeeper:
  enabled: true
  replicaCount: 1  # Single replica for dev

  image:
    tag: 3.7.1  # Official recommended version

  resources:
    requests:
      cpu: 100m
      memory: 256Mi
    limits:
      cpu: 500m
      memory: 512Mi

  persistence:
    enabled: true
    size: 5Gi

# OpenTelemetry Collector - Data ingestion endpoint for all telemetry
otelCollector:
  enabled: true
  replicaCount: 1

  image:
    repository: signoz/signoz-otel-collector
    tag: v0.129.12  # Latest recommended version

  # OpAMP Configuration - Enabled for dynamic configuration management
  # Note: OpAMP allows remote configuration management via SigNoz backend
  # This replaces the manual kubectl patch approach

  # Init containers for the Otel Collector pod
  initContainers:
    fix-postgres-tls:
      enabled: true
      image:
        registry: docker.io
        repository: busybox
        tag: 1.35
        pullPolicy: IfNotPresent
      command:
        - sh
        - -c
        - |
          echo "Fixing PostgreSQL TLS file permissions..."
          cp /etc/postgres-tls-source/* /etc/postgres-tls/
          chmod 600 /etc/postgres-tls/server-key.pem
          chmod 644 /etc/postgres-tls/server-cert.pem
          chmod 644 /etc/postgres-tls/ca-cert.pem
          echo "PostgreSQL TLS permissions fixed"
      volumeMounts:
        - name: postgres-tls-source
          mountPath: /etc/postgres-tls-source
          readOnly: true
        - name: postgres-tls-fixed
          mountPath: /etc/postgres-tls
          readOnly: false

  # Service configuration - expose both gRPC and HTTP endpoints
  service:
    type: ClusterIP
    ports:
      # gRPC receivers
      - name: otlp-grpc
        port: 4317
        targetPort: 4317
        protocol: TCP
      # HTTP receivers
      - name: otlp-http
        port: 4318
        targetPort: 4318
        protocol: TCP
      # Prometheus remote write
      - name: prometheus
        port: 8889
        targetPort: 8889
        protocol: TCP
      # Metrics
      - name: metrics
        port: 8888
        targetPort: 8888
        protocol: TCP

  resources:
    requests:
      cpu: 50m   # Reduced from 100m
      memory: 128Mi  # Reduced from 256Mi
    limits:
      cpu: 500m
      memory: 512Mi

  # Additional environment variables for receivers
  additionalEnvs:
    POSTGRES_MONITOR_USER: "monitoring"
    POSTGRES_MONITOR_PASSWORD: "monitoring_369f9c001f242b07ef9e2826e17169ca"
    REDIS_PASSWORD: "OxdmdJjdVNXp37MNC2IFoMnTpfGGFv1k"
    RABBITMQ_USER: "bakery"
    RABBITMQ_PASSWORD: "forecast123"

  # Mount TLS certificates for secure connections
  extraVolumes:
    - name: redis-tls
      secret:
        secretName: redis-tls-secret
    - name: postgres-tls
      secret:
        secretName: postgres-tls
    - name: postgres-tls-fixed
      emptyDir: {}

  extraVolumeMounts:
    - name: redis-tls
      mountPath: /etc/redis-tls
      readOnly: true
    - name: postgres-tls
      mountPath: /etc/postgres-tls-source
      readOnly: true
    - name: postgres-tls-fixed
      mountPath: /etc/postgres-tls
      readOnly: false

  # Enable OpAMP for dynamic configuration management
  command:
    name: /signoz-otel-collector
    extraArgs:
      - --config=/conf/otel-collector-config.yaml
      - --manager-config=/conf/otel-collector-opamp-config.yaml
      - --feature-gates=-pkg.translator.prometheus.NormalizeName

  # OpenTelemetry Collector configuration
  config:
    # Connectors - bridge between pipelines
    connectors:
      signozmeter:
        dimensions:
          - name: service.name
          - name: deployment.environment
          - name: host.name
        metrics_flush_interval: 1h

    receivers:
      # OTLP receivers for traces, metrics, and logs from applications
      # All application telemetry is pushed via OTLP protocol
      otlp:
        protocols:
          grpc:
            endpoint: 0.0.0.0:4317
          http:
            endpoint: 0.0.0.0:4318
            cors:
              allowed_origins:
                - "*"

      # PostgreSQL receivers for database metrics
      # ENABLED: Monitor users configured and credentials stored in secrets
      # Collects metrics directly from PostgreSQL databases with proper TLS
      postgresql/auth:
        endpoint: auth-db-service.bakery-ia:5432
        username: ${env:POSTGRES_MONITOR_USER}
        password: ${env:POSTGRES_MONITOR_PASSWORD}
        databases:
          - auth_db
        collection_interval: 60s
        tls:
          insecure: false
          cert_file: /etc/postgres-tls/server-cert.pem
          key_file: /etc/postgres-tls/server-key.pem
          ca_file: /etc/postgres-tls/ca-cert.pem

      postgresql/inventory:
        endpoint: inventory-db-service.bakery-ia:5432
        username: ${env:POSTGRES_MONITOR_USER}
        password: ${env:POSTGRES_MONITOR_PASSWORD}
        databases:
          - inventory_db
        collection_interval: 60s
        tls:
          insecure: false
          cert_file: /etc/postgres-tls/server-cert.pem
          key_file: /etc/postgres-tls/server-key.pem
          ca_file: /etc/postgres-tls/ca-cert.pem

      postgresql/orders:
        endpoint: orders-db-service.bakery-ia:5432
        username: ${env:POSTGRES_MONITOR_USER}
        password: ${env:POSTGRES_MONITOR_PASSWORD}
        databases:
          - orders_db
        collection_interval: 60s
        tls:
          insecure: false
          cert_file: /etc/postgres-tls/server-cert.pem
          key_file: /etc/postgres-tls/server-key.pem
          ca_file: /etc/postgres-tls/ca-cert.pem

      postgresql/ai-insights:
        endpoint: ai-insights-db-service.bakery-ia:5432
        username: ${env:POSTGRES_MONITOR_USER}
        password: ${env:POSTGRES_MONITOR_PASSWORD}
        databases:
          - ai_insights_db
        collection_interval: 60s
        tls:
          insecure: false
          cert_file: /etc/postgres-tls/server-cert.pem
          key_file: /etc/postgres-tls/server-key.pem
          ca_file: /etc/postgres-tls/ca-cert.pem

      postgresql/alert-processor:
        endpoint: alert-processor-db-service.bakery-ia:5432
        username: ${env:POSTGRES_MONITOR_USER}
        password: ${env:POSTGRES_MONITOR_PASSWORD}
        databases:
          - alert_processor_db
        collection_interval: 60s
        tls:
          insecure: false
          cert_file: /etc/postgres-tls/server-cert.pem
          key_file: /etc/postgres-tls/server-key.pem
          ca_file: /etc/postgres-tls/ca-cert.pem

      postgresql/distribution:
        endpoint: distribution-db-service.bakery-ia:5432
        username: ${env:POSTGRES_MONITOR_USER}
        password: ${env:POSTGRES_MONITOR_PASSWORD}
        databases:
          - distribution_db
        collection_interval: 60s
        tls:
          insecure: false
          cert_file: /etc/postgres-tls/server-cert.pem
          key_file: /etc/postgres-tls/server-key.pem
          ca_file: /etc/postgres-tls/ca-cert.pem

      postgresql/external:
        endpoint: external-db-service.bakery-ia:5432
        username: ${env:POSTGRES_MONITOR_USER}
        password: ${env:POSTGRES_MONITOR_PASSWORD}
        databases:
          - external_db
        collection_interval: 60s
        tls:
          insecure: false
          cert_file: /etc/postgres-tls/server-cert.pem
          key_file: /etc/postgres-tls/server-key.pem
          ca_file: /etc/postgres-tls/ca-cert.pem

      postgresql/forecasting:
        endpoint: forecasting-db-service.bakery-ia:5432
        username: ${env:POSTGRES_MONITOR_USER}
        password: ${env:POSTGRES_MONITOR_PASSWORD}
        databases:
          - forecasting_db
        collection_interval: 60s
        tls:
          insecure: false
          cert_file: /etc/postgres-tls/server-cert.pem
          key_file: /etc/postgres-tls/server-key.pem
          ca_file: /etc/postgres-tls/ca-cert.pem

      postgresql/notification:
        endpoint: notification-db-service.bakery-ia:5432
        username: ${env:POSTGRES_MONITOR_USER}
        password: ${env:POSTGRES_MONITOR_PASSWORD}
        databases:
          - notification_db
        collection_interval: 60s
        tls:
          insecure: false
          cert_file: /etc/postgres-tls/server-cert.pem
          key_file: /etc/postgres-tls/server-key.pem
          ca_file: /etc/postgres-tls/ca-cert.pem

      postgresql/orchestrator:
        endpoint: orchestrator-db-service.bakery-ia:5432
        username: ${env:POSTGRES_MONITOR_USER}
        password: ${env:POSTGRES_MONITOR_PASSWORD}
        databases:
          - orchestrator_db
        collection_interval: 60s
        tls:
          insecure: false
          cert_file: /etc/postgres-tls/server-cert.pem
          key_file: /etc/postgres-tls/server-key.pem
          ca_file: /etc/postgres-tls/ca-cert.pem

      postgresql/pos:
        endpoint: pos-db-service.bakery-ia:5432
        username: ${env:POSTGRES_MONITOR_USER}
        password: ${env:POSTGRES_MONITOR_PASSWORD}
        databases:
          - pos_db
        collection_interval: 60s
        tls:
          insecure: false
          cert_file: /etc/postgres-tls/server-cert.pem
          key_file: /etc/postgres-tls/server-key.pem
          ca_file: /etc/postgres-tls/ca-cert.pem

      postgresql/procurement:
        endpoint: procurement-db-service.bakery-ia:5432
        username: ${env:POSTGRES_MONITOR_USER}
        password: ${env:POSTGRES_MONITOR_PASSWORD}
        databases:
          - procurement_db
        collection_interval: 60s
        tls:
          insecure: false
          cert_file: /etc/postgres-tls/server-cert.pem
          key_file: /etc/postgres-tls/server-key.pem
          ca_file: /etc/postgres-tls/ca-cert.pem

      postgresql/production:
        endpoint: production-db-service.bakery-ia:5432
        username: ${env:POSTGRES_MONITOR_USER}
        password: ${env:POSTGRES_MONITOR_PASSWORD}
        databases:
          - production_db
        collection_interval: 60s
        tls:
          insecure: false
          cert_file: /etc/postgres-tls/server-cert.pem
          key_file: /etc/postgres-tls/server-key.pem
          ca_file: /etc/postgres-tls/ca-cert.pem

      postgresql/recipes:
        endpoint: recipes-db-service.bakery-ia:5432
        username: ${env:POSTGRES_MONITOR_USER}
        password: ${env:POSTGRES_MONITOR_PASSWORD}
        databases:
          - recipes_db
        collection_interval: 60s
        tls:
          insecure: false
          cert_file: /etc/postgres-tls/server-cert.pem
          key_file: /etc/postgres-tls/server-key.pem
          ca_file: /etc/postgres-tls/ca-cert.pem

      postgresql/sales:
        endpoint: sales-db-service.bakery-ia:5432
        username: ${env:POSTGRES_MONITOR_USER}
        password: ${env:POSTGRES_MONITOR_PASSWORD}
        databases:
          - sales_db
        collection_interval: 60s
        tls:
          insecure: false
          cert_file: /etc/postgres-tls/server-cert.pem
          key_file: /etc/postgres-tls/server-key.pem
          ca_file: /etc/postgres-tls/ca-cert.pem

      postgresql/suppliers:
        endpoint: suppliers-db-service.bakery-ia:5432
        username: ${env:POSTGRES_MONITOR_USER}
        password: ${env:POSTGRES_MONITOR_PASSWORD}
        databases:
          - suppliers_db
        collection_interval: 60s
        tls:
          insecure: false
          cert_file: /etc/postgres-tls/server-cert.pem
          key_file: /etc/postgres-tls/server-key.pem
          ca_file: /etc/postgres-tls/ca-cert.pem

      postgresql/tenant:
        endpoint: tenant-db-service.bakery-ia:5432
        username: ${env:POSTGRES_MONITOR_USER}
        password: ${env:POSTGRES_MONITOR_PASSWORD}
        databases:
          - tenant_db
        collection_interval: 60s
        tls:
          insecure: false
          cert_file: /etc/postgres-tls/server-cert.pem
          key_file: /etc/postgres-tls/server-key.pem
          ca_file: /etc/postgres-tls/ca-cert.pem

      postgresql/training:
        endpoint: training-db-service.bakery-ia:5432
        username: ${env:POSTGRES_MONITOR_USER}
        password: ${env:POSTGRES_MONITOR_PASSWORD}
        databases:
          - training_db
        collection_interval: 60s
        tls:
          insecure: false
          cert_file: /etc/postgres-tls/server-cert.pem
          key_file: /etc/postgres-tls/server-key.pem
          ca_file: /etc/postgres-tls/ca-cert.pem

      # Redis receiver for cache metrics
      # ENABLED: Using existing credentials from redis-secrets with TLS
      redis:
        endpoint: redis-service.bakery-ia:6379
        password: ${env:REDIS_PASSWORD}
        collection_interval: 60s
        transport: tcp
        tls:
          insecure_skip_verify: false
          cert_file: /etc/redis-tls/redis-cert.pem
          key_file: /etc/redis-tls/redis-key.pem
          ca_file: /etc/redis-tls/ca-cert.pem
        metrics:
          redis.maxmemory:
            enabled: true
          redis.cmd.latency:
            enabled: true

      # RabbitMQ receiver via management API
      # ENABLED: Using existing credentials from rabbitmq-secrets
      rabbitmq:
        endpoint: http://rabbitmq-service.bakery-ia:15672
        username: ${env:RABBITMQ_USER}
        password: ${env:RABBITMQ_PASSWORD}
        collection_interval: 30s

    processors:
      # Batch processor for better performance (optimized for high throughput)
      batch:
        timeout: 1s
        send_batch_size: 10000  # Increased from 1024 for better performance
        send_batch_max_size: 10000

      # Batch processor for meter data
      batch/meter:
        timeout: 1s
        send_batch_size: 20000
        send_batch_max_size: 25000

      # Memory limiter to prevent OOM
      memory_limiter:
        check_interval: 1s
        limit_mib: 400
        spike_limit_mib: 100

      # Resource detection
      resourcedetection:
        detectors: [env, system, docker]
        timeout: 5s

      # SigNoz span metrics processor with delta aggregation (recommended)
      # Generates RED metrics (Rate, Error, Duration) from trace spans
      signozspanmetrics/delta:
        aggregation_temporality: AGGREGATION_TEMPORALITY_DELTA
        metrics_exporter: signozclickhousemetrics
        latency_histogram_buckets: [100us, 1ms, 2ms, 6ms, 10ms, 50ms, 100ms, 250ms, 500ms, 1000ms, 1400ms, 2000ms, 5s, 10s, 20s, 40s, 60s]
        dimensions_cache_size: 100000
        dimensions:
          - name: service.namespace
            default: default
          - name: deployment.environment
            default: default
          - name: signoz.collector.id

    exporters:
      # ClickHouse exporter for traces
      clickhousetraces:
        datasource: tcp://admin:27ff0399-0d3a-4bd8-919d-17c2181e6fb9@signoz-clickhouse:9000/?database=signoz_traces
        timeout: 10s
        retry_on_failure:
          enabled: true
          initial_interval: 5s
          max_interval: 30s
          max_elapsed_time: 300s

      # ClickHouse exporter for metrics
      signozclickhousemetrics:
        dsn: "tcp://admin:27ff0399-0d3a-4bd8-919d-17c2181e6fb9@signoz-clickhouse:9000/signoz_metrics"
        timeout: 10s
        retry_on_failure:
          enabled: true
          initial_interval: 5s
          max_interval: 30s
          max_elapsed_time: 300s

      # ClickHouse exporter for meter data (usage metrics)
      signozclickhousemeter:
        dsn: "tcp://admin:27ff0399-0d3a-4bd8-919d-17c2181e6fb9@signoz-clickhouse:9000/signoz_meter"
        timeout: 45s
        sending_queue:
          enabled: false

      # ClickHouse exporter for logs
      clickhouselogsexporter:
        dsn: tcp://admin:27ff0399-0d3a-4bd8-919d-17c2181e6fb9@signoz-clickhouse:9000/?database=signoz_logs
        timeout: 10s
        retry_on_failure:
          enabled: true
          initial_interval: 5s
          max_interval: 30s

      # Metadata exporter for service metadata
      metadataexporter:
        dsn: "tcp://admin:27ff0399-0d3a-4bd8-919d-17c2181e6fb9@signoz-clickhouse:9000/signoz_metadata"
        timeout: 10s
        cache:
          provider: in_memory

      # Debug exporter for debugging (optional)
      debug:
        verbosity: detailed
        sampling_initial: 5
        sampling_thereafter: 200

    service:
      pipelines:
        # Traces pipeline - exports to ClickHouse and signozmeter connector
        traces:
          receivers: [otlp]
          processors: [memory_limiter, batch, signozspanmetrics/delta, resourcedetection]
          exporters: [clickhousetraces, metadataexporter, signozmeter]

        # Metrics pipeline
        metrics:
          receivers: [otlp, 
            postgresql/auth, postgresql/inventory, postgresql/orders, 
            postgresql/ai-insights, postgresql/alert-processor, postgresql/distribution,
            postgresql/external, postgresql/forecasting, postgresql/notification,
            postgresql/orchestrator, postgresql/pos, postgresql/procurement,
            postgresql/production, postgresql/recipes, postgresql/sales,
            postgresql/suppliers, postgresql/tenant, postgresql/training,
            redis, rabbitmq]
          processors: [memory_limiter, batch, resourcedetection]
          exporters: [signozclickhousemetrics]

        # Meter pipeline - receives from signozmeter connector
        metrics/meter:
          receivers: [signozmeter]
          processors: [batch/meter]
          exporters: [signozclickhousemeter]

        # Logs pipeline
        logs:
          receivers: [otlp]
          processors: [memory_limiter, batch, resourcedetection]
          exporters: [clickhouselogsexporter]

# Additional Configuration
serviceAccount:
  create: true
  annotations: {}
  name: ""

# Security Context
securityContext:
  runAsNonRoot: true
  runAsUser: 1000
  fsGroup: 1000

# Network Policies (disabled for dev)
networkPolicy:
  enabled: false

# Monitoring SigNoz itself
selfMonitoring:
  enabled: true
  serviceMonitor:
    enabled: false