854 lines
27 KiB
YAML
854 lines
27 KiB
YAML
# SigNoz Helm Chart Values - Development Environment
|
|
# Optimized for local development with minimal resource usage
|
|
# DEPLOYED IN bakery-ia NAMESPACE - Ingress managed by bakery-ingress
|
|
#
|
|
# Official Chart: https://github.com/SigNoz/charts
|
|
# Install Command: helm install signoz signoz/signoz -n bakery-ia -f signoz-values-dev.yaml
|
|
|
|
global:
|
|
storageClass: "standard"
|
|
clusterName: "bakery-ia-dev"
|
|
domain: "monitoring.bakery-ia.local"
|
|
# Docker Hub credentials - applied to all sub-charts (including Zookeeper, ClickHouse, etc)
|
|
imagePullSecrets:
|
|
- dockerhub-creds
|
|
|
|
# Docker Hub credentials for pulling images (root level for SigNoz components)
|
|
imagePullSecrets:
|
|
- dockerhub-creds
|
|
|
|
# SignOz Main Component (includes frontend and query service)
|
|
signoz:
|
|
replicaCount: 1
|
|
|
|
service:
|
|
type: ClusterIP
|
|
port: 8080
|
|
|
|
# DISABLE built-in ingress - using unified bakery-ingress instead
|
|
# Route configured in infrastructure/kubernetes/overlays/dev/dev-ingress.yaml
|
|
ingress:
|
|
enabled: false
|
|
|
|
resources:
|
|
requests:
|
|
cpu: 100m # Combined frontend + query service
|
|
memory: 256Mi
|
|
limits:
|
|
cpu: 1000m
|
|
memory: 1Gi
|
|
|
|
# Environment variables (new format - replaces configVars)
|
|
env:
|
|
signoz_telemetrystore_provider: "clickhouse"
|
|
dot_metrics_enabled: "true"
|
|
signoz_emailing_enabled: "false"
|
|
signoz_alertmanager_provider: "signoz"
|
|
# Retention for dev (7 days)
|
|
signoz_traces_ttl_duration_hrs: "168"
|
|
signoz_metrics_ttl_duration_hrs: "168"
|
|
signoz_logs_ttl_duration_hrs: "168"
|
|
# OpAMP Server Configuration - DISABLED for dev (causes gRPC instability)
|
|
signoz_opamp_server_enabled: "false"
|
|
# signoz_opamp_server_endpoint: "0.0.0.0:4320"
|
|
|
|
persistence:
|
|
enabled: true
|
|
size: 5Gi
|
|
storageClass: "standard"
|
|
|
|
# AlertManager Configuration
|
|
alertmanager:
|
|
replicaCount: 1
|
|
image:
|
|
repository: signoz/alertmanager
|
|
tag: 0.23.5
|
|
pullPolicy: IfNotPresent
|
|
|
|
service:
|
|
type: ClusterIP
|
|
port: 9093
|
|
|
|
resources:
|
|
requests:
|
|
cpu: 25m # Reduced for local dev
|
|
memory: 64Mi # Reduced for local dev
|
|
limits:
|
|
cpu: 200m
|
|
memory: 256Mi
|
|
|
|
persistence:
|
|
enabled: true
|
|
size: 2Gi
|
|
storageClass: "standard"
|
|
|
|
config:
|
|
global:
|
|
resolve_timeout: 5m
|
|
route:
|
|
group_by: ['alertname', 'cluster', 'service']
|
|
group_wait: 10s
|
|
group_interval: 10s
|
|
repeat_interval: 12h
|
|
receiver: 'default'
|
|
receivers:
|
|
- name: 'default'
|
|
# Add email, slack, webhook configs here
|
|
|
|
# ClickHouse Configuration - Time Series Database
|
|
# Minimal resources for local development on constrained Kind cluster
|
|
clickhouse:
|
|
enabled: true
|
|
installCustomStorageClass: false
|
|
|
|
image:
|
|
registry: docker.io
|
|
repository: clickhouse/clickhouse-server
|
|
tag: 25.5.6 # Official recommended version
|
|
|
|
# Reduce ClickHouse resource requests for local dev
|
|
clickhouse:
|
|
resources:
|
|
requests:
|
|
cpu: 200m # Reduced from default 500m
|
|
memory: 512Mi
|
|
limits:
|
|
cpu: 1000m
|
|
memory: 1Gi
|
|
|
|
persistence:
|
|
enabled: true
|
|
size: 20Gi
|
|
|
|
# Zookeeper Configuration (required by ClickHouse)
|
|
zookeeper:
|
|
enabled: true
|
|
replicaCount: 1 # Single replica for dev
|
|
|
|
image:
|
|
tag: 3.7.1 # Official recommended version
|
|
|
|
resources:
|
|
requests:
|
|
cpu: 100m
|
|
memory: 256Mi
|
|
limits:
|
|
cpu: 500m
|
|
memory: 512Mi
|
|
|
|
persistence:
|
|
enabled: true
|
|
size: 5Gi
|
|
|
|
# OpenTelemetry Collector - Data ingestion endpoint for all telemetry
|
|
otelCollector:
|
|
enabled: true
|
|
replicaCount: 1
|
|
|
|
image:
|
|
repository: signoz/signoz-otel-collector
|
|
tag: v0.129.12 # Latest recommended version
|
|
|
|
# OpAMP Configuration - DISABLED for development
|
|
# OpAMP is designed for production with remote config management
|
|
# In dev, it causes gRPC instability and collector reloads
|
|
# We use static configuration instead
|
|
|
|
# Init containers for the Otel Collector pod
|
|
initContainers:
|
|
fix-postgres-tls:
|
|
enabled: true
|
|
image:
|
|
registry: docker.io
|
|
repository: busybox
|
|
tag: 1.35
|
|
pullPolicy: IfNotPresent
|
|
command:
|
|
- sh
|
|
- -c
|
|
- |
|
|
echo "Fixing PostgreSQL TLS file permissions..."
|
|
cp /etc/postgres-tls-source/* /etc/postgres-tls/
|
|
chmod 600 /etc/postgres-tls/server-key.pem
|
|
chmod 644 /etc/postgres-tls/server-cert.pem
|
|
chmod 644 /etc/postgres-tls/ca-cert.pem
|
|
echo "PostgreSQL TLS permissions fixed"
|
|
volumeMounts:
|
|
- name: postgres-tls-source
|
|
mountPath: /etc/postgres-tls-source
|
|
readOnly: true
|
|
- name: postgres-tls-fixed
|
|
mountPath: /etc/postgres-tls
|
|
readOnly: false
|
|
|
|
# Service configuration - expose both gRPC and HTTP endpoints
|
|
service:
|
|
type: ClusterIP
|
|
ports:
|
|
# gRPC receivers
|
|
- name: otlp-grpc
|
|
port: 4317
|
|
targetPort: 4317
|
|
protocol: TCP
|
|
# HTTP receivers
|
|
- name: otlp-http
|
|
port: 4318
|
|
targetPort: 4318
|
|
protocol: TCP
|
|
# Prometheus remote write
|
|
- name: prometheus
|
|
port: 8889
|
|
targetPort: 8889
|
|
protocol: TCP
|
|
# Metrics
|
|
- name: metrics
|
|
port: 8888
|
|
targetPort: 8888
|
|
protocol: TCP
|
|
|
|
resources:
|
|
requests:
|
|
cpu: 50m # Reduced from 100m
|
|
memory: 128Mi # Reduced from 256Mi
|
|
limits:
|
|
cpu: 500m
|
|
memory: 512Mi
|
|
|
|
# Additional environment variables for receivers
|
|
additionalEnvs:
|
|
POSTGRES_MONITOR_USER: "monitoring"
|
|
POSTGRES_MONITOR_PASSWORD: "monitoring_369f9c001f242b07ef9e2826e17169ca"
|
|
REDIS_PASSWORD: "OxdmdJjdVNXp37MNC2IFoMnTpfGGFv1k"
|
|
RABBITMQ_USER: "bakery"
|
|
RABBITMQ_PASSWORD: "forecast123"
|
|
|
|
# Mount TLS certificates for secure connections
|
|
extraVolumes:
|
|
- name: redis-tls
|
|
secret:
|
|
secretName: redis-tls-secret
|
|
- name: postgres-tls
|
|
secret:
|
|
secretName: postgres-tls
|
|
- name: postgres-tls-fixed
|
|
emptyDir: {}
|
|
- name: varlogpods
|
|
hostPath:
|
|
path: /var/log/pods
|
|
|
|
extraVolumeMounts:
|
|
- name: redis-tls
|
|
mountPath: /etc/redis-tls
|
|
readOnly: true
|
|
- name: postgres-tls
|
|
mountPath: /etc/postgres-tls-source
|
|
readOnly: true
|
|
- name: postgres-tls-fixed
|
|
mountPath: /etc/postgres-tls
|
|
readOnly: false
|
|
- name: varlogpods
|
|
mountPath: /var/log/pods
|
|
readOnly: true
|
|
|
|
# Disable OpAMP - use static configuration only
|
|
# Use 'args' instead of 'extraArgs' to completely override the command
|
|
command:
|
|
name: /signoz-otel-collector
|
|
args:
|
|
- --config=/conf/otel-collector-config.yaml
|
|
- --feature-gates=-pkg.translator.prometheus.NormalizeName
|
|
|
|
# OpenTelemetry Collector configuration
|
|
config:
|
|
# Connectors - bridge between pipelines
|
|
connectors:
|
|
signozmeter:
|
|
dimensions:
|
|
- name: service.name
|
|
- name: deployment.environment
|
|
- name: host.name
|
|
metrics_flush_interval: 1h
|
|
|
|
receivers:
|
|
# OTLP receivers for traces, metrics, and logs from applications
|
|
# All application telemetry is pushed via OTLP protocol
|
|
otlp:
|
|
protocols:
|
|
grpc:
|
|
endpoint: 0.0.0.0:4317
|
|
http:
|
|
endpoint: 0.0.0.0:4318
|
|
cors:
|
|
allowed_origins:
|
|
- "*"
|
|
|
|
# Filelog receiver for Kubernetes pod logs
|
|
# Collects container stdout/stderr from /var/log/pods
|
|
filelog:
|
|
include:
|
|
- /var/log/pods/*/*/*.log
|
|
exclude:
|
|
# Exclude SigNoz's own logs to avoid recursive collection
|
|
- /var/log/pods/bakery-ia_signoz-*/*/*.log
|
|
include_file_path: true
|
|
include_file_name: false
|
|
operators:
|
|
# Parse CRI-O / containerd log format
|
|
- type: regex_parser
|
|
regex: '^(?P<time>[^ ]+) (?P<stream>stdout|stderr) (?P<logtag>[^ ]*) (?P<log>.*)$'
|
|
timestamp:
|
|
parse_from: attributes.time
|
|
layout: '%Y-%m-%dT%H:%M:%S.%LZ'
|
|
# Fix timestamp parsing - extract from the parsed time field
|
|
- type: move
|
|
from: attributes.time
|
|
to: attributes.timestamp
|
|
# Extract Kubernetes metadata from file path
|
|
- type: regex_parser
|
|
id: extract_metadata_from_filepath
|
|
regex: '^.*\/(?P<namespace>[^_]+)_(?P<pod_name>[^_]+)_(?P<uid>[^\/]+)\/(?P<container_name>[^\._]+)\/(?P<restart_count>\d+)\.log$'
|
|
parse_from: attributes["log.file.path"]
|
|
# Move metadata to resource attributes
|
|
- type: move
|
|
from: attributes.namespace
|
|
to: resource["k8s.namespace.name"]
|
|
- type: move
|
|
from: attributes.pod_name
|
|
to: resource["k8s.pod.name"]
|
|
- type: move
|
|
from: attributes.container_name
|
|
to: resource["k8s.container.name"]
|
|
- type: move
|
|
from: attributes.log
|
|
to: body
|
|
|
|
# Kubernetes Cluster Receiver - Collects cluster-level metrics
|
|
# Provides information about nodes, namespaces, pods, and other cluster resources
|
|
k8s_cluster:
|
|
collection_interval: 30s
|
|
node_conditions_to_report:
|
|
- Ready
|
|
- MemoryPressure
|
|
- DiskPressure
|
|
- PIDPressure
|
|
- NetworkUnavailable
|
|
allocatable_types_to_report:
|
|
- cpu
|
|
- memory
|
|
- pods
|
|
|
|
|
|
|
|
# PostgreSQL receivers for database metrics
|
|
# ENABLED: Monitor users configured and credentials stored in secrets
|
|
# Collects metrics directly from PostgreSQL databases with proper TLS
|
|
postgresql/auth:
|
|
endpoint: auth-db-service.bakery-ia:5432
|
|
username: ${env:POSTGRES_MONITOR_USER}
|
|
password: ${env:POSTGRES_MONITOR_PASSWORD}
|
|
databases:
|
|
- auth_db
|
|
collection_interval: 60s
|
|
tls:
|
|
insecure: false
|
|
cert_file: /etc/postgres-tls/server-cert.pem
|
|
key_file: /etc/postgres-tls/server-key.pem
|
|
ca_file: /etc/postgres-tls/ca-cert.pem
|
|
|
|
postgresql/inventory:
|
|
endpoint: inventory-db-service.bakery-ia:5432
|
|
username: ${env:POSTGRES_MONITOR_USER}
|
|
password: ${env:POSTGRES_MONITOR_PASSWORD}
|
|
databases:
|
|
- inventory_db
|
|
collection_interval: 60s
|
|
tls:
|
|
insecure: false
|
|
cert_file: /etc/postgres-tls/server-cert.pem
|
|
key_file: /etc/postgres-tls/server-key.pem
|
|
ca_file: /etc/postgres-tls/ca-cert.pem
|
|
|
|
postgresql/orders:
|
|
endpoint: orders-db-service.bakery-ia:5432
|
|
username: ${env:POSTGRES_MONITOR_USER}
|
|
password: ${env:POSTGRES_MONITOR_PASSWORD}
|
|
databases:
|
|
- orders_db
|
|
collection_interval: 60s
|
|
tls:
|
|
insecure: false
|
|
cert_file: /etc/postgres-tls/server-cert.pem
|
|
key_file: /etc/postgres-tls/server-key.pem
|
|
ca_file: /etc/postgres-tls/ca-cert.pem
|
|
|
|
postgresql/ai-insights:
|
|
endpoint: ai-insights-db-service.bakery-ia:5432
|
|
username: ${env:POSTGRES_MONITOR_USER}
|
|
password: ${env:POSTGRES_MONITOR_PASSWORD}
|
|
databases:
|
|
- ai_insights_db
|
|
collection_interval: 60s
|
|
tls:
|
|
insecure: false
|
|
cert_file: /etc/postgres-tls/server-cert.pem
|
|
key_file: /etc/postgres-tls/server-key.pem
|
|
ca_file: /etc/postgres-tls/ca-cert.pem
|
|
|
|
postgresql/alert-processor:
|
|
endpoint: alert-processor-db-service.bakery-ia:5432
|
|
username: ${env:POSTGRES_MONITOR_USER}
|
|
password: ${env:POSTGRES_MONITOR_PASSWORD}
|
|
databases:
|
|
- alert_processor_db
|
|
collection_interval: 60s
|
|
tls:
|
|
insecure: false
|
|
cert_file: /etc/postgres-tls/server-cert.pem
|
|
key_file: /etc/postgres-tls/server-key.pem
|
|
ca_file: /etc/postgres-tls/ca-cert.pem
|
|
|
|
postgresql/distribution:
|
|
endpoint: distribution-db-service.bakery-ia:5432
|
|
username: ${env:POSTGRES_MONITOR_USER}
|
|
password: ${env:POSTGRES_MONITOR_PASSWORD}
|
|
databases:
|
|
- distribution_db
|
|
collection_interval: 60s
|
|
tls:
|
|
insecure: false
|
|
cert_file: /etc/postgres-tls/server-cert.pem
|
|
key_file: /etc/postgres-tls/server-key.pem
|
|
ca_file: /etc/postgres-tls/ca-cert.pem
|
|
|
|
postgresql/external:
|
|
endpoint: external-db-service.bakery-ia:5432
|
|
username: ${env:POSTGRES_MONITOR_USER}
|
|
password: ${env:POSTGRES_MONITOR_PASSWORD}
|
|
databases:
|
|
- external_db
|
|
collection_interval: 60s
|
|
tls:
|
|
insecure: false
|
|
cert_file: /etc/postgres-tls/server-cert.pem
|
|
key_file: /etc/postgres-tls/server-key.pem
|
|
ca_file: /etc/postgres-tls/ca-cert.pem
|
|
|
|
postgresql/forecasting:
|
|
endpoint: forecasting-db-service.bakery-ia:5432
|
|
username: ${env:POSTGRES_MONITOR_USER}
|
|
password: ${env:POSTGRES_MONITOR_PASSWORD}
|
|
databases:
|
|
- forecasting_db
|
|
collection_interval: 60s
|
|
tls:
|
|
insecure: false
|
|
cert_file: /etc/postgres-tls/server-cert.pem
|
|
key_file: /etc/postgres-tls/server-key.pem
|
|
ca_file: /etc/postgres-tls/ca-cert.pem
|
|
|
|
postgresql/notification:
|
|
endpoint: notification-db-service.bakery-ia:5432
|
|
username: ${env:POSTGRES_MONITOR_USER}
|
|
password: ${env:POSTGRES_MONITOR_PASSWORD}
|
|
databases:
|
|
- notification_db
|
|
collection_interval: 60s
|
|
tls:
|
|
insecure: false
|
|
cert_file: /etc/postgres-tls/server-cert.pem
|
|
key_file: /etc/postgres-tls/server-key.pem
|
|
ca_file: /etc/postgres-tls/ca-cert.pem
|
|
|
|
postgresql/orchestrator:
|
|
endpoint: orchestrator-db-service.bakery-ia:5432
|
|
username: ${env:POSTGRES_MONITOR_USER}
|
|
password: ${env:POSTGRES_MONITOR_PASSWORD}
|
|
databases:
|
|
- orchestrator_db
|
|
collection_interval: 60s
|
|
tls:
|
|
insecure: false
|
|
cert_file: /etc/postgres-tls/server-cert.pem
|
|
key_file: /etc/postgres-tls/server-key.pem
|
|
ca_file: /etc/postgres-tls/ca-cert.pem
|
|
|
|
postgresql/pos:
|
|
endpoint: pos-db-service.bakery-ia:5432
|
|
username: ${env:POSTGRES_MONITOR_USER}
|
|
password: ${env:POSTGRES_MONITOR_PASSWORD}
|
|
databases:
|
|
- pos_db
|
|
collection_interval: 60s
|
|
tls:
|
|
insecure: false
|
|
cert_file: /etc/postgres-tls/server-cert.pem
|
|
key_file: /etc/postgres-tls/server-key.pem
|
|
ca_file: /etc/postgres-tls/ca-cert.pem
|
|
|
|
postgresql/procurement:
|
|
endpoint: procurement-db-service.bakery-ia:5432
|
|
username: ${env:POSTGRES_MONITOR_USER}
|
|
password: ${env:POSTGRES_MONITOR_PASSWORD}
|
|
databases:
|
|
- procurement_db
|
|
collection_interval: 60s
|
|
tls:
|
|
insecure: false
|
|
cert_file: /etc/postgres-tls/server-cert.pem
|
|
key_file: /etc/postgres-tls/server-key.pem
|
|
ca_file: /etc/postgres-tls/ca-cert.pem
|
|
|
|
postgresql/production:
|
|
endpoint: production-db-service.bakery-ia:5432
|
|
username: ${env:POSTGRES_MONITOR_USER}
|
|
password: ${env:POSTGRES_MONITOR_PASSWORD}
|
|
databases:
|
|
- production_db
|
|
collection_interval: 60s
|
|
tls:
|
|
insecure: false
|
|
cert_file: /etc/postgres-tls/server-cert.pem
|
|
key_file: /etc/postgres-tls/server-key.pem
|
|
ca_file: /etc/postgres-tls/ca-cert.pem
|
|
|
|
postgresql/recipes:
|
|
endpoint: recipes-db-service.bakery-ia:5432
|
|
username: ${env:POSTGRES_MONITOR_USER}
|
|
password: ${env:POSTGRES_MONITOR_PASSWORD}
|
|
databases:
|
|
- recipes_db
|
|
collection_interval: 60s
|
|
tls:
|
|
insecure: false
|
|
cert_file: /etc/postgres-tls/server-cert.pem
|
|
key_file: /etc/postgres-tls/server-key.pem
|
|
ca_file: /etc/postgres-tls/ca-cert.pem
|
|
|
|
postgresql/sales:
|
|
endpoint: sales-db-service.bakery-ia:5432
|
|
username: ${env:POSTGRES_MONITOR_USER}
|
|
password: ${env:POSTGRES_MONITOR_PASSWORD}
|
|
databases:
|
|
- sales_db
|
|
collection_interval: 60s
|
|
tls:
|
|
insecure: false
|
|
cert_file: /etc/postgres-tls/server-cert.pem
|
|
key_file: /etc/postgres-tls/server-key.pem
|
|
ca_file: /etc/postgres-tls/ca-cert.pem
|
|
|
|
postgresql/suppliers:
|
|
endpoint: suppliers-db-service.bakery-ia:5432
|
|
username: ${env:POSTGRES_MONITOR_USER}
|
|
password: ${env:POSTGRES_MONITOR_PASSWORD}
|
|
databases:
|
|
- suppliers_db
|
|
collection_interval: 60s
|
|
tls:
|
|
insecure: false
|
|
cert_file: /etc/postgres-tls/server-cert.pem
|
|
key_file: /etc/postgres-tls/server-key.pem
|
|
ca_file: /etc/postgres-tls/ca-cert.pem
|
|
|
|
postgresql/tenant:
|
|
endpoint: tenant-db-service.bakery-ia:5432
|
|
username: ${env:POSTGRES_MONITOR_USER}
|
|
password: ${env:POSTGRES_MONITOR_PASSWORD}
|
|
databases:
|
|
- tenant_db
|
|
collection_interval: 60s
|
|
tls:
|
|
insecure: false
|
|
cert_file: /etc/postgres-tls/server-cert.pem
|
|
key_file: /etc/postgres-tls/server-key.pem
|
|
ca_file: /etc/postgres-tls/ca-cert.pem
|
|
|
|
postgresql/training:
|
|
endpoint: training-db-service.bakery-ia:5432
|
|
username: ${env:POSTGRES_MONITOR_USER}
|
|
password: ${env:POSTGRES_MONITOR_PASSWORD}
|
|
databases:
|
|
- training_db
|
|
collection_interval: 60s
|
|
tls:
|
|
insecure: false
|
|
cert_file: /etc/postgres-tls/server-cert.pem
|
|
key_file: /etc/postgres-tls/server-key.pem
|
|
ca_file: /etc/postgres-tls/ca-cert.pem
|
|
|
|
# Redis receiver for cache metrics
|
|
# ENABLED: Using existing credentials from redis-secrets with TLS
|
|
redis:
|
|
endpoint: redis-service.bakery-ia:6379
|
|
password: ${env:REDIS_PASSWORD}
|
|
collection_interval: 60s
|
|
transport: tcp
|
|
tls:
|
|
insecure_skip_verify: false
|
|
cert_file: /etc/redis-tls/redis-cert.pem
|
|
key_file: /etc/redis-tls/redis-key.pem
|
|
ca_file: /etc/redis-tls/ca-cert.pem
|
|
metrics:
|
|
redis.maxmemory:
|
|
enabled: true
|
|
redis.cmd.latency:
|
|
enabled: true
|
|
|
|
# RabbitMQ receiver via management API
|
|
# ENABLED: Using existing credentials from rabbitmq-secrets
|
|
rabbitmq:
|
|
endpoint: http://rabbitmq-service.bakery-ia:15672
|
|
username: ${env:RABBITMQ_USER}
|
|
password: ${env:RABBITMQ_PASSWORD}
|
|
collection_interval: 30s
|
|
|
|
# Prometheus Receiver - Scrapes metrics from Kubernetes API
|
|
# Simplified configuration using only Kubernetes API metrics
|
|
prometheus:
|
|
config:
|
|
scrape_configs:
|
|
- job_name: 'kubernetes-nodes-cadvisor'
|
|
scrape_interval: 30s
|
|
scrape_timeout: 10s
|
|
scheme: https
|
|
tls_config:
|
|
insecure_skip_verify: true
|
|
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
|
|
kubernetes_sd_configs:
|
|
- role: node
|
|
relabel_configs:
|
|
- action: labelmap
|
|
regex: __meta_kubernetes_node_label_(.+)
|
|
- target_label: __address__
|
|
replacement: kubernetes.default.svc:443
|
|
- source_labels: [__meta_kubernetes_node_name]
|
|
regex: (.+)
|
|
target_label: __metrics_path__
|
|
replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
|
|
- job_name: 'kubernetes-apiserver'
|
|
scrape_interval: 30s
|
|
scrape_timeout: 10s
|
|
scheme: https
|
|
tls_config:
|
|
insecure_skip_verify: true
|
|
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
|
|
kubernetes_sd_configs:
|
|
- role: endpoints
|
|
relabel_configs:
|
|
- source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
|
|
action: keep
|
|
regex: default;kubernetes;https
|
|
|
|
processors:
|
|
# Batch processor for better performance (optimized for high throughput)
|
|
batch:
|
|
timeout: 1s
|
|
send_batch_size: 10000 # Increased from 1024 for better performance
|
|
send_batch_max_size: 10000
|
|
|
|
# Batch processor for meter data
|
|
batch/meter:
|
|
timeout: 1s
|
|
send_batch_size: 20000
|
|
send_batch_max_size: 25000
|
|
|
|
# Memory limiter to prevent OOM
|
|
memory_limiter:
|
|
check_interval: 1s
|
|
limit_mib: 400
|
|
spike_limit_mib: 100
|
|
|
|
# Resource detection
|
|
resourcedetection:
|
|
detectors: [env, system, docker]
|
|
timeout: 5s
|
|
|
|
# Kubernetes attributes processor - CRITICAL for logs
|
|
# Extracts pod, namespace, container metadata from log attributes
|
|
k8sattributes:
|
|
auth_type: "serviceAccount"
|
|
passthrough: false
|
|
extract:
|
|
metadata:
|
|
- k8s.pod.name
|
|
- k8s.pod.uid
|
|
- k8s.deployment.name
|
|
- k8s.namespace.name
|
|
- k8s.node.name
|
|
- k8s.container.name
|
|
labels:
|
|
- tag_name: "app"
|
|
- tag_name: "pod-template-hash"
|
|
annotations:
|
|
- tag_name: "description"
|
|
|
|
# SigNoz span metrics processor with delta aggregation (recommended)
|
|
# Generates RED metrics (Rate, Error, Duration) from trace spans
|
|
signozspanmetrics/delta:
|
|
aggregation_temporality: AGGREGATION_TEMPORALITY_DELTA
|
|
metrics_exporter: signozclickhousemetrics
|
|
latency_histogram_buckets: [100us, 1ms, 2ms, 6ms, 10ms, 50ms, 100ms, 250ms, 500ms, 1000ms, 1400ms, 2000ms, 5s, 10s, 20s, 40s, 60s]
|
|
dimensions_cache_size: 100000
|
|
dimensions:
|
|
- name: service.namespace
|
|
default: default
|
|
- name: deployment.environment
|
|
default: default
|
|
- name: signoz.collector.id
|
|
|
|
exporters:
|
|
# ClickHouse exporter for traces
|
|
clickhousetraces:
|
|
datasource: tcp://admin:27ff0399-0d3a-4bd8-919d-17c2181e6fb9@signoz-clickhouse:9000/?database=signoz_traces
|
|
timeout: 10s
|
|
retry_on_failure:
|
|
enabled: true
|
|
initial_interval: 5s
|
|
max_interval: 30s
|
|
max_elapsed_time: 300s
|
|
|
|
# ClickHouse exporter for metrics
|
|
signozclickhousemetrics:
|
|
dsn: "tcp://admin:27ff0399-0d3a-4bd8-919d-17c2181e6fb9@signoz-clickhouse:9000/signoz_metrics"
|
|
timeout: 10s
|
|
retry_on_failure:
|
|
enabled: true
|
|
initial_interval: 5s
|
|
max_interval: 30s
|
|
max_elapsed_time: 300s
|
|
|
|
# ClickHouse exporter for meter data (usage metrics)
|
|
signozclickhousemeter:
|
|
dsn: "tcp://admin:27ff0399-0d3a-4bd8-919d-17c2181e6fb9@signoz-clickhouse:9000/signoz_meter"
|
|
timeout: 45s
|
|
sending_queue:
|
|
enabled: false
|
|
|
|
# ClickHouse exporter for logs
|
|
clickhouselogsexporter:
|
|
dsn: tcp://admin:27ff0399-0d3a-4bd8-919d-17c2181e6fb9@signoz-clickhouse:9000/?database=signoz_logs
|
|
timeout: 10s
|
|
retry_on_failure:
|
|
enabled: true
|
|
initial_interval: 5s
|
|
max_interval: 30s
|
|
|
|
# Metadata exporter for service metadata
|
|
metadataexporter:
|
|
dsn: "tcp://admin:27ff0399-0d3a-4bd8-919d-17c2181e6fb9@signoz-clickhouse:9000/signoz_metadata"
|
|
timeout: 10s
|
|
cache:
|
|
provider: in_memory
|
|
|
|
# Debug exporter for debugging (optional)
|
|
debug:
|
|
verbosity: detailed
|
|
sampling_initial: 5
|
|
sampling_thereafter: 200
|
|
|
|
service:
|
|
pipelines:
|
|
# Traces pipeline - exports to ClickHouse and signozmeter connector
|
|
traces:
|
|
receivers: [otlp]
|
|
processors: [memory_limiter, batch, signozspanmetrics/delta, resourcedetection]
|
|
exporters: [clickhousetraces, metadataexporter, signozmeter]
|
|
|
|
# Metrics pipeline
|
|
metrics:
|
|
receivers: [otlp,
|
|
postgresql/auth, postgresql/inventory, postgresql/orders,
|
|
postgresql/ai-insights, postgresql/alert-processor, postgresql/distribution,
|
|
postgresql/external, postgresql/forecasting, postgresql/notification,
|
|
postgresql/orchestrator, postgresql/pos, postgresql/procurement,
|
|
postgresql/production, postgresql/recipes, postgresql/sales,
|
|
postgresql/suppliers, postgresql/tenant, postgresql/training,
|
|
redis, rabbitmq, k8s_cluster, prometheus]
|
|
processors: [memory_limiter, batch, resourcedetection]
|
|
exporters: [signozclickhousemetrics]
|
|
|
|
# Meter pipeline - receives from signozmeter connector
|
|
metrics/meter:
|
|
receivers: [signozmeter]
|
|
processors: [batch/meter]
|
|
exporters: [signozclickhousemeter]
|
|
|
|
# Logs pipeline - includes both OTLP and Kubernetes pod logs
|
|
logs:
|
|
receivers: [otlp, filelog]
|
|
processors: [memory_limiter, batch, resourcedetection, k8sattributes]
|
|
exporters: [clickhouselogsexporter]
|
|
|
|
# ClusterRole configuration for Kubernetes monitoring
|
|
# CRITICAL: Required for k8s_cluster receiver to access Kubernetes API
|
|
# Without these permissions, k8s metrics will not appear in SigNoz UI
|
|
clusterRole:
|
|
create: true
|
|
name: "signoz-otel-collector-bakery-ia"
|
|
annotations: {}
|
|
# Complete RBAC rules required by k8sclusterreceiver
|
|
# Based on OpenTelemetry and SigNoz official documentation
|
|
rules:
|
|
# Core API group - fundamental Kubernetes resources
|
|
- apiGroups: [""]
|
|
resources:
|
|
- "events"
|
|
- "namespaces"
|
|
- "nodes"
|
|
- "nodes/proxy"
|
|
- "nodes/metrics"
|
|
- "nodes/spec"
|
|
- "pods"
|
|
- "pods/status"
|
|
- "replicationcontrollers"
|
|
- "replicationcontrollers/status"
|
|
- "resourcequotas"
|
|
- "services"
|
|
- "endpoints"
|
|
verbs: ["get", "list", "watch"]
|
|
# Apps API group - modern workload controllers
|
|
- apiGroups: ["apps"]
|
|
resources: ["deployments", "daemonsets", "statefulsets", "replicasets"]
|
|
verbs: ["get", "list", "watch"]
|
|
# Batch API group - job management
|
|
- apiGroups: ["batch"]
|
|
resources: ["jobs", "cronjobs"]
|
|
verbs: ["get", "list", "watch"]
|
|
# Autoscaling API group - HPA metrics (CRITICAL)
|
|
- apiGroups: ["autoscaling"]
|
|
resources: ["horizontalpodautoscalers"]
|
|
verbs: ["get", "list", "watch"]
|
|
# Extensions API group - legacy support
|
|
- apiGroups: ["extensions"]
|
|
resources: ["deployments", "daemonsets", "replicasets"]
|
|
verbs: ["get", "list", "watch"]
|
|
# Metrics API group - resource metrics
|
|
- apiGroups: ["metrics.k8s.io"]
|
|
resources: ["nodes", "pods"]
|
|
verbs: ["get", "list", "watch"]
|
|
clusterRoleBinding:
|
|
annotations: {}
|
|
name: "signoz-otel-collector-bakery-ia"
|
|
|
|
# Additional Configuration
|
|
serviceAccount:
|
|
create: true
|
|
annotations: {}
|
|
name: "signoz-otel-collector"
|
|
|
|
# Security Context
|
|
securityContext:
|
|
runAsNonRoot: true
|
|
runAsUser: 1000
|
|
fsGroup: 1000
|
|
|
|
# Network Policies (disabled for dev)
|
|
networkPolicy:
|
|
enabled: false
|
|
|
|
# Monitoring SigNoz itself
|
|
selfMonitoring:
|
|
enabled: true
|
|
serviceMonitor:
|
|
enabled: false
|