Add new infra architecture 5
This commit is contained in:
Urtzi Alfaro
@@ -10,844 +10,3 @@ global:
|
||||
clusterName: "bakery-ia-dev"
|
||||
domain: "monitoring.bakery-ia.local"
|
||||
# Docker Hub credentials - applied to all sub-charts (including Zookeeper, ClickHouse, etc)
|
||||
imagePullSecrets:
|
||||
- dockerhub-creds
|
||||
|
||||
# Docker Hub credentials for pulling images (root level for SigNoz components)
|
||||
imagePullSecrets:
|
||||
- dockerhub-creds
|
||||
|
||||
# SignOz Main Component (includes frontend and query service)
|
||||
signoz:
|
||||
replicaCount: 1
|
||||
|
||||
service:
|
||||
type: ClusterIP
|
||||
port: 8080
|
||||
|
||||
# DISABLE built-in ingress - using unified bakery-ingress instead
|
||||
# Route configured in infrastructure/kubernetes/overlays/dev/dev-ingress.yaml
|
||||
ingress:
|
||||
enabled: false
|
||||
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m # Combined frontend + query service
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: 1000m
|
||||
memory: 1Gi
|
||||
|
||||
# Environment variables (new format - replaces configVars)
|
||||
env:
|
||||
signoz_telemetrystore_provider: "clickhouse"
|
||||
dot_metrics_enabled: "true"
|
||||
signoz_emailing_enabled: "false"
|
||||
signoz_alertmanager_provider: "signoz"
|
||||
# Retention for dev (7 days)
|
||||
signoz_traces_ttl_duration_hrs: "168"
|
||||
signoz_metrics_ttl_duration_hrs: "168"
|
||||
signoz_logs_ttl_duration_hrs: "168"
|
||||
# OpAMP Server Configuration - DISABLED for dev (causes gRPC instability)
|
||||
signoz_opamp_server_enabled: "false"
|
||||
# signoz_opamp_server_endpoint: "0.0.0.0:4320"
|
||||
|
||||
persistence:
|
||||
enabled: true
|
||||
size: 5Gi
|
||||
storageClass: "standard"
|
||||
|
||||
# AlertManager Configuration
|
||||
alertmanager:
|
||||
replicaCount: 1
|
||||
image:
|
||||
repository: signoz/alertmanager
|
||||
tag: 0.23.5
|
||||
pullPolicy: IfNotPresent
|
||||
|
||||
service:
|
||||
type: ClusterIP
|
||||
port: 9093
|
||||
|
||||
resources:
|
||||
requests:
|
||||
cpu: 25m # Reduced for local dev
|
||||
memory: 64Mi # Reduced for local dev
|
||||
limits:
|
||||
cpu: 200m
|
||||
memory: 256Mi
|
||||
|
||||
persistence:
|
||||
enabled: true
|
||||
size: 2Gi
|
||||
storageClass: "standard"
|
||||
|
||||
config:
|
||||
global:
|
||||
resolve_timeout: 5m
|
||||
route:
|
||||
group_by: ['alertname', 'cluster', 'service']
|
||||
group_wait: 10s
|
||||
group_interval: 10s
|
||||
repeat_interval: 12h
|
||||
receiver: 'default'
|
||||
receivers:
|
||||
- name: 'default'
|
||||
# Add email, slack, webhook configs here
|
||||
|
||||
# ClickHouse Configuration - Time Series Database
|
||||
# Minimal resources for local development on constrained Kind cluster
|
||||
clickhouse:
|
||||
enabled: true
|
||||
installCustomStorageClass: false
|
||||
|
||||
image:
|
||||
registry: docker.io
|
||||
repository: clickhouse/clickhouse-server
|
||||
tag: 25.5.6 # Official recommended version
|
||||
|
||||
# Reduce ClickHouse resource requests for local dev
|
||||
clickhouse:
|
||||
resources:
|
||||
requests:
|
||||
cpu: 200m # Reduced from default 500m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: 1000m
|
||||
memory: 1Gi
|
||||
|
||||
persistence:
|
||||
enabled: true
|
||||
size: 20Gi
|
||||
|
||||
# Zookeeper Configuration (required by ClickHouse)
|
||||
zookeeper:
|
||||
enabled: true
|
||||
replicaCount: 1 # Single replica for dev
|
||||
|
||||
image:
|
||||
tag: 3.7.1 # Official recommended version
|
||||
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 512Mi
|
||||
|
||||
persistence:
|
||||
enabled: true
|
||||
size: 5Gi
|
||||
|
||||
# OpenTelemetry Collector - Data ingestion endpoint for all telemetry
|
||||
otelCollector:
|
||||
enabled: true
|
||||
replicaCount: 1
|
||||
|
||||
image:
|
||||
repository: signoz/signoz-otel-collector
|
||||
tag: v0.129.12 # Latest recommended version
|
||||
|
||||
# OpAMP Configuration - DISABLED for development
|
||||
# OpAMP is designed for production with remote config management
|
||||
# In dev, it causes gRPC instability and collector reloads
|
||||
# We use static configuration instead
|
||||
|
||||
# Init containers for the Otel Collector pod
|
||||
initContainers:
|
||||
fix-postgres-tls:
|
||||
enabled: true
|
||||
image:
|
||||
registry: docker.io
|
||||
repository: busybox
|
||||
tag: 1.35
|
||||
pullPolicy: IfNotPresent
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
echo "Fixing PostgreSQL TLS file permissions..."
|
||||
cp /etc/postgres-tls-source/* /etc/postgres-tls/
|
||||
chmod 600 /etc/postgres-tls/server-key.pem
|
||||
chmod 644 /etc/postgres-tls/server-cert.pem
|
||||
chmod 644 /etc/postgres-tls/ca-cert.pem
|
||||
echo "PostgreSQL TLS permissions fixed"
|
||||
volumeMounts:
|
||||
- name: postgres-tls-source
|
||||
mountPath: /etc/postgres-tls-source
|
||||
readOnly: true
|
||||
- name: postgres-tls-fixed
|
||||
mountPath: /etc/postgres-tls
|
||||
readOnly: false
|
||||
|
||||
# Service configuration - expose both gRPC and HTTP endpoints
|
||||
service:
|
||||
type: ClusterIP
|
||||
ports:
|
||||
# gRPC receivers
|
||||
- name: otlp-grpc
|
||||
port: 4317
|
||||
targetPort: 4317
|
||||
protocol: TCP
|
||||
# HTTP receivers
|
||||
- name: otlp-http
|
||||
port: 4318
|
||||
targetPort: 4318
|
||||
protocol: TCP
|
||||
# Prometheus remote write
|
||||
- name: prometheus
|
||||
port: 8889
|
||||
targetPort: 8889
|
||||
protocol: TCP
|
||||
# Metrics
|
||||
- name: metrics
|
||||
port: 8888
|
||||
targetPort: 8888
|
||||
protocol: TCP
|
||||
|
||||
resources:
|
||||
requests:
|
||||
cpu: 50m # Reduced from 100m
|
||||
memory: 128Mi # Reduced from 256Mi
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 512Mi
|
||||
|
||||
# Additional environment variables for receivers
|
||||
additionalEnvs:
|
||||
POSTGRES_MONITOR_USER: "monitoring"
|
||||
POSTGRES_MONITOR_PASSWORD: "monitoring_369f9c001f242b07ef9e2826e17169ca"
|
||||
REDIS_PASSWORD: "OxdmdJjdVNXp37MNC2IFoMnTpfGGFv1k"
|
||||
RABBITMQ_USER: "bakery"
|
||||
RABBITMQ_PASSWORD: "forecast123"
|
||||
|
||||
# Mount TLS certificates for secure connections
|
||||
extraVolumes:
|
||||
- name: redis-tls
|
||||
secret:
|
||||
secretName: redis-tls-secret
|
||||
- name: postgres-tls
|
||||
secret:
|
||||
secretName: postgres-tls
|
||||
- name: postgres-tls-fixed
|
||||
emptyDir: {}
|
||||
- name: varlogpods
|
||||
hostPath:
|
||||
path: /var/log/pods
|
||||
|
||||
extraVolumeMounts:
|
||||
- name: redis-tls
|
||||
mountPath: /etc/redis-tls
|
||||
readOnly: true
|
||||
- name: postgres-tls
|
||||
mountPath: /etc/postgres-tls-source
|
||||
readOnly: true
|
||||
- name: postgres-tls-fixed
|
||||
mountPath: /etc/postgres-tls
|
||||
readOnly: false
|
||||
- name: varlogpods
|
||||
mountPath: /var/log/pods
|
||||
readOnly: true
|
||||
|
||||
# Disable OpAMP - use static configuration only
|
||||
# Use 'args' instead of 'extraArgs' to completely override the command
|
||||
command:
|
||||
name: /signoz-otel-collector
|
||||
args:
|
||||
- --config=/conf/otel-collector-config.yaml
|
||||
- --feature-gates=-pkg.translator.prometheus.NormalizeName
|
||||
|
||||
# OpenTelemetry Collector configuration
|
||||
config:
|
||||
# Connectors - bridge between pipelines
|
||||
connectors:
|
||||
signozmeter:
|
||||
dimensions:
|
||||
- name: service.name
|
||||
- name: deployment.environment
|
||||
- name: host.name
|
||||
metrics_flush_interval: 1h
|
||||
|
||||
receivers:
|
||||
# OTLP receivers for traces, metrics, and logs from applications
|
||||
# All application telemetry is pushed via OTLP protocol
|
||||
otlp:
|
||||
protocols:
|
||||
grpc:
|
||||
endpoint: 0.0.0.0:4317
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
cors:
|
||||
allowed_origins:
|
||||
- "*"
|
||||
|
||||
# Filelog receiver for Kubernetes pod logs
|
||||
# Collects container stdout/stderr from /var/log/pods
|
||||
filelog:
|
||||
include:
|
||||
- /var/log/pods/*/*/*.log
|
||||
exclude:
|
||||
# Exclude SigNoz's own logs to avoid recursive collection
|
||||
- /var/log/pods/bakery-ia_signoz-*/*/*.log
|
||||
include_file_path: true
|
||||
include_file_name: false
|
||||
operators:
|
||||
# Parse CRI-O / containerd log format
|
||||
- type: regex_parser
|
||||
regex: '^(?P<time>[^ ]+) (?P<stream>stdout|stderr) (?P<logtag>[^ ]*) (?P<log>.*)$'
|
||||
timestamp:
|
||||
parse_from: attributes.time
|
||||
layout: '%Y-%m-%dT%H:%M:%S.%LZ'
|
||||
# Fix timestamp parsing - extract from the parsed time field
|
||||
- type: move
|
||||
from: attributes.time
|
||||
to: attributes.timestamp
|
||||
# Extract Kubernetes metadata from file path
|
||||
- type: regex_parser
|
||||
id: extract_metadata_from_filepath
|
||||
regex: '^.*\/(?P<namespace>[^_]+)_(?P<pod_name>[^_]+)_(?P<uid>[^\/]+)\/(?P<container_name>[^\._]+)\/(?P<restart_count>\d+)\.log$'
|
||||
parse_from: attributes["log.file.path"]
|
||||
# Move metadata to resource attributes
|
||||
- type: move
|
||||
from: attributes.namespace
|
||||
to: resource["k8s.namespace.name"]
|
||||
- type: move
|
||||
from: attributes.pod_name
|
||||
to: resource["k8s.pod.name"]
|
||||
- type: move
|
||||
from: attributes.container_name
|
||||
to: resource["k8s.container.name"]
|
||||
- type: move
|
||||
from: attributes.log
|
||||
to: body
|
||||
|
||||
# Kubernetes Cluster Receiver - Collects cluster-level metrics
|
||||
# Provides information about nodes, namespaces, pods, and other cluster resources
|
||||
k8s_cluster:
|
||||
collection_interval: 30s
|
||||
node_conditions_to_report:
|
||||
- Ready
|
||||
- MemoryPressure
|
||||
- DiskPressure
|
||||
- PIDPressure
|
||||
- NetworkUnavailable
|
||||
allocatable_types_to_report:
|
||||
- cpu
|
||||
- memory
|
||||
- pods
|
||||
|
||||
|
||||
|
||||
# PostgreSQL receivers for database metrics
|
||||
# ENABLED: Monitor users configured and credentials stored in secrets
|
||||
# Collects metrics directly from PostgreSQL databases with proper TLS
|
||||
postgresql/auth:
|
||||
endpoint: auth-db-service.bakery-ia:5432
|
||||
username: ${env:POSTGRES_MONITOR_USER}
|
||||
password: ${env:POSTGRES_MONITOR_PASSWORD}
|
||||
databases:
|
||||
- auth_db
|
||||
collection_interval: 60s
|
||||
tls:
|
||||
insecure: false
|
||||
cert_file: /etc/postgres-tls/server-cert.pem
|
||||
key_file: /etc/postgres-tls/server-key.pem
|
||||
ca_file: /etc/postgres-tls/ca-cert.pem
|
||||
|
||||
postgresql/inventory:
|
||||
endpoint: inventory-db-service.bakery-ia:5432
|
||||
username: ${env:POSTGRES_MONITOR_USER}
|
||||
password: ${env:POSTGRES_MONITOR_PASSWORD}
|
||||
databases:
|
||||
- inventory_db
|
||||
collection_interval: 60s
|
||||
tls:
|
||||
insecure: false
|
||||
cert_file: /etc/postgres-tls/server-cert.pem
|
||||
key_file: /etc/postgres-tls/server-key.pem
|
||||
ca_file: /etc/postgres-tls/ca-cert.pem
|
||||
|
||||
postgresql/orders:
|
||||
endpoint: orders-db-service.bakery-ia:5432
|
||||
username: ${env:POSTGRES_MONITOR_USER}
|
||||
password: ${env:POSTGRES_MONITOR_PASSWORD}
|
||||
databases:
|
||||
- orders_db
|
||||
collection_interval: 60s
|
||||
tls:
|
||||
insecure: false
|
||||
cert_file: /etc/postgres-tls/server-cert.pem
|
||||
key_file: /etc/postgres-tls/server-key.pem
|
||||
ca_file: /etc/postgres-tls/ca-cert.pem
|
||||
|
||||
postgresql/ai-insights:
|
||||
endpoint: ai-insights-db-service.bakery-ia:5432
|
||||
username: ${env:POSTGRES_MONITOR_USER}
|
||||
password: ${env:POSTGRES_MONITOR_PASSWORD}
|
||||
databases:
|
||||
- ai_insights_db
|
||||
collection_interval: 60s
|
||||
tls:
|
||||
insecure: false
|
||||
cert_file: /etc/postgres-tls/server-cert.pem
|
||||
key_file: /etc/postgres-tls/server-key.pem
|
||||
ca_file: /etc/postgres-tls/ca-cert.pem
|
||||
|
||||
postgresql/alert-processor:
|
||||
endpoint: alert-processor-db-service.bakery-ia:5432
|
||||
username: ${env:POSTGRES_MONITOR_USER}
|
||||
password: ${env:POSTGRES_MONITOR_PASSWORD}
|
||||
databases:
|
||||
- alert_processor_db
|
||||
collection_interval: 60s
|
||||
tls:
|
||||
insecure: false
|
||||
cert_file: /etc/postgres-tls/server-cert.pem
|
||||
key_file: /etc/postgres-tls/server-key.pem
|
||||
ca_file: /etc/postgres-tls/ca-cert.pem
|
||||
|
||||
postgresql/distribution:
|
||||
endpoint: distribution-db-service.bakery-ia:5432
|
||||
username: ${env:POSTGRES_MONITOR_USER}
|
||||
password: ${env:POSTGRES_MONITOR_PASSWORD}
|
||||
databases:
|
||||
- distribution_db
|
||||
collection_interval: 60s
|
||||
tls:
|
||||
insecure: false
|
||||
cert_file: /etc/postgres-tls/server-cert.pem
|
||||
key_file: /etc/postgres-tls/server-key.pem
|
||||
ca_file: /etc/postgres-tls/ca-cert.pem
|
||||
|
||||
postgresql/external:
|
||||
endpoint: external-db-service.bakery-ia:5432
|
||||
username: ${env:POSTGRES_MONITOR_USER}
|
||||
password: ${env:POSTGRES_MONITOR_PASSWORD}
|
||||
databases:
|
||||
- external_db
|
||||
collection_interval: 60s
|
||||
tls:
|
||||
insecure: false
|
||||
cert_file: /etc/postgres-tls/server-cert.pem
|
||||
key_file: /etc/postgres-tls/server-key.pem
|
||||
ca_file: /etc/postgres-tls/ca-cert.pem
|
||||
|
||||
postgresql/forecasting:
|
||||
endpoint: forecasting-db-service.bakery-ia:5432
|
||||
username: ${env:POSTGRES_MONITOR_USER}
|
||||
password: ${env:POSTGRES_MONITOR_PASSWORD}
|
||||
databases:
|
||||
- forecasting_db
|
||||
collection_interval: 60s
|
||||
tls:
|
||||
insecure: false
|
||||
cert_file: /etc/postgres-tls/server-cert.pem
|
||||
key_file: /etc/postgres-tls/server-key.pem
|
||||
ca_file: /etc/postgres-tls/ca-cert.pem
|
||||
|
||||
postgresql/notification:
|
||||
endpoint: notification-db-service.bakery-ia:5432
|
||||
username: ${env:POSTGRES_MONITOR_USER}
|
||||
password: ${env:POSTGRES_MONITOR_PASSWORD}
|
||||
databases:
|
||||
- notification_db
|
||||
collection_interval: 60s
|
||||
tls:
|
||||
insecure: false
|
||||
cert_file: /etc/postgres-tls/server-cert.pem
|
||||
key_file: /etc/postgres-tls/server-key.pem
|
||||
ca_file: /etc/postgres-tls/ca-cert.pem
|
||||
|
||||
postgresql/orchestrator:
|
||||
endpoint: orchestrator-db-service.bakery-ia:5432
|
||||
username: ${env:POSTGRES_MONITOR_USER}
|
||||
password: ${env:POSTGRES_MONITOR_PASSWORD}
|
||||
databases:
|
||||
- orchestrator_db
|
||||
collection_interval: 60s
|
||||
tls:
|
||||
insecure: false
|
||||
cert_file: /etc/postgres-tls/server-cert.pem
|
||||
key_file: /etc/postgres-tls/server-key.pem
|
||||
ca_file: /etc/postgres-tls/ca-cert.pem
|
||||
|
||||
postgresql/pos:
|
||||
endpoint: pos-db-service.bakery-ia:5432
|
||||
username: ${env:POSTGRES_MONITOR_USER}
|
||||
password: ${env:POSTGRES_MONITOR_PASSWORD}
|
||||
databases:
|
||||
- pos_db
|
||||
collection_interval: 60s
|
||||
tls:
|
||||
insecure: false
|
||||
cert_file: /etc/postgres-tls/server-cert.pem
|
||||
key_file: /etc/postgres-tls/server-key.pem
|
||||
ca_file: /etc/postgres-tls/ca-cert.pem
|
||||
|
||||
postgresql/procurement:
|
||||
endpoint: procurement-db-service.bakery-ia:5432
|
||||
username: ${env:POSTGRES_MONITOR_USER}
|
||||
password: ${env:POSTGRES_MONITOR_PASSWORD}
|
||||
databases:
|
||||
- procurement_db
|
||||
collection_interval: 60s
|
||||
tls:
|
||||
insecure: false
|
||||
cert_file: /etc/postgres-tls/server-cert.pem
|
||||
key_file: /etc/postgres-tls/server-key.pem
|
||||
ca_file: /etc/postgres-tls/ca-cert.pem
|
||||
|
||||
postgresql/production:
|
||||
endpoint: production-db-service.bakery-ia:5432
|
||||
username: ${env:POSTGRES_MONITOR_USER}
|
||||
password: ${env:POSTGRES_MONITOR_PASSWORD}
|
||||
databases:
|
||||
- production_db
|
||||
collection_interval: 60s
|
||||
tls:
|
||||
insecure: false
|
||||
cert_file: /etc/postgres-tls/server-cert.pem
|
||||
key_file: /etc/postgres-tls/server-key.pem
|
||||
ca_file: /etc/postgres-tls/ca-cert.pem
|
||||
|
||||
postgresql/recipes:
|
||||
endpoint: recipes-db-service.bakery-ia:5432
|
||||
username: ${env:POSTGRES_MONITOR_USER}
|
||||
password: ${env:POSTGRES_MONITOR_PASSWORD}
|
||||
databases:
|
||||
- recipes_db
|
||||
collection_interval: 60s
|
||||
tls:
|
||||
insecure: false
|
||||
cert_file: /etc/postgres-tls/server-cert.pem
|
||||
key_file: /etc/postgres-tls/server-key.pem
|
||||
ca_file: /etc/postgres-tls/ca-cert.pem
|
||||
|
||||
postgresql/sales:
|
||||
endpoint: sales-db-service.bakery-ia:5432
|
||||
username: ${env:POSTGRES_MONITOR_USER}
|
||||
password: ${env:POSTGRES_MONITOR_PASSWORD}
|
||||
databases:
|
||||
- sales_db
|
||||
collection_interval: 60s
|
||||
tls:
|
||||
insecure: false
|
||||
cert_file: /etc/postgres-tls/server-cert.pem
|
||||
key_file: /etc/postgres-tls/server-key.pem
|
||||
ca_file: /etc/postgres-tls/ca-cert.pem
|
||||
|
||||
postgresql/suppliers:
|
||||
endpoint: suppliers-db-service.bakery-ia:5432
|
||||
username: ${env:POSTGRES_MONITOR_USER}
|
||||
password: ${env:POSTGRES_MONITOR_PASSWORD}
|
||||
databases:
|
||||
- suppliers_db
|
||||
collection_interval: 60s
|
||||
tls:
|
||||
insecure: false
|
||||
cert_file: /etc/postgres-tls/server-cert.pem
|
||||
key_file: /etc/postgres-tls/server-key.pem
|
||||
ca_file: /etc/postgres-tls/ca-cert.pem
|
||||
|
||||
postgresql/tenant:
|
||||
endpoint: tenant-db-service.bakery-ia:5432
|
||||
username: ${env:POSTGRES_MONITOR_USER}
|
||||
password: ${env:POSTGRES_MONITOR_PASSWORD}
|
||||
databases:
|
||||
- tenant_db
|
||||
collection_interval: 60s
|
||||
tls:
|
||||
insecure: false
|
||||
cert_file: /etc/postgres-tls/server-cert.pem
|
||||
key_file: /etc/postgres-tls/server-key.pem
|
||||
ca_file: /etc/postgres-tls/ca-cert.pem
|
||||
|
||||
postgresql/training:
|
||||
endpoint: training-db-service.bakery-ia:5432
|
||||
username: ${env:POSTGRES_MONITOR_USER}
|
||||
password: ${env:POSTGRES_MONITOR_PASSWORD}
|
||||
databases:
|
||||
- training_db
|
||||
collection_interval: 60s
|
||||
tls:
|
||||
insecure: false
|
||||
cert_file: /etc/postgres-tls/server-cert.pem
|
||||
key_file: /etc/postgres-tls/server-key.pem
|
||||
ca_file: /etc/postgres-tls/ca-cert.pem
|
||||
|
||||
# Redis receiver for cache metrics
|
||||
# ENABLED: Using existing credentials from redis-secrets with TLS
|
||||
redis:
|
||||
endpoint: redis-service.bakery-ia:6379
|
||||
password: ${env:REDIS_PASSWORD}
|
||||
collection_interval: 60s
|
||||
transport: tcp
|
||||
tls:
|
||||
insecure_skip_verify: false
|
||||
cert_file: /etc/redis-tls/redis-cert.pem
|
||||
key_file: /etc/redis-tls/redis-key.pem
|
||||
ca_file: /etc/redis-tls/ca-cert.pem
|
||||
metrics:
|
||||
redis.maxmemory:
|
||||
enabled: true
|
||||
redis.cmd.latency:
|
||||
enabled: true
|
||||
|
||||
# RabbitMQ receiver via management API
|
||||
# ENABLED: Using existing credentials from rabbitmq-secrets
|
||||
rabbitmq:
|
||||
endpoint: http://rabbitmq-service.bakery-ia:15672
|
||||
username: ${env:RABBITMQ_USER}
|
||||
password: ${env:RABBITMQ_PASSWORD}
|
||||
collection_interval: 30s
|
||||
|
||||
# Prometheus Receiver - Scrapes metrics from Kubernetes API
|
||||
# Simplified configuration using only Kubernetes API metrics
|
||||
prometheus:
|
||||
config:
|
||||
scrape_configs:
|
||||
- job_name: 'kubernetes-nodes-cadvisor'
|
||||
scrape_interval: 30s
|
||||
scrape_timeout: 10s
|
||||
scheme: https
|
||||
tls_config:
|
||||
insecure_skip_verify: true
|
||||
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
|
||||
kubernetes_sd_configs:
|
||||
- role: node
|
||||
relabel_configs:
|
||||
- action: labelmap
|
||||
regex: __meta_kubernetes_node_label_(.+)
|
||||
- target_label: __address__
|
||||
replacement: kubernetes.default.svc:443
|
||||
- source_labels: [__meta_kubernetes_node_name]
|
||||
regex: (.+)
|
||||
target_label: __metrics_path__
|
||||
replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
|
||||
- job_name: 'kubernetes-apiserver'
|
||||
scrape_interval: 30s
|
||||
scrape_timeout: 10s
|
||||
scheme: https
|
||||
tls_config:
|
||||
insecure_skip_verify: true
|
||||
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
|
||||
kubernetes_sd_configs:
|
||||
- role: endpoints
|
||||
relabel_configs:
|
||||
- source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
|
||||
action: keep
|
||||
regex: default;kubernetes;https
|
||||
|
||||
processors:
|
||||
# Batch processor for better performance (optimized for high throughput)
|
||||
batch:
|
||||
timeout: 1s
|
||||
send_batch_size: 10000 # Increased from 1024 for better performance
|
||||
send_batch_max_size: 10000
|
||||
|
||||
# Batch processor for meter data
|
||||
batch/meter:
|
||||
timeout: 1s
|
||||
send_batch_size: 20000
|
||||
send_batch_max_size: 25000
|
||||
|
||||
# Memory limiter to prevent OOM
|
||||
memory_limiter:
|
||||
check_interval: 1s
|
||||
limit_mib: 400
|
||||
spike_limit_mib: 100
|
||||
|
||||
# Resource detection
|
||||
resourcedetection:
|
||||
detectors: [env, system, docker]
|
||||
timeout: 5s
|
||||
|
||||
# Kubernetes attributes processor - CRITICAL for logs
|
||||
# Extracts pod, namespace, container metadata from log attributes
|
||||
k8sattributes:
|
||||
auth_type: "serviceAccount"
|
||||
passthrough: false
|
||||
extract:
|
||||
metadata:
|
||||
- k8s.pod.name
|
||||
- k8s.pod.uid
|
||||
- k8s.deployment.name
|
||||
- k8s.namespace.name
|
||||
- k8s.node.name
|
||||
- k8s.container.name
|
||||
labels:
|
||||
- tag_name: "app"
|
||||
- tag_name: "pod-template-hash"
|
||||
annotations:
|
||||
- tag_name: "description"
|
||||
|
||||
# SigNoz span metrics processor with delta aggregation (recommended)
|
||||
# Generates RED metrics (Rate, Error, Duration) from trace spans
|
||||
signozspanmetrics/delta:
|
||||
aggregation_temporality: AGGREGATION_TEMPORALITY_DELTA
|
||||
metrics_exporter: signozclickhousemetrics
|
||||
latency_histogram_buckets: [100us, 1ms, 2ms, 6ms, 10ms, 50ms, 100ms, 250ms, 500ms, 1000ms, 1400ms, 2000ms, 5s, 10s, 20s, 40s, 60s]
|
||||
dimensions_cache_size: 100000
|
||||
dimensions:
|
||||
- name: service.namespace
|
||||
default: default
|
||||
- name: deployment.environment
|
||||
default: default
|
||||
- name: signoz.collector.id
|
||||
|
||||
exporters:
|
||||
# ClickHouse exporter for traces
|
||||
clickhousetraces:
|
||||
datasource: tcp://admin:27ff0399-0d3a-4bd8-919d-17c2181e6fb9@signoz-clickhouse:9000/?database=signoz_traces
|
||||
timeout: 10s
|
||||
retry_on_failure:
|
||||
enabled: true
|
||||
initial_interval: 5s
|
||||
max_interval: 30s
|
||||
max_elapsed_time: 300s
|
||||
|
||||
# ClickHouse exporter for metrics
|
||||
signozclickhousemetrics:
|
||||
dsn: "tcp://admin:27ff0399-0d3a-4bd8-919d-17c2181e6fb9@signoz-clickhouse:9000/signoz_metrics"
|
||||
timeout: 10s
|
||||
retry_on_failure:
|
||||
enabled: true
|
||||
initial_interval: 5s
|
||||
max_interval: 30s
|
||||
max_elapsed_time: 300s
|
||||
|
||||
# ClickHouse exporter for meter data (usage metrics)
|
||||
signozclickhousemeter:
|
||||
dsn: "tcp://admin:27ff0399-0d3a-4bd8-919d-17c2181e6fb9@signoz-clickhouse:9000/signoz_meter"
|
||||
timeout: 45s
|
||||
sending_queue:
|
||||
enabled: false
|
||||
|
||||
# ClickHouse exporter for logs
|
||||
clickhouselogsexporter:
|
||||
dsn: tcp://admin:27ff0399-0d3a-4bd8-919d-17c2181e6fb9@signoz-clickhouse:9000/?database=signoz_logs
|
||||
timeout: 10s
|
||||
retry_on_failure:
|
||||
enabled: true
|
||||
initial_interval: 5s
|
||||
max_interval: 30s
|
||||
|
||||
# Metadata exporter for service metadata
|
||||
metadataexporter:
|
||||
dsn: "tcp://admin:27ff0399-0d3a-4bd8-919d-17c2181e6fb9@signoz-clickhouse:9000/signoz_metadata"
|
||||
timeout: 10s
|
||||
cache:
|
||||
provider: in_memory
|
||||
|
||||
# Debug exporter for debugging (optional)
|
||||
debug:
|
||||
verbosity: detailed
|
||||
sampling_initial: 5
|
||||
sampling_thereafter: 200
|
||||
|
||||
service:
|
||||
pipelines:
|
||||
# Traces pipeline - exports to ClickHouse and signozmeter connector
|
||||
traces:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch, signozspanmetrics/delta, resourcedetection]
|
||||
exporters: [clickhousetraces, metadataexporter, signozmeter]
|
||||
|
||||
# Metrics pipeline
|
||||
metrics:
|
||||
receivers: [otlp,
|
||||
postgresql/auth, postgresql/inventory, postgresql/orders,
|
||||
postgresql/ai-insights, postgresql/alert-processor, postgresql/distribution,
|
||||
postgresql/external, postgresql/forecasting, postgresql/notification,
|
||||
postgresql/orchestrator, postgresql/pos, postgresql/procurement,
|
||||
postgresql/production, postgresql/recipes, postgresql/sales,
|
||||
postgresql/suppliers, postgresql/tenant, postgresql/training,
|
||||
redis, rabbitmq, k8s_cluster, prometheus]
|
||||
processors: [memory_limiter, batch, resourcedetection]
|
||||
exporters: [signozclickhousemetrics]
|
||||
|
||||
# Meter pipeline - receives from signozmeter connector
|
||||
metrics/meter:
|
||||
receivers: [signozmeter]
|
||||
processors: [batch/meter]
|
||||
exporters: [signozclickhousemeter]
|
||||
|
||||
# Logs pipeline - includes both OTLP and Kubernetes pod logs
|
||||
logs:
|
||||
receivers: [otlp, filelog]
|
||||
processors: [memory_limiter, batch, resourcedetection, k8sattributes]
|
||||
exporters: [clickhouselogsexporter]
|
||||
|
||||
# ClusterRole configuration for Kubernetes monitoring
|
||||
# CRITICAL: Required for k8s_cluster receiver to access Kubernetes API
|
||||
# Without these permissions, k8s metrics will not appear in SigNoz UI
|
||||
clusterRole:
|
||||
create: true
|
||||
name: "signoz-otel-collector-bakery-ia"
|
||||
annotations: {}
|
||||
# Complete RBAC rules required by k8sclusterreceiver
|
||||
# Based on OpenTelemetry and SigNoz official documentation
|
||||
rules:
|
||||
# Core API group - fundamental Kubernetes resources
|
||||
- apiGroups: [""]
|
||||
resources:
|
||||
- "events"
|
||||
- "namespaces"
|
||||
- "nodes"
|
||||
- "nodes/proxy"
|
||||
- "nodes/metrics"
|
||||
- "nodes/spec"
|
||||
- "pods"
|
||||
- "pods/status"
|
||||
- "replicationcontrollers"
|
||||
- "replicationcontrollers/status"
|
||||
- "resourcequotas"
|
||||
- "services"
|
||||
- "endpoints"
|
||||
verbs: ["get", "list", "watch"]
|
||||
# Apps API group - modern workload controllers
|
||||
- apiGroups: ["apps"]
|
||||
resources: ["deployments", "daemonsets", "statefulsets", "replicasets"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
# Batch API group - job management
|
||||
- apiGroups: ["batch"]
|
||||
resources: ["jobs", "cronjobs"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
# Autoscaling API group - HPA metrics (CRITICAL)
|
||||
- apiGroups: ["autoscaling"]
|
||||
resources: ["horizontalpodautoscalers"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
# Extensions API group - legacy support
|
||||
- apiGroups: ["extensions"]
|
||||
resources: ["deployments", "daemonsets", "replicasets"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
# Metrics API group - resource metrics
|
||||
- apiGroups: ["metrics.k8s.io"]
|
||||
resources: ["nodes", "pods"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
clusterRoleBinding:
|
||||
annotations: {}
|
||||
name: "signoz-otel-collector-bakery-ia"
|
||||
|
||||
# Additional Configuration
|
||||
serviceAccount:
|
||||
create: true
|
||||
annotations: {}
|
||||
name: "signoz-otel-collector"
|
||||
|
||||
# Security Context
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 1000
|
||||
fsGroup: 1000
|
||||
|
||||
# Network Policies (disabled for dev)
|
||||
networkPolicy:
|
||||
enabled: false
|
||||
|
||||
# Monitoring SigNoz itself
|
||||
selfMonitoring:
|
||||
enabled: true
|
||||
serviceMonitor:
|
||||
enabled: false
|
||||
|
||||
Reference in New Issue
Block a user