Fix SigNoz OTel Collector configuration and disable OpAMP
Root Cause Analysis:
- OTel Collector was starting but OpAMP was overwriting config with "nop" receivers/exporters
- ClickHouse authentication was failing due to missing credentials in DSN strings
- Redis/PostgreSQL/RabbitMQ receivers had missing TLS certs causing startup failures
Changes:
1. Fixed ClickHouse Exporters:
- Added admin credentials to clickhousetraces datasource
- Added admin credentials to clickhouselogsexporter dsn
- Now using: tcp://admin:27ff0399-0d3a-4bd8-919d-17c2181e6fb9@signoz-clickhouse:9000/
2. Disabled Unconfigured Receivers:
- Commented out PostgreSQL receivers (no monitor users configured)
- Commented out Redis receiver (TLS certificates not available)
- Commented out RabbitMQ receiver (credentials not configured)
- Updated metrics pipeline to use only OTLP receiver
3. OpAMP Disabled:
- OpAMP was causing collector to use nop exporters/receivers
- Cannot disable via Helm (extraArgs appends, doesn't replace)
- Must apply kubectl patch after Helm install:
kubectl patch deployment signoz-otel-collector --type=json -p='[{"op":"replace","path":"/spec/template/spec/containers/0/args","value":["--config=/conf/otel-collector-config.yaml","--feature-gates=-pkg.translator.prometheus.NormalizeName"]}]'
Results:
✅ OTel Collector successfully receiving traces (97+ spans)
✅ Services connecting without UNAVAILABLE errors
✅ No ClickHouse authentication failures
✅ All pipelines active (traces, metrics, logs)
Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
Urtzi Alfaro
@@ -146,6 +146,10 @@ otelCollector:
|
||||
repository: signoz/signoz-otel-collector
|
||||
tag: v0.129.12 # Latest recommended version
|
||||
|
||||
# NOTE: OpAMP is disabled via kubectl patch on the deployment
|
||||
# Cannot disable via Helm values as extraArgs appends instead of replaces
|
||||
# Patch command: kubectl patch deployment signoz-otel-collector --type=json -p='[{"op":"replace","path":"/spec/template/spec/containers/0/args","value":["--config=/conf/otel-collector-config.yaml","--feature-gates=-pkg.translator.prometheus.NormalizeName"]}]'
|
||||
|
||||
# Service configuration - expose both gRPC and HTTP endpoints
|
||||
service:
|
||||
type: ClusterIP
|
||||
@@ -204,36 +208,37 @@ otelCollector:
|
||||
- "*"
|
||||
|
||||
# PostgreSQL receivers for database metrics
|
||||
# DISABLED: Monitor users not configured yet
|
||||
# Collects metrics directly from PostgreSQL databases
|
||||
postgresql/auth:
|
||||
endpoint: auth-db-service.bakery-ia:5432
|
||||
username: ${POSTGRES_MONITOR_USER}
|
||||
password: ${POSTGRES_MONITOR_PASSWORD}
|
||||
databases:
|
||||
- auth_db
|
||||
collection_interval: 60s
|
||||
tls:
|
||||
insecure: false
|
||||
# postgresql/auth:
|
||||
# endpoint: auth-db-service.bakery-ia:5432
|
||||
# username: ${POSTGRES_MONITOR_USER}
|
||||
# password: ${POSTGRES_MONITOR_PASSWORD}
|
||||
# databases:
|
||||
# - auth_db
|
||||
# collection_interval: 60s
|
||||
# tls:
|
||||
# insecure: false
|
||||
|
||||
postgresql/inventory:
|
||||
endpoint: inventory-db-service.bakery-ia:5432
|
||||
username: ${POSTGRES_MONITOR_USER}
|
||||
password: ${POSTGRES_MONITOR_PASSWORD}
|
||||
databases:
|
||||
- inventory_db
|
||||
collection_interval: 60s
|
||||
tls:
|
||||
insecure: false
|
||||
# postgresql/inventory:
|
||||
# endpoint: inventory-db-service.bakery-ia:5432
|
||||
# username: ${POSTGRES_MONITOR_USER}
|
||||
# password: ${POSTGRES_MONITOR_PASSWORD}
|
||||
# databases:
|
||||
# - inventory_db
|
||||
# collection_interval: 60s
|
||||
# tls:
|
||||
# insecure: false
|
||||
|
||||
postgresql/orders:
|
||||
endpoint: orders-db-service.bakery-ia:5432
|
||||
username: ${POSTGRES_MONITOR_USER}
|
||||
password: ${POSTGRES_MONITOR_PASSWORD}
|
||||
databases:
|
||||
- orders_db
|
||||
collection_interval: 60s
|
||||
tls:
|
||||
insecure: false
|
||||
# postgresql/orders:
|
||||
# endpoint: orders-db-service.bakery-ia:5432
|
||||
# username: ${POSTGRES_MONITOR_USER}
|
||||
# password: ${POSTGRES_MONITOR_PASSWORD}
|
||||
# databases:
|
||||
# - orders_db
|
||||
# collection_interval: 60s
|
||||
# tls:
|
||||
# insecure: false
|
||||
|
||||
# Add more PostgreSQL databases as needed
|
||||
# postgresql/SERVICE:
|
||||
@@ -241,22 +246,24 @@ otelCollector:
|
||||
# ...
|
||||
|
||||
# Redis receiver for cache metrics
|
||||
redis:
|
||||
endpoint: redis-service.bakery-ia:6379
|
||||
password: ${REDIS_PASSWORD}
|
||||
collection_interval: 60s
|
||||
tls:
|
||||
insecure: false
|
||||
cert_file: /etc/redis-tls/redis-cert.pem
|
||||
key_file: /etc/redis-tls/redis-key.pem
|
||||
ca_file: /etc/redis-tls/ca-cert.pem
|
||||
# DISABLED: TLS certificates not configured yet
|
||||
# redis:
|
||||
# endpoint: redis-service.bakery-ia:6379
|
||||
# password: ${REDIS_PASSWORD}
|
||||
# collection_interval: 60s
|
||||
# tls:
|
||||
# insecure: false
|
||||
# cert_file: /etc/redis-tls/redis-cert.pem
|
||||
# key_file: /etc/redis-tls/redis-key.pem
|
||||
# ca_file: /etc/redis-tls/ca-cert.pem
|
||||
|
||||
# RabbitMQ receiver via management API
|
||||
rabbitmq:
|
||||
endpoint: http://rabbitmq-service.bakery-ia:15672
|
||||
username: ${RABBITMQ_USER}
|
||||
password: ${RABBITMQ_PASSWORD}
|
||||
collection_interval: 60s
|
||||
# DISABLED: RabbitMQ credentials not configured yet
|
||||
# rabbitmq:
|
||||
# endpoint: http://rabbitmq-service.bakery-ia:15672
|
||||
# username: ${RABBITMQ_USER}
|
||||
# password: ${RABBITMQ_PASSWORD}
|
||||
# collection_interval: 60s
|
||||
|
||||
processors:
|
||||
# Batch processor for better performance (optimized for high throughput)
|
||||
@@ -299,7 +306,7 @@ otelCollector:
|
||||
exporters:
|
||||
# ClickHouse exporter for traces
|
||||
clickhousetraces:
|
||||
datasource: tcp://signoz-clickhouse:9000/?database=signoz_traces
|
||||
datasource: tcp://admin:27ff0399-0d3a-4bd8-919d-17c2181e6fb9@signoz-clickhouse:9000/?database=signoz_traces
|
||||
timeout: 10s
|
||||
retry_on_failure:
|
||||
enabled: true
|
||||
@@ -326,7 +333,7 @@ otelCollector:
|
||||
|
||||
# ClickHouse exporter for logs
|
||||
clickhouselogsexporter:
|
||||
dsn: tcp://signoz-clickhouse:9000/?database=signoz_logs
|
||||
dsn: tcp://admin:27ff0399-0d3a-4bd8-919d-17c2181e6fb9@signoz-clickhouse:9000/?database=signoz_logs
|
||||
timeout: 10s
|
||||
retry_on_failure:
|
||||
enabled: true
|
||||
@@ -356,7 +363,7 @@ otelCollector:
|
||||
|
||||
# Metrics pipeline
|
||||
metrics:
|
||||
receivers: [otlp, postgresql/auth, postgresql/inventory, postgresql/orders, redis, rabbitmq]
|
||||
receivers: [otlp] # Database/cache receivers disabled until credentials configured
|
||||
processors: [memory_limiter, batch, resourcedetection]
|
||||
exporters: [signozclickhousemetrics]
|
||||
|
||||
|
||||
Reference in New Issue
Block a user