Imporve monitoring 3

infrastructure/helm/signoz-values-dev.yaml

@@ -181,6 +181,15 @@ otelCollector:
 
   # OpenTelemetry Collector configuration
   config:
+    # Connectors - bridge between pipelines
+    connectors:
+      signozmeter:
+        dimensions:
+          - name: service.name
+          - name: deployment.environment
+          - name: host.name
+        metrics_flush_interval: 1h
+
     receivers:
       # OTLP receivers for traces, metrics, and logs from applications
       # All application telemetry is pushed via OTLP protocol
@@ -256,6 +265,12 @@ otelCollector:
         send_batch_size: 10000  # Increased from 1024 for better performance
         send_batch_max_size: 10000
 
+      # Batch processor for meter data
+      batch/meter:
+        timeout: 1s
+        send_batch_size: 20000
+        send_batch_max_size: 25000
+
       # Memory limiter to prevent OOM
       memory_limiter:
         check_interval: 1s
@@ -267,11 +282,19 @@ otelCollector:
         detectors: [env, system, docker]
         timeout: 5s
 
-      # Span metrics processor for automatic service metrics
-      spanmetrics:
+      # SigNoz span metrics processor with delta aggregation (recommended)
+      # Generates RED metrics (Rate, Error, Duration) from trace spans
+      signozspanmetrics/delta:
+        aggregation_temporality: AGGREGATION_TEMPORALITY_DELTA
         metrics_exporter: signozclickhousemetrics
-        latency_histogram_buckets: [2ms, 4ms, 6ms, 8ms, 10ms, 50ms, 100ms, 200ms, 400ms, 800ms, 1s, 1400ms, 2s, 5s, 10s, 15s]
-        dimensions_cache_size: 10000
+        latency_histogram_buckets: [100us, 1ms, 2ms, 6ms, 10ms, 50ms, 100ms, 250ms, 500ms, 1000ms, 1400ms, 2000ms, 5s, 10s, 20s, 40s, 60s]
+        dimensions_cache_size: 100000
+        dimensions:
+          - name: service.namespace
+            default: default
+          - name: deployment.environment
+            default: default
+          - name: signoz.collector.id
 
     exporters:
       # ClickHouse exporter for traces
@@ -294,6 +317,13 @@ otelCollector:
           max_interval: 30s
           max_elapsed_time: 300s
 
+      # ClickHouse exporter for meter data (usage metrics)
+      signozclickhousemeter:
+        dsn: "tcp://admin:27ff0399-0d3a-4bd8-919d-17c2181e6fb9@signoz-clickhouse:9000/signoz_meter"
+        timeout: 45s
+        sending_queue:
+          enabled: false
+
       # ClickHouse exporter for logs
       clickhouselogsexporter:
         dsn: tcp://signoz-clickhouse:9000/?database=signoz_logs
@@ -303,6 +333,13 @@ otelCollector:
           initial_interval: 5s
           max_interval: 30s
 
+      # Metadata exporter for service metadata
+      metadataexporter:
+        dsn: "tcp://admin:27ff0399-0d3a-4bd8-919d-17c2181e6fb9@signoz-clickhouse:9000/signoz_metadata"
+        timeout: 10s
+        cache:
+          provider: in_memory
+
       # Debug exporter for debugging (optional)
       debug:
         verbosity: detailed
@@ -311,11 +348,11 @@ otelCollector:
 
     service:
       pipelines:
-        # Traces pipeline
+        # Traces pipeline - exports to ClickHouse and signozmeter connector
         traces:
           receivers: [otlp]
-          processors: [memory_limiter, batch, spanmetrics, resourcedetection]
-          exporters: [clickhousetraces]
+          processors: [memory_limiter, batch, signozspanmetrics/delta, resourcedetection]
+          exporters: [clickhousetraces, metadataexporter, signozmeter]
 
         # Metrics pipeline
         metrics:
@@ -323,6 +360,12 @@ otelCollector:
           processors: [memory_limiter, batch, resourcedetection]
           exporters: [signozclickhousemetrics]
 
+        # Meter pipeline - receives from signozmeter connector
+        metrics/meter:
+          receivers: [signozmeter]
+          processors: [batch/meter]
+          exporters: [signozclickhousemeter]
+
         # Logs pipeline
         logs:
           receivers: [otlp]