Add signoz

services/alert_processor/app/main.py

@@ -4,25 +4,52 @@ Alert Processor Service v2.0
 Main FastAPI application with RabbitMQ consumer lifecycle management.
 """
 
-from fastapi import FastAPI
+from fastapi import FastAPI, Response
 from fastapi.middleware.cors import CORSMiddleware
 from contextlib import asynccontextmanager
 import structlog
+import os
 
 from app.core.config import settings
 from app.consumer.event_consumer import EventConsumer
 from app.api import alerts, sse
 from shared.redis_utils import initialize_redis, close_redis
+from shared.monitoring.logging import setup_logging
+from shared.monitoring.metrics import MetricsCollector, add_metrics_middleware
 
-# Configure structured logging
-structlog.configure(
-    processors=[
-        structlog.processors.TimeStamper(fmt="iso"),
-        structlog.processors.add_log_level,
-        structlog.processors.JSONRenderer()
-    ]
-)
+# OpenTelemetry imports
+from opentelemetry import trace
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor
+from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
+from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
+from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor
+from opentelemetry.instrumentation.redis import RedisInstrumentor
+from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor
+from opentelemetry.sdk.resources import Resource
 
+# Configure OpenTelemetry tracing
+def setup_tracing(service_name: str = "alert-processor"):
+    """Initialize OpenTelemetry tracing with OTLP exporter for Jaeger"""
+    resource = Resource.create({"service.name": service_name})
+
+    otlp_exporter = OTLPSpanExporter(
+        endpoint=os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel-collector.monitoring.svc.cluster.local:4317"),
+        insecure=True
+    )
+
+    provider = TracerProvider(resource=resource)
+    processor = BatchSpanProcessor(otlp_exporter)
+    provider.add_span_processor(processor)
+    trace.set_tracer_provider(provider)
+
+    return provider
+
+# Initialize tracing
+tracer_provider = setup_tracing("alert-processor")
+
+# Setup logging
+setup_logging("alert-processor", getattr(settings, 'LOG_LEVEL', 'INFO'))
 logger = structlog.get_logger()
 
 # Global consumer instance
@@ -54,6 +81,10 @@ async def lifespan(app: FastAPI):
         consumer = EventConsumer()
         await consumer.start()
         logger.info("alert_processor_started")
+
+        # Start metrics server
+        metrics_collector.start_metrics_server(8080)
+        logger.info("Metrics server started on port 8080")
     except Exception as e:
         logger.error("alert_processor_startup_failed", error=str(e))
         raise
@@ -79,6 +110,24 @@ app = FastAPI(
     debug=settings.DEBUG
 )
 
+# Instrument FastAPI with OpenTelemetry
+FastAPIInstrumentor.instrument_app(app)
+
+# Instrument httpx for outgoing requests
+HTTPXClientInstrumentor().instrument()
+
+# Instrument Redis
+RedisInstrumentor().instrument()
+
+# Instrument SQLAlchemy
+SQLAlchemyInstrumentor().instrument()
+
+# Initialize metrics collector
+metrics_collector = MetricsCollector("alert-processor")
+
+# Add metrics middleware to track HTTP requests
+add_metrics_middleware(app, metrics_collector)
+
 # CORS middleware
 app.add_middleware(
     CORSMiddleware,
@@ -126,6 +175,15 @@ async def root():
     }
 
 
+@app.get("/metrics")
+async def metrics():
+    """Prometheus metrics endpoint"""
+    return Response(
+        content=metrics_collector.get_metrics(),
+        media_type="text/plain; version=0.0.4; charset=utf-8"
+    )
+
+
 if __name__ == "__main__":
     import uvicorn