Fix shared issues
This commit is contained in:
Urtzi Alfaro
@@ -1,147 +1,150 @@
|
||||
# shared/monitoring/metrics.py
|
||||
# ================================================================
|
||||
# shared/monitoring/metrics.py - FIXED VERSION
|
||||
# ================================================================
|
||||
"""
|
||||
Metrics collection for microservices
|
||||
Centralized metrics collection for microservices - Fixed middleware issue
|
||||
"""
|
||||
|
||||
import time
|
||||
import logging
|
||||
from typing import Dict, Any, List # Added List import
|
||||
from prometheus_client import Counter, Histogram, Gauge, start_http_server
|
||||
from functools import wraps
|
||||
from prometheus_client import generate_latest # Moved this import here for consistency
|
||||
from fastapi import Request
|
||||
from typing import Dict, Any, List, Optional
|
||||
from prometheus_client import Counter, Histogram, Gauge, start_http_server, generate_latest
|
||||
from fastapi import Request, Response
|
||||
from threading import Lock
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Prometheus metrics
|
||||
REQUEST_COUNT = Counter(
|
||||
# Global registry for metrics collectors
|
||||
_metrics_registry: Dict[str, 'MetricsCollector'] = {}
|
||||
_registry_lock = Lock()
|
||||
|
||||
# Default Prometheus metrics
|
||||
DEFAULT_REQUEST_COUNT = Counter(
|
||||
'http_requests_total',
|
||||
'Total HTTP requests',
|
||||
['method', 'endpoint', 'status_code', 'service']
|
||||
)
|
||||
|
||||
REQUEST_DURATION = Histogram(
|
||||
DEFAULT_REQUEST_DURATION = Histogram(
|
||||
'http_request_duration_seconds',
|
||||
'HTTP request duration in seconds',
|
||||
['method', 'endpoint', 'service']
|
||||
)
|
||||
|
||||
ACTIVE_CONNECTIONS = Gauge(
|
||||
DEFAULT_ACTIVE_CONNECTIONS = Gauge(
|
||||
'active_connections',
|
||||
'Active database connections',
|
||||
['service']
|
||||
)
|
||||
|
||||
TRAINING_JOBS = Counter(
|
||||
'training_jobs_total',
|
||||
'Total training jobs',
|
||||
['status', 'service']
|
||||
)
|
||||
|
||||
FORECASTS_GENERATED = Counter(
|
||||
'forecasts_generated_total',
|
||||
'Total forecasts generated',
|
||||
['service']
|
||||
)
|
||||
|
||||
class MetricsCollector:
|
||||
"""Metrics collector for microservices"""
|
||||
"""Thread-safe metrics collector for microservices"""
|
||||
|
||||
def __init__(self, service_name: str):
|
||||
self.service_name = service_name
|
||||
self.start_time = time.time()
|
||||
# Initialize dictionaries to hold custom counters and histograms
|
||||
self._counters: Dict[str, Counter] = {}
|
||||
self._histograms: Dict[str, Histogram] = {}
|
||||
self._gauges: Dict[str, Gauge] = {}
|
||||
self._lock = Lock()
|
||||
|
||||
# Register in global registry
|
||||
with _registry_lock:
|
||||
_metrics_registry[service_name] = self
|
||||
|
||||
def start_metrics_server(self, port: int = 8080):
|
||||
"""Start Prometheus metrics server"""
|
||||
try:
|
||||
start_http_server(port)
|
||||
logger.info(f"Metrics server started on port {port}")
|
||||
logger.info(f"Metrics server started on port {port} for {self.service_name}")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to start metrics server: {e}")
|
||||
logger.error(f"Failed to start metrics server for {self.service_name}: {e}")
|
||||
|
||||
def record_request(self, method: str, endpoint: str, status_code: int, duration: float):
|
||||
"""Record HTTP request metrics"""
|
||||
REQUEST_COUNT.labels(
|
||||
method=method,
|
||||
endpoint=endpoint,
|
||||
status_code=status_code,
|
||||
service=self.service_name
|
||||
).inc()
|
||||
|
||||
REQUEST_DURATION.labels(
|
||||
method=method,
|
||||
endpoint=endpoint,
|
||||
service=self.service_name
|
||||
).observe(duration)
|
||||
|
||||
def record_training_job(self, status: str):
|
||||
"""Record training job metrics"""
|
||||
TRAINING_JOBS.labels(
|
||||
status=status,
|
||||
service=self.service_name
|
||||
).inc()
|
||||
|
||||
def record_forecast_generated(self):
|
||||
"""Record forecast generation metrics"""
|
||||
FORECASTS_GENERATED.labels(
|
||||
service=self.service_name
|
||||
).inc()
|
||||
|
||||
def set_active_connections(self, count: int):
|
||||
"""Set active database connections"""
|
||||
ACTIVE_CONNECTIONS.labels(
|
||||
service=self.service_name
|
||||
).set(count)
|
||||
|
||||
def register_counter(self, name: str, documentation: str, labels: List[str] = None):
|
||||
def register_counter(self, name: str, documentation: str, labels: List[str] = None) -> Counter:
|
||||
"""Register a custom Counter metric."""
|
||||
if name not in self._counters:
|
||||
with self._lock:
|
||||
if name in self._counters:
|
||||
logger.warning(f"Counter '{name}' already registered for {self.service_name}")
|
||||
return self._counters[name]
|
||||
|
||||
if labels is None:
|
||||
labels = ['service']
|
||||
elif 'service' not in labels:
|
||||
labels.append('service')
|
||||
# Pass labelnames as a keyword argument
|
||||
self._counters[name] = Counter(name, documentation, labelnames=labels)
|
||||
logger.info(f"Registered counter: {name}")
|
||||
else:
|
||||
logger.warning(f"Counter '{name}' already registered.")
|
||||
return self._counters[name] # Return the counter for direct use if needed
|
||||
|
||||
try:
|
||||
counter = Counter(f"{self.service_name.replace('-', '_')}_{name}", documentation, labelnames=labels)
|
||||
self._counters[name] = counter
|
||||
logger.info(f"Registered counter: {name} for {self.service_name}")
|
||||
return counter
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to register counter {name} for {self.service_name}: {e}")
|
||||
raise
|
||||
|
||||
def register_histogram(self, name: str, documentation: str, labels: List[str] = None,
|
||||
buckets: tuple = Histogram.DEFAULT_BUCKETS) -> Histogram:
|
||||
"""Register a custom Histogram metric."""
|
||||
with self._lock:
|
||||
if name in self._histograms:
|
||||
logger.warning(f"Histogram '{name}' already registered for {self.service_name}")
|
||||
return self._histograms[name]
|
||||
|
||||
if labels is None:
|
||||
labels = ['service']
|
||||
elif 'service' not in labels:
|
||||
labels.append('service')
|
||||
|
||||
try:
|
||||
histogram = Histogram(f"{self.service_name.replace('-', '_')}_{name}", documentation,
|
||||
labelnames=labels, buckets=buckets)
|
||||
self._histograms[name] = histogram
|
||||
logger.info(f"Registered histogram: {name} for {self.service_name}")
|
||||
return histogram
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to register histogram {name} for {self.service_name}: {e}")
|
||||
raise
|
||||
|
||||
def register_gauge(self, name: str, documentation: str, labels: List[str] = None) -> Gauge:
|
||||
"""Register a custom Gauge metric."""
|
||||
with self._lock:
|
||||
if name in self._gauges:
|
||||
logger.warning(f"Gauge '{name}' already registered for {self.service_name}")
|
||||
return self._gauges[name]
|
||||
|
||||
if labels is None:
|
||||
labels = ['service']
|
||||
elif 'service' not in labels:
|
||||
labels.append('service')
|
||||
|
||||
try:
|
||||
gauge = Gauge(f"{self.service_name.replace('-', '_')}_{name}", documentation, labelnames=labels)
|
||||
self._gauges[name] = gauge
|
||||
logger.info(f"Registered gauge: {name} for {self.service_name}")
|
||||
return gauge
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to register gauge {name} for {self.service_name}: {e}")
|
||||
raise
|
||||
|
||||
def increment_counter(self, name: str, value: int = 1, labels: Dict[str, str] = None):
|
||||
"""Increment a custom Counter metric."""
|
||||
"""Increment a counter metric."""
|
||||
if name not in self._counters:
|
||||
logger.error(f"Counter '{name}' not registered. Cannot increment.")
|
||||
logger.error(f"Counter '{name}' not registered for {self.service_name}. Cannot increment.")
|
||||
return
|
||||
|
||||
# Ensure the 'service' label is always present
|
||||
if labels is None:
|
||||
labels = {'service': self.service_name}
|
||||
elif 'service' not in labels:
|
||||
labels['service'] = self.service_name
|
||||
|
||||
self._counters[name].labels(**labels).inc(value)
|
||||
|
||||
def register_histogram(self, name: str, documentation: str, labels: List[str] = None, buckets: tuple = Histogram.DEFAULT_BUCKETS):
|
||||
"""Register a custom Histogram metric."""
|
||||
if name not in self._histograms:
|
||||
if labels is None:
|
||||
labels = ['service']
|
||||
elif 'service' not in labels:
|
||||
labels.append('service')
|
||||
# Pass labelnames and buckets as keyword arguments
|
||||
self._histograms[name] = Histogram(name, documentation, labelnames=labels, buckets=buckets)
|
||||
logger.info(f"Registered histogram: {name}")
|
||||
else:
|
||||
logger.warning(f"Histogram '{name}' already registered.")
|
||||
return self._histograms[name] # Return the histogram for direct use if needed
|
||||
try:
|
||||
self._counters[name].labels(**labels).inc(value)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to increment counter {name} for {self.service_name}: {e}")
|
||||
|
||||
def observe_histogram(self, name: str, value: float, labels: Dict[str, str] = None):
|
||||
"""Observe a custom Histogram metric."""
|
||||
"""Observe a histogram metric."""
|
||||
if name not in self._histograms:
|
||||
logger.error(f"Histogram '{name}' not registered. Cannot observe.")
|
||||
logger.error(f"Histogram '{name}' not registered for {self.service_name}. Cannot observe.")
|
||||
return
|
||||
|
||||
if labels is None:
|
||||
@@ -149,145 +152,146 @@ class MetricsCollector:
|
||||
elif 'service' not in labels:
|
||||
labels['service'] = self.service_name
|
||||
|
||||
self._histograms[name].labels(**labels).observe(value)
|
||||
try:
|
||||
self._histograms[name].labels(**labels).observe(value)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to observe histogram {name} for {self.service_name}: {e}")
|
||||
|
||||
def set_gauge(self, name: str, value: float, labels: Dict[str, str] = None):
|
||||
"""Set a gauge metric."""
|
||||
if name not in self._gauges:
|
||||
logger.error(f"Gauge '{name}' not registered for {self.service_name}. Cannot set.")
|
||||
return
|
||||
|
||||
if labels is None:
|
||||
labels = {'service': self.service_name}
|
||||
elif 'service' not in labels:
|
||||
labels['service'] = self.service_name
|
||||
|
||||
try:
|
||||
self._gauges[name].labels(**labels).set(value)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to set gauge {name} for {self.service_name}: {e}")
|
||||
|
||||
def record_request(self, method: str, endpoint: str, status_code: int, duration: float):
|
||||
"""Record HTTP request metrics using default metrics."""
|
||||
try:
|
||||
DEFAULT_REQUEST_COUNT.labels(
|
||||
method=method,
|
||||
endpoint=endpoint,
|
||||
status_code=status_code,
|
||||
service=self.service_name
|
||||
).inc()
|
||||
|
||||
DEFAULT_REQUEST_DURATION.labels(
|
||||
method=method,
|
||||
endpoint=endpoint,
|
||||
service=self.service_name
|
||||
).observe(duration)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to record request metrics for {self.service_name}: {e}")
|
||||
|
||||
def set_active_connections(self, count: int):
|
||||
"""Set active database connections using default gauge."""
|
||||
try:
|
||||
DEFAULT_ACTIVE_CONNECTIONS.labels(service=self.service_name).set(count)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to set active connections for {self.service_name}: {e}")
|
||||
|
||||
def get_metrics(self) -> str:
|
||||
"""Return Prometheus metrics in exposition format."""
|
||||
return generate_latest().decode('utf-8')
|
||||
try:
|
||||
return generate_latest().decode('utf-8')
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to generate metrics for {self.service_name}: {e}")
|
||||
return ""
|
||||
|
||||
|
||||
def metrics_middleware(metrics_collector: MetricsCollector):
|
||||
"""Middleware to collect metrics"""
|
||||
|
||||
async def middleware(request, call_next):
|
||||
start_time = time.time()
|
||||
|
||||
response = await call_next(request)
|
||||
|
||||
duration = time.time() - start_time
|
||||
|
||||
# Use the specific record_request for HTTP requests
|
||||
metrics_collector.record_request(
|
||||
method=request.method,
|
||||
endpoint=request.url.path,
|
||||
status_code=response.status_code,
|
||||
duration=duration
|
||||
)
|
||||
|
||||
return response
|
||||
|
||||
return middleware
|
||||
def get_metrics_collector(service_name: str) -> Optional[MetricsCollector]:
|
||||
"""Get metrics collector by service name from global registry."""
|
||||
with _registry_lock:
|
||||
return _metrics_registry.get(service_name)
|
||||
|
||||
|
||||
def setup_metrics(app):
|
||||
def create_metrics_collector(service_name: str) -> MetricsCollector:
|
||||
"""
|
||||
Setup metrics collection for FastAPI app
|
||||
|
||||
Args:
|
||||
app: FastAPI application instance
|
||||
|
||||
Returns:
|
||||
MetricsCollector: Configured metrics collector
|
||||
Create metrics collector without adding middleware.
|
||||
This should be called BEFORE app startup, not during lifespan.
|
||||
"""
|
||||
# Get existing or create new
|
||||
existing = get_metrics_collector(service_name)
|
||||
if existing:
|
||||
return existing
|
||||
|
||||
# Get service name from app title or default
|
||||
service_name = getattr(app, 'title', 'unknown-service').lower().replace(' ', '-')
|
||||
|
||||
# Create metrics collector for this service
|
||||
metrics_collector = MetricsCollector(service_name)
|
||||
|
||||
# Add metrics middleware to collect HTTP request metrics
|
||||
return MetricsCollector(service_name)
|
||||
|
||||
|
||||
def add_metrics_middleware(app, metrics_collector: MetricsCollector):
|
||||
"""
|
||||
Add metrics middleware to app. Must be called BEFORE app startup.
|
||||
"""
|
||||
@app.middleware("http")
|
||||
async def collect_metrics_middleware(request: Request, call_next):
|
||||
async def metrics_middleware(request: Request, call_next):
|
||||
start_time = time.time()
|
||||
|
||||
# Process the request
|
||||
response = await call_next(request)
|
||||
|
||||
# Calculate duration
|
||||
duration = time.time() - start_time
|
||||
|
||||
# Record metrics
|
||||
metrics_collector.record_request(
|
||||
method=request.method,
|
||||
endpoint=request.url.path,
|
||||
status_code=response.status_code,
|
||||
duration=duration
|
||||
)
|
||||
|
||||
return response
|
||||
|
||||
# Add metrics endpoint if it doesn't exist
|
||||
@app.get("/metrics")
|
||||
async def prometheus_metrics():
|
||||
"""Prometheus metrics endpoint"""
|
||||
from prometheus_client import generate_latest
|
||||
return Response(
|
||||
content=generate_latest(),
|
||||
media_type="text/plain; version=0.0.4; charset=utf-8"
|
||||
)
|
||||
|
||||
# Store metrics collector in app state for later access
|
||||
app.state.metrics_collector = metrics_collector
|
||||
|
||||
logger.info(f"Metrics collection setup completed for service: {service_name}")
|
||||
try:
|
||||
response = await call_next(request)
|
||||
duration = time.time() - start_time
|
||||
|
||||
# Record request metrics
|
||||
metrics_collector.record_request(
|
||||
method=request.method,
|
||||
endpoint=request.url.path,
|
||||
status_code=response.status_code,
|
||||
duration=duration
|
||||
)
|
||||
|
||||
return response
|
||||
except Exception as e:
|
||||
duration = time.time() - start_time
|
||||
# Record failed request
|
||||
metrics_collector.record_request(
|
||||
method=request.method,
|
||||
endpoint=request.url.path,
|
||||
status_code=500,
|
||||
duration=duration
|
||||
)
|
||||
raise
|
||||
|
||||
return metrics_collector
|
||||
|
||||
|
||||
# Alternative simplified setup function for services that don't need complex metrics
|
||||
def setup_basic_metrics(app, service_name: str = None):
|
||||
def add_metrics_endpoint(app, metrics_collector: MetricsCollector):
|
||||
"""Add metrics endpoint to app"""
|
||||
@app.get("/metrics")
|
||||
async def prometheus_metrics():
|
||||
"""Prometheus metrics endpoint"""
|
||||
return Response(
|
||||
content=metrics_collector.get_metrics(),
|
||||
media_type="text/plain; version=0.0.4; charset=utf-8"
|
||||
)
|
||||
|
||||
|
||||
def setup_metrics_early(app, service_name: str = None) -> MetricsCollector:
|
||||
"""
|
||||
Setup basic metrics collection without complex dependencies
|
||||
|
||||
Args:
|
||||
app: FastAPI application instance
|
||||
service_name: Optional service name override
|
||||
|
||||
Returns:
|
||||
Simple metrics dict
|
||||
Setup metrics collection BEFORE app startup.
|
||||
This must be called before adding any middleware or starting the app.
|
||||
"""
|
||||
if service_name is None:
|
||||
service_name = getattr(app, 'title', 'unknown-service').lower().replace(' ', '-')
|
||||
service_name = getattr(app, 'title', 'unknown-service').lower().replace(' ', '-').replace('.', '_')
|
||||
|
||||
# Simple in-memory metrics
|
||||
metrics_data = {
|
||||
"requests_total": 0,
|
||||
"requests_by_method": {},
|
||||
"requests_by_status": {},
|
||||
"service_name": service_name,
|
||||
"start_time": time.time()
|
||||
}
|
||||
# Create metrics collector
|
||||
metrics_collector = create_metrics_collector(service_name)
|
||||
|
||||
@app.middleware("http")
|
||||
async def simple_metrics_middleware(request: Request, call_next):
|
||||
# Increment total requests
|
||||
metrics_data["requests_total"] += 1
|
||||
|
||||
# Track by method
|
||||
method = request.method
|
||||
metrics_data["requests_by_method"][method] = metrics_data["requests_by_method"].get(method, 0) + 1
|
||||
|
||||
# Process request
|
||||
response = await call_next(request)
|
||||
|
||||
# Track by status code
|
||||
status = str(response.status_code)
|
||||
metrics_data["requests_by_status"][status] = metrics_data["requests_by_status"].get(status, 0) + 1
|
||||
|
||||
return response
|
||||
# Add middleware (must be before app starts)
|
||||
add_metrics_middleware(app, metrics_collector)
|
||||
|
||||
@app.get("/metrics")
|
||||
async def simple_metrics():
|
||||
"""Simple metrics endpoint"""
|
||||
uptime = time.time() - metrics_data["start_time"]
|
||||
return {
|
||||
**metrics_data,
|
||||
"uptime_seconds": round(uptime, 2)
|
||||
}
|
||||
# Add metrics endpoint
|
||||
add_metrics_endpoint(app, metrics_collector)
|
||||
|
||||
app.state.simple_metrics = metrics_data
|
||||
# Store in app state for access from routes
|
||||
app.state.metrics_collector = metrics_collector
|
||||
|
||||
logger.info(f"Basic metrics setup completed for service: {service_name}")
|
||||
|
||||
return metrics_data
|
||||
logger.info(f"Metrics setup completed for service: {service_name}")
|
||||
return metrics_collector
|
||||
|
||||
|
||||
Reference in New Issue
Block a user