bakery-ia/shared/monitoring/logs_exporter.py

"""
OpenTelemetry Logs Integration for SigNoz
Exports structured logs to SigNoz via OpenTelemetry Collector
"""

import os
import logging
import structlog
from typing import Optional
from opentelemetry._logs import set_logger_provider
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
try:
    from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
except ImportError:
    try:
        from opentelemetry.exporter.otlp.proto.http.log_exporter import OTLPLogExporter
    except ImportError:
        OTLPLogExporter = None
from opentelemetry.sdk.resources import Resource, SERVICE_NAME, SERVICE_VERSION

logger = structlog.get_logger()


def setup_otel_logging(
    service_name: str,
    service_version: str = "1.0.0",
    otel_endpoint: Optional[str] = None,
    enable_console: bool = True
) -> Optional[LoggingHandler]:
    """
    Setup OpenTelemetry logging to export logs to SigNoz.

    This integrates with Python's standard logging to automatically
    export all log records to SigNoz via the OTLP protocol.

    Args:
        service_name: Name of the service (e.g., "auth-service")
        service_version: Version of the service
        otel_endpoint: OpenTelemetry collector endpoint (default from env)
        enable_console: Whether to also log to console (default: True)

    Returns:
        LoggingHandler instance if successful, None otherwise

    Example:
        from shared.monitoring.logs_exporter import setup_otel_logging

        # Setup during service initialization
        setup_otel_logging("auth-service", "1.0.0")

        # Now all standard logging calls will be exported to SigNoz
        import logging
        logger = logging.getLogger(__name__)
        logger.info("This will appear in SigNoz!")
    """

    # Check if logging export is enabled
    if os.getenv("OTEL_LOGS_EXPORTER", "").lower() != "otlp":
        logger.info(
            "OpenTelemetry logs export disabled",
            service=service_name,
            reason="OTEL_LOGS_EXPORTER not set to 'otlp'"
        )
        return None

    # Get OTLP endpoint from environment or parameter
    # For logs, we need to use the HTTP endpoint (port 4318), not the gRPC endpoint (port 4317)
    if otel_endpoint is None:
        # Try logs-specific endpoint first, then fall back to general OTLP endpoint
        otel_endpoint = os.getenv(
            "OTEL_EXPORTER_OTLP_LOGS_ENDPOINT",
            os.getenv("OTEL_COLLECTOR_ENDPOINT", "http://signoz-otel-collector.bakery-ia:4318")
        )
        
        logger.info(f"Original OTLP endpoint for logs: {otel_endpoint}")
        
        # If we got the tracing endpoint (4317), switch to logs endpoint (4318)
        if otel_endpoint.endswith(":4317"):
            logger.info("Converting tracing endpoint (4317) to logs endpoint (4318)")
            otel_endpoint = otel_endpoint.replace(":4317", ":4318")
        
        logger.info(f"Final OTLP endpoint for logs: {otel_endpoint}")

    # Ensure endpoint has proper protocol prefix
    if not otel_endpoint.startswith(("http://", "https://")):
        # Default to HTTP for insecure connections
        otel_endpoint = f"http://{otel_endpoint}"

    # Ensure endpoint has /v1/logs path for HTTP
    if not otel_endpoint.endswith("/v1/logs"):
        otel_endpoint = f"{otel_endpoint}/v1/logs"

    try:
        # Check if OTLPLogExporter is available
        if OTLPLogExporter is None:
            logger.warning(
                "OpenTelemetry HTTP OTLP exporter not available",
                service=service_name,
                reason="opentelemetry-exporter-otlp-proto-http package not installed"
            )
            return None

        # Create resource with service information
        resource = Resource(attributes={
            SERVICE_NAME: service_name,
            SERVICE_VERSION: service_version,
            "deployment.environment": os.getenv("ENVIRONMENT", "development"),
            "k8s.namespace.name": os.getenv("K8S_NAMESPACE", "bakery-ia"),
            "k8s.pod.name": os.getenv("HOSTNAME", "unknown"),
        })

        # Configure logger provider
        logger_provider = LoggerProvider(resource=resource)
        set_logger_provider(logger_provider)

        # Configure OTLP exporter for logs
        otlp_exporter = OTLPLogExporter(
            endpoint=otel_endpoint,
            timeout=10
        )

        # Add log record processor with batching
        log_processor = BatchLogRecordProcessor(otlp_exporter)
        logger_provider.add_log_record_processor(log_processor)

        # Create logging handler that bridges standard logging to OpenTelemetry
        otel_handler = LoggingHandler(
            level=logging.NOTSET,  # Capture all levels
            logger_provider=logger_provider
        )

        # Add handler to root logger
        root_logger = logging.getLogger()
        root_logger.addHandler(otel_handler)

        logger.info(
            "OpenTelemetry logs export configured",
            service=service_name,
            otel_endpoint=otel_endpoint,
            console_logging=enable_console
        )

        return otel_handler

    except Exception as e:
        logger.error(
            "Failed to setup OpenTelemetry logs export",
            service=service_name,
            error=str(e),
            reason="Will continue with standard logging only"
        )
        return None


def add_log_context(**context):
    """
    Add contextual information to logs that will be sent to SigNoz.

    This is useful for adding request IDs, user IDs, tenant IDs, etc.
    that help with filtering and correlation in SigNoz.

    Args:
        **context: Key-value pairs to add to log context

    Example:
        from shared.monitoring.logs_exporter import add_log_context

        # Add context for current request
        add_log_context(
            request_id="req_123",
            user_id="user_456",
            tenant_id="tenant_789"
        )

        # Now all logs will include this context
        logger.info("Processing order")  # Will include request_id, user_id, tenant_id
    """
    # This works with structlog's context binding
    bound_logger = structlog.get_logger()
    return bound_logger.bind(**context)


def get_current_trace_context() -> dict:
    """
    Get current trace context for log correlation.

    Returns a dict with trace_id and span_id if available,
    which can be added to log records for correlation with traces.

    Returns:
        Dict with trace_id and span_id, or empty dict if no active trace

    Example:
        from shared.monitoring.logs_exporter import get_current_trace_context

        # Get trace context and add to logs
        trace_ctx = get_current_trace_context()
        logger.info("Processing request", **trace_ctx)
    """
    from opentelemetry import trace

    span = trace.get_current_span()
    if span and span.get_span_context().is_valid:
        return {
            "trace_id": format(span.get_span_context().trace_id, '032x'),
            "span_id": format(span.get_span_context().span_id, '016x'),
        }
    return {}


class StructlogOTELProcessor:
    """
    Structlog processor that adds OpenTelemetry trace context to logs.

    This automatically adds trace_id and span_id to all log records,
    enabling correlation between logs and traces in SigNoz.

    Usage:
        import structlog
        from shared.monitoring.logs_exporter import StructlogOTELProcessor

        structlog.configure(
            processors=[
                StructlogOTELProcessor(),
                # ... other processors
            ]
        )
    """

    def __call__(self, logger, method_name, event_dict):
        """Add trace context to log event"""
        trace_ctx = get_current_trace_context()
        if trace_ctx:
            event_dict.update(trace_ctx)
        return event_dict