# ================================================================ # shared/monitoring/logging.py # ================================================================ """ Centralized logging configuration for microservices """ import logging import logging.config import os import sys import resource from typing import Dict, Any def setup_logging(service_name: str, log_level: str = "INFO", enable_json: bool = False, enable_file: bool = True) -> None: """ Set up logging configuration for a microservice with improved error handling. Args: service_name: Name of the service for log identification log_level: Logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL) enable_json: Whether to use JSON formatting enable_file: Whether to enable file logging """ # Check file descriptor limits try: soft_limit, hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE) if soft_limit < 1024: print(f"Warning: Low file descriptor limit ({soft_limit}). Consider increasing with 'ulimit -n'") if soft_limit < 256: print("Critical: File descriptor limit is very low. File logging may fail.") enable_file = False except Exception: # resource module might not be available on all platforms pass # Create logs directory if it doesn't exist and file logging is enabled log_dir = "/var/log" if enable_file: try: # First try to create/write to /var/log test_file = os.path.join(log_dir, f".{service_name}_test") with open(test_file, 'w') as f: f.write("test") os.remove(test_file) except (PermissionError, OSError): # Fallback to local directory if can't write to /var/log log_dir = "./logs" print(f"Warning: Could not write to /var/log, using {log_dir}") try: os.makedirs(log_dir, exist_ok=True) except Exception as e: print(f"Warning: Could not create log directory {log_dir}: {e}") enable_file = False # Disable file logging if we can't create directory # Define formatters formatters = { "standard": { "format": "%(asctime)s [%(levelname)s] %(name)s: %(message)s", "datefmt": "%Y-%m-%d %H:%M:%S" }, "detailed": { "format": "%(asctime)s [%(levelname)s] %(name)s [%(filename)s:%(lineno)d] %(funcName)s(): %(message)s", "datefmt": "%Y-%m-%d %H:%M:%S" } } # Add JSON formatter if requested and available if enable_json: try: import pythonjsonlogger.jsonlogger formatters["json"] = { "()": "pythonjsonlogger.jsonlogger.JsonFormatter", "format": "%(asctime)s %(name)s %(levelname)s %(message)s %(filename)s %(lineno)d" } except ImportError: print("Warning: pythonjsonlogger not available, falling back to standard formatting") enable_json = False # Define handlers handlers = { "console": { "class": "logging.StreamHandler", "level": log_level, "formatter": "json" if enable_json else "standard", "stream": "ext://sys.stdout" } } # Add file handler if enabled if enable_file: try: # Test if we can actually write to the log file location test_filename = f"{log_dir}/{service_name}.log" test_dir = os.path.dirname(test_filename) if not os.access(test_dir, os.W_OK): print(f"Warning: Cannot write to log directory {test_dir}, disabling file logging") enable_file = False else: handlers["file"] = { "class": "logging.FileHandler", "level": log_level, "formatter": "detailed", "filename": test_filename, "mode": "a", "encoding": "utf-8" } except Exception as e: print(f"Warning: Could not configure file handler: {e}") enable_file = False # Add logstash handler if in production logstash_host = os.getenv("LOGSTASH_HOST") if logstash_host and os.getenv("ENVIRONMENT") == "production": try: handlers["logstash"] = { "class": "logstash.TCPLogstashHandler", "host": logstash_host, "port": int(os.getenv("LOGSTASH_PORT", "5000")), "version": 1, "message_type": "logstash", "fqdn": False, "tags": [service_name] } except Exception as e: print(f"Warning: Could not setup logstash handler: {e}") # Define root logger configuration root_handlers = ["console"] if enable_file: root_handlers.append("file") if "logstash" in handlers: root_handlers.append("logstash") # Complete logging configuration config: Dict[str, Any] = { "version": 1, "disable_existing_loggers": False, "formatters": formatters, "handlers": handlers, "loggers": { "": { # Root logger "handlers": root_handlers, "level": log_level, "propagate": False }, "uvicorn": { "handlers": ["console"], "level": log_level, "propagate": False }, "uvicorn.access": { "handlers": ["console"], "level": log_level, "propagate": False }, "sqlalchemy": { "handlers": ["console"], "level": "WARNING", # Reduce SQL logging noise "propagate": False }, "httpx": { "handlers": ["console"], "level": "WARNING", # Reduce HTTP client logging "propagate": False } } } try: logging.config.dictConfig(config) logger = logging.getLogger(__name__) logger.info(f"Logging configured for {service_name} at level {log_level}") if enable_file: logger.info(f"File logging enabled at {log_dir}/{service_name}.log") else: logger.info("File logging disabled") except Exception as e: # Fallback to basic logging if configuration fails logging.basicConfig( level=getattr(logging, log_level.upper()), format="%(asctime)s [%(levelname)s] %(name)s: %(message)s", handlers=[logging.StreamHandler(sys.stdout)] ) logger = logging.getLogger(__name__) logger.error(f"Failed to configure advanced logging for {service_name}: {e}") logger.info(f"Using basic logging configuration for {service_name}") # Additional debugging for file handler issues if "file" in str(e).lower() or "handler" in str(e).lower(): logger.error(f"File handler configuration failed. Check permissions for {log_dir}") logger.error(f"Current working directory: {os.getcwd()}") logger.error(f"Attempting to write to: {log_dir}/{service_name}.log")