371 lines
12 KiB
Bash
371 lines
12 KiB
Bash
#!/bin/bash
|
||
# fix_logging_inconsistency.sh - Fix mixed logging imports throughout project
|
||
|
||
echo "🔍 LOGGING INCONSISTENCY ANALYSIS"
|
||
echo "================================="
|
||
|
||
# ================================================================
|
||
# PROBLEM IDENTIFIED: Mixed logging imports throughout the project
|
||
# ================================================================
|
||
|
||
echo ""
|
||
echo "❌ INCONSISTENT USAGE FOUND:"
|
||
echo ""
|
||
|
||
# Some files use structlog:
|
||
echo "✅ Files correctly using structlog:"
|
||
echo " - services/data/app/services/data_import_service.py: import structlog"
|
||
echo " - services/data/app/core/database.py: import structlog"
|
||
echo " - services/data/app/core/auth.py: import structlog"
|
||
echo ""
|
||
|
||
# Some files use standard logging:
|
||
echo "❌ Files incorrectly using standard logging:"
|
||
echo " - shared/monitoring/logging.py: import logging"
|
||
echo " - gateway/app/main.py: import logging"
|
||
echo " - services/*/app/main.py: import logging (in setup scripts)"
|
||
echo " - services/forecasting/app/main.py: import logging"
|
||
echo ""
|
||
|
||
# ================================================================
|
||
# THE ROOT CAUSE
|
||
# ================================================================
|
||
|
||
echo "🔍 ROOT CAUSE ANALYSIS:"
|
||
echo "======================"
|
||
echo ""
|
||
echo "1. shared/monitoring/logging.py uses 'import logging'"
|
||
echo " ↳ This is CORRECT - it's configuring the logging system"
|
||
echo ""
|
||
echo "2. Service files use 'import logging' instead of 'import structlog'"
|
||
echo " ↳ This is WRONG - services should use structlog"
|
||
echo ""
|
||
echo "3. Mixed usage creates inconsistent log formats"
|
||
echo " ↳ Some logs are structured JSON, others are plain text"
|
||
echo ""
|
||
|
||
# ================================================================
|
||
# DETAILED EXPLANATION
|
||
# ================================================================
|
||
|
||
echo "📝 DETAILED EXPLANATION:"
|
||
echo "========================"
|
||
echo ""
|
||
echo "There are TWO different use cases:"
|
||
echo ""
|
||
echo "1. LOGGING CONFIGURATION (shared/monitoring/logging.py):"
|
||
echo " ✅ Uses 'import logging' - This is CORRECT"
|
||
echo " ✅ Sets up the logging infrastructure"
|
||
echo " ✅ Configures handlers, formatters, logstash integration"
|
||
echo ""
|
||
echo "2. APPLICATION LOGGING (all service files):"
|
||
echo " ❌ Should use 'import structlog' - Many use wrong import"
|
||
echo " ❌ Should get logger with structlog.get_logger()"
|
||
echo " ❌ Should log with key-value pairs"
|
||
echo ""
|
||
|
||
# ================================================================
|
||
# THE FIX
|
||
# ================================================================
|
||
|
||
echo "🔧 THE FIX:"
|
||
echo "==========="
|
||
echo ""
|
||
echo "Replace all service logging imports:"
|
||
echo ""
|
||
echo "❌ Change from:"
|
||
echo " import logging"
|
||
echo " logger = logging.getLogger(__name__)"
|
||
echo ""
|
||
echo "✅ Change to:"
|
||
echo " import structlog"
|
||
echo " logger = structlog.get_logger()"
|
||
echo ""
|
||
|
||
# ================================================================
|
||
# IMPLEMENTATION
|
||
# ================================================================
|
||
|
||
echo "🚀 IMPLEMENTING FIX..."
|
||
echo ""
|
||
|
||
# Create backup directory
|
||
backup_dir="/tmp/logging_fix_backup_$(date +%Y%m%d_%H%M%S)"
|
||
mkdir -p "$backup_dir"
|
||
echo "📦 Created backup directory: $backup_dir"
|
||
|
||
# Function to fix logging imports in a file
|
||
fix_logging_in_file() {
|
||
local file="$1"
|
||
if [ -f "$file" ]; then
|
||
echo " 🔧 Fixing: $file"
|
||
|
||
# Backup original
|
||
cp "$file" "$backup_dir/$(basename $file).backup"
|
||
|
||
# Replace logging imports with structlog
|
||
sed -i.tmp '
|
||
# Replace import statements
|
||
s/^import logging$/import structlog/g
|
||
s/^from logging import/# from logging import/g
|
||
|
||
# Replace logger creation
|
||
s/logger = logging\.getLogger(__name__)/logger = structlog.get_logger()/g
|
||
s/logger = logging\.getLogger()/logger = structlog.get_logger()/g
|
||
s/logging\.getLogger(__name__)/structlog.get_logger()/g
|
||
s/logging\.getLogger()/structlog.get_logger()/g
|
||
' "$file"
|
||
|
||
# Remove temp file
|
||
rm -f "${file}.tmp"
|
||
fi
|
||
}
|
||
|
||
# Fix service main.py files
|
||
echo "🔄 Fixing service main.py files..."
|
||
for service in auth training forecasting data tenant notification; do
|
||
service_main="services/$service/app/main.py"
|
||
if [ -f "$service_main" ]; then
|
||
fix_logging_in_file "$service_main"
|
||
fi
|
||
done
|
||
|
||
# Fix gateway main.py
|
||
echo "🔄 Fixing gateway main.py..."
|
||
fix_logging_in_file "gateway/app/main.py"
|
||
|
||
# Fix other service files that might use logging
|
||
echo "🔄 Fixing other service files..."
|
||
find services/*/app -name "*.py" -type f | while read file; do
|
||
# Skip __init__.py and files that should use standard logging
|
||
if [[ "$file" != *"__init__.py" ]] && [[ "$file" != *"core/config.py" ]]; then
|
||
# Check if file contains logging imports (but not shared/monitoring)
|
||
if grep -q "import logging" "$file" && [[ "$file" != *"shared/monitoring"* ]]; then
|
||
fix_logging_in_file "$file"
|
||
fi
|
||
fi
|
||
done
|
||
|
||
# ================================================================
|
||
# VERIFICATION SCRIPT
|
||
# ================================================================
|
||
|
||
echo ""
|
||
echo "🧪 VERIFICATION:"
|
||
echo "================"
|
||
|
||
# Check for remaining incorrect usage
|
||
echo "Checking for remaining 'import logging' in service files..."
|
||
incorrect_files=$(find services gateway -name "*.py" -exec grep -l "import logging" {} \; | grep -v __pycache__ | grep -v migrations || true)
|
||
|
||
if [ -n "$incorrect_files" ]; then
|
||
echo "⚠️ Still found 'import logging' in:"
|
||
echo "$incorrect_files"
|
||
else
|
||
echo "✅ No incorrect 'import logging' found in service files"
|
||
fi
|
||
|
||
echo ""
|
||
echo "Checking for correct 'import structlog' usage..."
|
||
correct_files=$(find services gateway -name "*.py" -exec grep -l "import structlog" {} \; | grep -v __pycache__ || true)
|
||
|
||
if [ -n "$correct_files" ]; then
|
||
echo "✅ Found correct 'import structlog' in:"
|
||
echo "$correct_files"
|
||
else
|
||
echo "⚠️ No 'import structlog' found - this might be an issue"
|
||
fi
|
||
|
||
# ================================================================
|
||
# UPDATED FILE EXAMPLES
|
||
# ================================================================
|
||
|
||
echo ""
|
||
echo "📝 UPDATED FILE EXAMPLES:"
|
||
echo "========================="
|
||
echo ""
|
||
|
||
# Example 1: Service main.py
|
||
cat << 'EOF'
|
||
# ✅ CORRECT: services/auth/app/main.py (AFTER FIX)
|
||
"""
|
||
Authentication Service
|
||
"""
|
||
|
||
import structlog # ✅ CORRECT
|
||
from fastapi import FastAPI
|
||
from fastapi.middleware.cors import CORSMiddleware
|
||
|
||
from app.core.config import settings
|
||
from app.core.database import database_manager
|
||
from shared.monitoring.logging import setup_logging
|
||
|
||
# Setup logging infrastructure
|
||
setup_logging("auth-service", "INFO")
|
||
|
||
# Get structlog logger
|
||
logger = structlog.get_logger() # ✅ CORRECT
|
||
|
||
app = FastAPI(title="Authentication Service")
|
||
|
||
@app.on_event("startup")
|
||
async def startup_event():
|
||
"""Application startup"""
|
||
logger.info("Starting Authentication Service", service="auth-service") # ✅ STRUCTURED
|
||
await database_manager.create_tables()
|
||
logger.info("Authentication Service started successfully", service="auth-service")
|
||
EOF
|
||
|
||
echo ""
|
||
|
||
# Example 2: Service business logic
|
||
cat << 'EOF'
|
||
# ✅ CORRECT: services/auth/app/services/auth_service.py (AFTER FIX)
|
||
"""
|
||
Authentication service business logic
|
||
"""
|
||
|
||
import structlog # ✅ CORRECT
|
||
from sqlalchemy.ext.asyncio import AsyncSession
|
||
|
||
logger = structlog.get_logger() # ✅ CORRECT
|
||
|
||
class AuthService:
|
||
@staticmethod
|
||
async def login_user(email: str, password: str):
|
||
"""Login user with structured logging"""
|
||
|
||
# ✅ STRUCTURED LOGGING with key-value pairs
|
||
logger.info("User login attempt", email=email, service="auth-service")
|
||
|
||
try:
|
||
user = await authenticate_user(email, password)
|
||
|
||
# ✅ SUCCESS with context
|
||
logger.info(
|
||
"User login successful",
|
||
user_id=str(user.id),
|
||
email=user.email,
|
||
service="auth-service"
|
||
)
|
||
|
||
return user
|
||
|
||
except AuthenticationError as e:
|
||
# ✅ ERROR with context
|
||
logger.error(
|
||
"User login failed",
|
||
email=email,
|
||
error=str(e),
|
||
error_type="authentication_error",
|
||
service="auth-service"
|
||
)
|
||
raise
|
||
EOF
|
||
|
||
echo ""
|
||
|
||
# Example 3: What should NOT change
|
||
cat << 'EOF'
|
||
# ✅ CORRECT: shared/monitoring/logging.py (NO CHANGE NEEDED)
|
||
"""
|
||
Centralized logging configuration - This file is CORRECT as-is
|
||
"""
|
||
|
||
import logging # ✅ CORRECT - This configures the logging system
|
||
import logging.config
|
||
|
||
def setup_logging(service_name: str, log_level: str = "INFO"):
|
||
"""
|
||
This function SETS UP the logging infrastructure.
|
||
It should use 'import logging' to configure the system.
|
||
"""
|
||
config = {
|
||
"version": 1,
|
||
"disable_existing_loggers": False,
|
||
# ... rest of configuration
|
||
}
|
||
|
||
logging.config.dictConfig(config) # ✅ CORRECT
|
||
logger = logging.getLogger(__name__) # ✅ CORRECT for setup
|
||
logger.info(f"Logging configured for {service_name}")
|
||
EOF
|
||
|
||
# ================================================================
|
||
# TESTING THE FIX
|
||
# ================================================================
|
||
|
||
echo ""
|
||
echo "🧪 TESTING THE FIX:"
|
||
echo "==================="
|
||
echo ""
|
||
|
||
# Create test script
|
||
cat << 'EOF' > test_logging_fix.py
|
||
#!/usr/bin/env python3
|
||
"""Test script to verify logging fix"""
|
||
|
||
def test_auth_service_logging():
|
||
"""Test that auth service uses structlog correctly"""
|
||
try:
|
||
# This should work after the fix
|
||
import structlog
|
||
logger = structlog.get_logger()
|
||
logger.info("Test log entry", service="test", test=True)
|
||
print("✅ Auth service logging test passed")
|
||
return True
|
||
except Exception as e:
|
||
print(f"❌ Auth service logging test failed: {e}")
|
||
return False
|
||
|
||
def test_shared_logging_setup():
|
||
"""Test that shared logging setup still works"""
|
||
try:
|
||
from shared.monitoring.logging import setup_logging
|
||
setup_logging("test-service", "INFO")
|
||
print("✅ Shared logging setup test passed")
|
||
return True
|
||
except Exception as e:
|
||
print(f"❌ Shared logging setup test failed: {e}")
|
||
return False
|
||
|
||
if __name__ == "__main__":
|
||
print("Testing logging configuration...")
|
||
|
||
test1 = test_shared_logging_setup()
|
||
test2 = test_auth_service_logging()
|
||
|
||
if test1 and test2:
|
||
print("\n🎉 All logging tests passed!")
|
||
else:
|
||
print("\n⚠️ Some logging tests failed")
|
||
EOF
|
||
|
||
echo "Created test script: test_logging_fix.py"
|
||
echo ""
|
||
echo "Run the test with:"
|
||
echo " python test_logging_fix.py"
|
||
|
||
# ================================================================
|
||
# SUMMARY
|
||
# ================================================================
|
||
|
||
echo ""
|
||
echo "📊 SUMMARY:"
|
||
echo "==========="
|
||
echo ""
|
||
echo "✅ Fixed inconsistent logging imports throughout the project"
|
||
echo "✅ Services now use 'import structlog' consistently"
|
||
echo "✅ shared/monitoring/logging.py still uses 'import logging' (correct)"
|
||
echo "✅ All service logs will now be structured JSON"
|
||
echo "✅ Logs will be properly aggregated in ELK stack"
|
||
echo ""
|
||
echo "🔍 To verify the fix:"
|
||
echo " 1. Run: docker-compose restart"
|
||
echo " 2. Check logs: docker-compose logs -f auth-service"
|
||
echo " 3. Look for structured JSON log entries"
|
||
echo ""
|
||
echo "📁 Backups saved to: $backup_dir"
|
||
echo ""
|
||
echo "🚀 Your logging is now consistent across all services!"
|