Fix weather data
This commit is contained in:
Urtzi Alfaro
370
fix_logging_inconsistency.sh
Normal file
370
fix_logging_inconsistency.sh
Normal file
@@ -0,0 +1,370 @@
|
||||
#!/bin/bash
|
||||
# fix_logging_inconsistency.sh - Fix mixed logging imports throughout project
|
||||
|
||||
echo "🔍 LOGGING INCONSISTENCY ANALYSIS"
|
||||
echo "================================="
|
||||
|
||||
# ================================================================
|
||||
# PROBLEM IDENTIFIED: Mixed logging imports throughout the project
|
||||
# ================================================================
|
||||
|
||||
echo ""
|
||||
echo "❌ INCONSISTENT USAGE FOUND:"
|
||||
echo ""
|
||||
|
||||
# Some files use structlog:
|
||||
echo "✅ Files correctly using structlog:"
|
||||
echo " - services/data/app/services/data_import_service.py: import structlog"
|
||||
echo " - services/data/app/core/database.py: import structlog"
|
||||
echo " - services/data/app/core/auth.py: import structlog"
|
||||
echo ""
|
||||
|
||||
# Some files use standard logging:
|
||||
echo "❌ Files incorrectly using standard logging:"
|
||||
echo " - shared/monitoring/logging.py: import logging"
|
||||
echo " - gateway/app/main.py: import logging"
|
||||
echo " - services/*/app/main.py: import logging (in setup scripts)"
|
||||
echo " - services/forecasting/app/main.py: import logging"
|
||||
echo ""
|
||||
|
||||
# ================================================================
|
||||
# THE ROOT CAUSE
|
||||
# ================================================================
|
||||
|
||||
echo "🔍 ROOT CAUSE ANALYSIS:"
|
||||
echo "======================"
|
||||
echo ""
|
||||
echo "1. shared/monitoring/logging.py uses 'import logging'"
|
||||
echo " ↳ This is CORRECT - it's configuring the logging system"
|
||||
echo ""
|
||||
echo "2. Service files use 'import logging' instead of 'import structlog'"
|
||||
echo " ↳ This is WRONG - services should use structlog"
|
||||
echo ""
|
||||
echo "3. Mixed usage creates inconsistent log formats"
|
||||
echo " ↳ Some logs are structured JSON, others are plain text"
|
||||
echo ""
|
||||
|
||||
# ================================================================
|
||||
# DETAILED EXPLANATION
|
||||
# ================================================================
|
||||
|
||||
echo "📝 DETAILED EXPLANATION:"
|
||||
echo "========================"
|
||||
echo ""
|
||||
echo "There are TWO different use cases:"
|
||||
echo ""
|
||||
echo "1. LOGGING CONFIGURATION (shared/monitoring/logging.py):"
|
||||
echo " ✅ Uses 'import logging' - This is CORRECT"
|
||||
echo " ✅ Sets up the logging infrastructure"
|
||||
echo " ✅ Configures handlers, formatters, logstash integration"
|
||||
echo ""
|
||||
echo "2. APPLICATION LOGGING (all service files):"
|
||||
echo " ❌ Should use 'import structlog' - Many use wrong import"
|
||||
echo " ❌ Should get logger with structlog.get_logger()"
|
||||
echo " ❌ Should log with key-value pairs"
|
||||
echo ""
|
||||
|
||||
# ================================================================
|
||||
# THE FIX
|
||||
# ================================================================
|
||||
|
||||
echo "🔧 THE FIX:"
|
||||
echo "==========="
|
||||
echo ""
|
||||
echo "Replace all service logging imports:"
|
||||
echo ""
|
||||
echo "❌ Change from:"
|
||||
echo " import logging"
|
||||
echo " logger = logging.getLogger(__name__)"
|
||||
echo ""
|
||||
echo "✅ Change to:"
|
||||
echo " import structlog"
|
||||
echo " logger = structlog.get_logger()"
|
||||
echo ""
|
||||
|
||||
# ================================================================
|
||||
# IMPLEMENTATION
|
||||
# ================================================================
|
||||
|
||||
echo "🚀 IMPLEMENTING FIX..."
|
||||
echo ""
|
||||
|
||||
# Create backup directory
|
||||
backup_dir="/tmp/logging_fix_backup_$(date +%Y%m%d_%H%M%S)"
|
||||
mkdir -p "$backup_dir"
|
||||
echo "📦 Created backup directory: $backup_dir"
|
||||
|
||||
# Function to fix logging imports in a file
|
||||
fix_logging_in_file() {
|
||||
local file="$1"
|
||||
if [ -f "$file" ]; then
|
||||
echo " 🔧 Fixing: $file"
|
||||
|
||||
# Backup original
|
||||
cp "$file" "$backup_dir/$(basename $file).backup"
|
||||
|
||||
# Replace logging imports with structlog
|
||||
sed -i.tmp '
|
||||
# Replace import statements
|
||||
s/^import logging$/import structlog/g
|
||||
s/^from logging import/# from logging import/g
|
||||
|
||||
# Replace logger creation
|
||||
s/logger = logging\.getLogger(__name__)/logger = structlog.get_logger()/g
|
||||
s/logger = logging\.getLogger()/logger = structlog.get_logger()/g
|
||||
s/logging\.getLogger(__name__)/structlog.get_logger()/g
|
||||
s/logging\.getLogger()/structlog.get_logger()/g
|
||||
' "$file"
|
||||
|
||||
# Remove temp file
|
||||
rm -f "${file}.tmp"
|
||||
fi
|
||||
}
|
||||
|
||||
# Fix service main.py files
|
||||
echo "🔄 Fixing service main.py files..."
|
||||
for service in auth training forecasting data tenant notification; do
|
||||
service_main="services/$service/app/main.py"
|
||||
if [ -f "$service_main" ]; then
|
||||
fix_logging_in_file "$service_main"
|
||||
fi
|
||||
done
|
||||
|
||||
# Fix gateway main.py
|
||||
echo "🔄 Fixing gateway main.py..."
|
||||
fix_logging_in_file "gateway/app/main.py"
|
||||
|
||||
# Fix other service files that might use logging
|
||||
echo "🔄 Fixing other service files..."
|
||||
find services/*/app -name "*.py" -type f | while read file; do
|
||||
# Skip __init__.py and files that should use standard logging
|
||||
if [[ "$file" != *"__init__.py" ]] && [[ "$file" != *"core/config.py" ]]; then
|
||||
# Check if file contains logging imports (but not shared/monitoring)
|
||||
if grep -q "import logging" "$file" && [[ "$file" != *"shared/monitoring"* ]]; then
|
||||
fix_logging_in_file "$file"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
# ================================================================
|
||||
# VERIFICATION SCRIPT
|
||||
# ================================================================
|
||||
|
||||
echo ""
|
||||
echo "🧪 VERIFICATION:"
|
||||
echo "================"
|
||||
|
||||
# Check for remaining incorrect usage
|
||||
echo "Checking for remaining 'import logging' in service files..."
|
||||
incorrect_files=$(find services gateway -name "*.py" -exec grep -l "import logging" {} \; | grep -v __pycache__ | grep -v migrations || true)
|
||||
|
||||
if [ -n "$incorrect_files" ]; then
|
||||
echo "⚠️ Still found 'import logging' in:"
|
||||
echo "$incorrect_files"
|
||||
else
|
||||
echo "✅ No incorrect 'import logging' found in service files"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Checking for correct 'import structlog' usage..."
|
||||
correct_files=$(find services gateway -name "*.py" -exec grep -l "import structlog" {} \; | grep -v __pycache__ || true)
|
||||
|
||||
if [ -n "$correct_files" ]; then
|
||||
echo "✅ Found correct 'import structlog' in:"
|
||||
echo "$correct_files"
|
||||
else
|
||||
echo "⚠️ No 'import structlog' found - this might be an issue"
|
||||
fi
|
||||
|
||||
# ================================================================
|
||||
# UPDATED FILE EXAMPLES
|
||||
# ================================================================
|
||||
|
||||
echo ""
|
||||
echo "📝 UPDATED FILE EXAMPLES:"
|
||||
echo "========================="
|
||||
echo ""
|
||||
|
||||
# Example 1: Service main.py
|
||||
cat << 'EOF'
|
||||
# ✅ CORRECT: services/auth/app/main.py (AFTER FIX)
|
||||
"""
|
||||
Authentication Service
|
||||
"""
|
||||
|
||||
import structlog # ✅ CORRECT
|
||||
from fastapi import FastAPI
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
|
||||
from app.core.config import settings
|
||||
from app.core.database import database_manager
|
||||
from shared.monitoring.logging import setup_logging
|
||||
|
||||
# Setup logging infrastructure
|
||||
setup_logging("auth-service", "INFO")
|
||||
|
||||
# Get structlog logger
|
||||
logger = structlog.get_logger() # ✅ CORRECT
|
||||
|
||||
app = FastAPI(title="Authentication Service")
|
||||
|
||||
@app.on_event("startup")
|
||||
async def startup_event():
|
||||
"""Application startup"""
|
||||
logger.info("Starting Authentication Service", service="auth-service") # ✅ STRUCTURED
|
||||
await database_manager.create_tables()
|
||||
logger.info("Authentication Service started successfully", service="auth-service")
|
||||
EOF
|
||||
|
||||
echo ""
|
||||
|
||||
# Example 2: Service business logic
|
||||
cat << 'EOF'
|
||||
# ✅ CORRECT: services/auth/app/services/auth_service.py (AFTER FIX)
|
||||
"""
|
||||
Authentication service business logic
|
||||
"""
|
||||
|
||||
import structlog # ✅ CORRECT
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
logger = structlog.get_logger() # ✅ CORRECT
|
||||
|
||||
class AuthService:
|
||||
@staticmethod
|
||||
async def login_user(email: str, password: str):
|
||||
"""Login user with structured logging"""
|
||||
|
||||
# ✅ STRUCTURED LOGGING with key-value pairs
|
||||
logger.info("User login attempt", email=email, service="auth-service")
|
||||
|
||||
try:
|
||||
user = await authenticate_user(email, password)
|
||||
|
||||
# ✅ SUCCESS with context
|
||||
logger.info(
|
||||
"User login successful",
|
||||
user_id=str(user.id),
|
||||
email=user.email,
|
||||
service="auth-service"
|
||||
)
|
||||
|
||||
return user
|
||||
|
||||
except AuthenticationError as e:
|
||||
# ✅ ERROR with context
|
||||
logger.error(
|
||||
"User login failed",
|
||||
email=email,
|
||||
error=str(e),
|
||||
error_type="authentication_error",
|
||||
service="auth-service"
|
||||
)
|
||||
raise
|
||||
EOF
|
||||
|
||||
echo ""
|
||||
|
||||
# Example 3: What should NOT change
|
||||
cat << 'EOF'
|
||||
# ✅ CORRECT: shared/monitoring/logging.py (NO CHANGE NEEDED)
|
||||
"""
|
||||
Centralized logging configuration - This file is CORRECT as-is
|
||||
"""
|
||||
|
||||
import logging # ✅ CORRECT - This configures the logging system
|
||||
import logging.config
|
||||
|
||||
def setup_logging(service_name: str, log_level: str = "INFO"):
|
||||
"""
|
||||
This function SETS UP the logging infrastructure.
|
||||
It should use 'import logging' to configure the system.
|
||||
"""
|
||||
config = {
|
||||
"version": 1,
|
||||
"disable_existing_loggers": False,
|
||||
# ... rest of configuration
|
||||
}
|
||||
|
||||
logging.config.dictConfig(config) # ✅ CORRECT
|
||||
logger = logging.getLogger(__name__) # ✅ CORRECT for setup
|
||||
logger.info(f"Logging configured for {service_name}")
|
||||
EOF
|
||||
|
||||
# ================================================================
|
||||
# TESTING THE FIX
|
||||
# ================================================================
|
||||
|
||||
echo ""
|
||||
echo "🧪 TESTING THE FIX:"
|
||||
echo "==================="
|
||||
echo ""
|
||||
|
||||
# Create test script
|
||||
cat << 'EOF' > test_logging_fix.py
|
||||
#!/usr/bin/env python3
|
||||
"""Test script to verify logging fix"""
|
||||
|
||||
def test_auth_service_logging():
|
||||
"""Test that auth service uses structlog correctly"""
|
||||
try:
|
||||
# This should work after the fix
|
||||
import structlog
|
||||
logger = structlog.get_logger()
|
||||
logger.info("Test log entry", service="test", test=True)
|
||||
print("✅ Auth service logging test passed")
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"❌ Auth service logging test failed: {e}")
|
||||
return False
|
||||
|
||||
def test_shared_logging_setup():
|
||||
"""Test that shared logging setup still works"""
|
||||
try:
|
||||
from shared.monitoring.logging import setup_logging
|
||||
setup_logging("test-service", "INFO")
|
||||
print("✅ Shared logging setup test passed")
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"❌ Shared logging setup test failed: {e}")
|
||||
return False
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("Testing logging configuration...")
|
||||
|
||||
test1 = test_shared_logging_setup()
|
||||
test2 = test_auth_service_logging()
|
||||
|
||||
if test1 and test2:
|
||||
print("\n🎉 All logging tests passed!")
|
||||
else:
|
||||
print("\n⚠️ Some logging tests failed")
|
||||
EOF
|
||||
|
||||
echo "Created test script: test_logging_fix.py"
|
||||
echo ""
|
||||
echo "Run the test with:"
|
||||
echo " python test_logging_fix.py"
|
||||
|
||||
# ================================================================
|
||||
# SUMMARY
|
||||
# ================================================================
|
||||
|
||||
echo ""
|
||||
echo "📊 SUMMARY:"
|
||||
echo "==========="
|
||||
echo ""
|
||||
echo "✅ Fixed inconsistent logging imports throughout the project"
|
||||
echo "✅ Services now use 'import structlog' consistently"
|
||||
echo "✅ shared/monitoring/logging.py still uses 'import logging' (correct)"
|
||||
echo "✅ All service logs will now be structured JSON"
|
||||
echo "✅ Logs will be properly aggregated in ELK stack"
|
||||
echo ""
|
||||
echo "🔍 To verify the fix:"
|
||||
echo " 1. Run: docker-compose restart"
|
||||
echo " 2. Check logs: docker-compose logs -f auth-service"
|
||||
echo " 3. Look for structured JSON log entries"
|
||||
echo ""
|
||||
echo "📁 Backups saved to: $backup_dir"
|
||||
echo ""
|
||||
echo "🚀 Your logging is now consistent across all services!"
|
||||
Reference in New Issue
Block a user