29d19087f1
- Updated all OpenTelemetry packages to latest versions: - opentelemetry-api: 1.27.0 → 1.39.1 - opentelemetry-sdk: 1.27.0 → 1.39.1 - opentelemetry-exporter-otlp-proto-grpc: 1.27.0 → 1.39.1 - opentelemetry-exporter-otlp-proto-http: 1.27.0 → 1.39.1 - opentelemetry-instrumentation-fastapi: 0.48b0 → 0.60b1 - opentelemetry-instrumentation-httpx: 0.48b0 → 0.60b1 - opentelemetry-instrumentation-redis: 0.48b0 → 0.60b1 - opentelemetry-instrumentation-sqlalchemy: 0.48b0 → 0.60b1 - Removed prometheus-client==0.23.1 from all services - Unified all services to use the same monitoring package versions Generated by Mistral Vibe. Co-Authored-By: Mistral Vibe <vibe@mistral.ai>
134 lines
4.4 KiB
Bash
Executable File
134 lines
4.4 KiB
Bash
Executable File
#!/bin/bash
|
|
# Setup script for database monitoring with OpenTelemetry and SigNoz
|
|
# This script creates monitoring users in PostgreSQL and deploys the collector
|
|
|
|
set -e
|
|
|
|
echo "========================================="
|
|
echo "Database Monitoring Setup for SigNoz"
|
|
echo "========================================="
|
|
echo ""
|
|
|
|
# Configuration
|
|
NAMESPACE="bakery-ia"
|
|
MONITOR_USER="otel_monitor"
|
|
MONITOR_PASSWORD=$(openssl rand -base64 32)
|
|
|
|
# PostgreSQL databases to monitor
|
|
DATABASES=(
|
|
"auth-db-service:auth_db"
|
|
"inventory-db-service:inventory_db"
|
|
"orders-db-service:orders_db"
|
|
"tenant-db-service:tenant_db"
|
|
"sales-db-service:sales_db"
|
|
"production-db-service:production_db"
|
|
"recipes-db-service:recipes_db"
|
|
"procurement-db-service:procurement_db"
|
|
"distribution-db-service:distribution_db"
|
|
"forecasting-db-service:forecasting_db"
|
|
"external-db-service:external_db"
|
|
"suppliers-db-service:suppliers_db"
|
|
"pos-db-service:pos_db"
|
|
"training-db-service:training_db"
|
|
"notification-db-service:notification_db"
|
|
"orchestrator-db-service:orchestrator_db"
|
|
"ai-insights-db-service:ai_insights_db"
|
|
)
|
|
|
|
echo "Step 1: Creating monitoring user in PostgreSQL databases"
|
|
echo "========================================="
|
|
echo ""
|
|
|
|
for db_entry in "${DATABASES[@]}"; do
|
|
IFS=':' read -r service dbname <<< "$db_entry"
|
|
|
|
echo "Creating monitoring user in $dbname..."
|
|
|
|
# Create monitoring user via kubectl exec
|
|
kubectl exec -n "$NAMESPACE" "deployment/${service%-service}" -- psql -U postgres -d "$dbname" -c "
|
|
DO \$\$
|
|
BEGIN
|
|
IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = '$MONITOR_USER') THEN
|
|
CREATE USER $MONITOR_USER WITH PASSWORD '$MONITOR_PASSWORD';
|
|
GRANT pg_monitor TO $MONITOR_USER;
|
|
GRANT CONNECT ON DATABASE $dbname TO $MONITOR_USER;
|
|
RAISE NOTICE 'User $MONITOR_USER created successfully';
|
|
ELSE
|
|
RAISE NOTICE 'User $MONITOR_USER already exists';
|
|
END IF;
|
|
END
|
|
\$\$;
|
|
" 2>/dev/null || echo " ⚠️ Warning: Could not create user in $dbname (may already exist or database not ready)"
|
|
|
|
echo ""
|
|
done
|
|
|
|
echo "✅ Monitoring users created"
|
|
echo ""
|
|
|
|
echo "Step 2: Creating Kubernetes secret for monitoring credentials"
|
|
echo "========================================="
|
|
echo ""
|
|
|
|
# Create secret for database monitoring
|
|
kubectl create secret generic database-monitor-secrets \
|
|
-n "$NAMESPACE" \
|
|
--from-literal=POSTGRES_MONITOR_USER="$MONITOR_USER" \
|
|
--from-literal=POSTGRES_MONITOR_PASSWORD="$MONITOR_PASSWORD" \
|
|
--dry-run=client -o yaml | kubectl apply -f -
|
|
|
|
echo "✅ Secret created: database-monitor-secrets"
|
|
echo ""
|
|
|
|
echo "Step 3: Deploying OpenTelemetry collector for database monitoring"
|
|
echo "========================================="
|
|
echo ""
|
|
|
|
kubectl apply -f infrastructure/kubernetes/base/monitoring/database-otel-collector.yaml
|
|
|
|
echo "✅ Database monitoring collector deployed"
|
|
echo ""
|
|
|
|
echo "Step 4: Waiting for collector to be ready"
|
|
echo "========================================="
|
|
echo ""
|
|
|
|
kubectl wait --for=condition=available --timeout=60s \
|
|
deployment/database-otel-collector -n "$NAMESPACE"
|
|
|
|
echo "✅ Collector is ready"
|
|
echo ""
|
|
|
|
echo "========================================="
|
|
echo "Database Monitoring Setup Complete!"
|
|
echo "========================================="
|
|
echo ""
|
|
echo "What's been configured:"
|
|
echo " ✅ Monitoring user created in all PostgreSQL databases"
|
|
echo " ✅ OpenTelemetry collector deployed for database metrics"
|
|
echo " ✅ Metrics exported to SigNoz"
|
|
echo ""
|
|
echo "Metrics being collected:"
|
|
echo " 📊 PostgreSQL: connections, commits, rollbacks, deadlocks, table sizes"
|
|
echo " 📊 Redis: memory usage, keyspace hits/misses, connected clients"
|
|
echo " 📊 RabbitMQ: queue depth, message rates, consumer count"
|
|
echo ""
|
|
echo "Next steps:"
|
|
echo " 1. Check collector logs:"
|
|
echo " kubectl logs -n $NAMESPACE deployment/database-otel-collector"
|
|
echo ""
|
|
echo " 2. View metrics in SigNoz:"
|
|
echo " - Go to https://monitoring.bakery-ia.local"
|
|
echo " - Create dashboard with queries like:"
|
|
echo " * postgresql.backends (connections)"
|
|
echo " * postgresql.database.size (database size)"
|
|
echo " * redis.memory.used (Redis memory)"
|
|
echo " * rabbitmq.message.current (queue depth)"
|
|
echo ""
|
|
echo " 3. Create alerts for:"
|
|
echo " - High connection count (approaching max_connections)"
|
|
echo " - Slow query detection (via application traces)"
|
|
echo " - High Redis memory usage"
|
|
echo " - RabbitMQ queue buildup"
|
|
echo ""
|