81 lines
4.1 KiB
Bash
81 lines
4.1 KiB
Bash
|
#!/bin/bash
|
||
|
|
# Usage: ./check_alert_flow.sh <TENANT_ID>
|
||
|
|
# Checks the complete alert flow for a demo session in production
|
||
|
|
|
||
|
|
# Use microk8s kubectl directly to avoid wrapper issues
|
||
|
|
KUBECTL="microk8s kubectl"
|
||
|
|
|
||
|
|
TENANT_ID="${1:-your-default-tenant-id}"
|
||
|
|
|
||
|
|
# Try different secret names for Redis password
|
||
|
|
REDIS_PASSWORD=$($KUBECTL get secret redis-credentials -n bakery-ia -o jsonpath='{.data.REDIS_PASSWORD}' 2>/dev/null | base64 -d 2>/dev/null)
|
||
|
|
if [ -z "$REDIS_PASSWORD" ]; then
|
||
|
|
REDIS_PASSWORD=$($KUBECTL get secret redis-secret -n bakery-ia -o jsonpath='{.data.REDIS_PASSWORD}' 2>/dev/null | base64 -d 2>/dev/null)
|
||
|
|
fi
|
||
|
|
if [ -z "$REDIS_PASSWORD" ]; then
|
||
|
|
# Try to get from configmap or use default
|
||
|
|
REDIS_PASSWORD="redis_pass123"
|
||
|
|
fi
|
||
|
|
|
||
|
|
echo "=========================================="
|
||
|
|
echo "Alert Flow Health Check for Tenant: $TENANT_ID"
|
||
|
|
echo "=========================================="
|
||
|
|
|
||
|
|
echo -e "\n=== 1. RabbitMQ Queues ==="
|
||
|
|
$KUBECTL exec -n bakery-ia deployment/rabbitmq -- \
|
||
|
|
rabbitmqctl list_queues name messages consumers 2>/dev/null | grep -E "alert|event" || echo " No alert/event queues found or RabbitMQ not accessible"
|
||
|
|
|
||
|
|
echo -e "\n=== 2. Alert Processor DB Events (last 10) ==="
|
||
|
|
$KUBECTL exec -n bakery-ia deployment/alert-processor-db -- \
|
||
|
|
psql -U alert_processor_user -d alert_processor_db -c \
|
||
|
|
"SELECT event_type, priority_score, type_class, status, created_at
|
||
|
|
FROM events WHERE tenant_id = '$TENANT_ID'
|
||
|
|
ORDER BY created_at DESC LIMIT 10;" 2>/dev/null || echo " Could not query alert-processor-db"
|
||
|
|
|
||
|
|
echo -e "\n=== 3. Redis SSE Channels ==="
|
||
|
|
$KUBECTL exec -n bakery-ia deployment/redis -- \
|
||
|
|
redis-cli -a "$REDIS_PASSWORD" --no-auth-warning PUBSUB CHANNELS "*" 2>/dev/null | grep -i "$TENANT_ID" || echo " No active SSE channels for this tenant (channels only exist when frontend is connected)"
|
||
|
|
|
||
|
|
echo -e "\n=== 4. Demo Session Status ==="
|
||
|
|
$KUBECTL exec -n bakery-ia deployment/demo-session-db -- \
|
||
|
|
psql -U demo_session_user -d demo_session_db -c \
|
||
|
|
"SELECT id, status, virtual_tenant_id, demo_account_type, data_cloned, created_at, cloning_completed_at
|
||
|
|
FROM demo_sessions
|
||
|
|
WHERE virtual_tenant_id::text = '$TENANT_ID' OR id::text = '$TENANT_ID'
|
||
|
|
ORDER BY created_at DESC LIMIT 1;" 2>/dev/null || echo " Could not query demo-session-db"
|
||
|
|
|
||
|
|
echo -e "\n=== 5. Recent Alert Processor Logs ==="
|
||
|
|
$KUBECTL logs -n bakery-ia deployment/alert-processor --tail=100 2>/dev/null | \
|
||
|
|
grep -iE "received|enriched|stored|error|consuming|rabbitmq|sse" | tail -15 || echo " No relevant logs found"
|
||
|
|
|
||
|
|
echo -e "\n=== 6. Gateway SSE Logs ==="
|
||
|
|
$KUBECTL logs -n bakery-ia deployment/gateway --tail=100 2>/dev/null | \
|
||
|
|
grep -iE "sse|pubsub|events_stream" | tail -10 || echo " No SSE logs found"
|
||
|
|
|
||
|
|
echo -e "\n=== 7. Service Health ==="
|
||
|
|
# Check deployment status directly
|
||
|
|
for deploy in alert-processor gateway demo-session-service orchestrator-service inventory-service production-service procurement-service rabbitmq redis; do
|
||
|
|
READY=$($KUBECTL get deployment/$deploy -n bakery-ia -o jsonpath='{.status.readyReplicas}' 2>/dev/null || echo "0")
|
||
|
|
DESIRED=$($KUBECTL get deployment/$deploy -n bakery-ia -o jsonpath='{.spec.replicas}' 2>/dev/null || echo "?")
|
||
|
|
if [ -z "$READY" ]; then READY="0"; fi
|
||
|
|
echo " $deploy: $READY/$DESIRED ready"
|
||
|
|
done
|
||
|
|
|
||
|
|
echo -e "\n=== 8. RabbitMQ Connection Status ==="
|
||
|
|
$KUBECTL exec -n bakery-ia deployment/rabbitmq -- \
|
||
|
|
rabbitmqctl list_connections name state 2>/dev/null | head -10 || echo " Could not list connections"
|
||
|
|
|
||
|
|
echo -e "\n=== 9. Recent Demo Session Service Logs (enrichment) ==="
|
||
|
|
$KUBECTL logs -n bakery-ia deployment/demo-session-service --tail=100 2>/dev/null | \
|
||
|
|
grep -iE "enrichment|alert|trigger|clone|post_clone" | tail -10 || echo " No relevant logs found"
|
||
|
|
|
||
|
|
echo -e "\n=== 10. Total Events in Alert Processor DB ==="
|
||
|
|
$KUBECTL exec -n bakery-ia deployment/alert-processor-db -- \
|
||
|
|
psql -U alert_processor_user -d alert_processor_db -c \
|
||
|
|
"SELECT COUNT(*) as total_events,
|
||
|
|
COUNT(*) FILTER (WHERE tenant_id = '$TENANT_ID') as tenant_events
|
||
|
|
FROM events;" 2>/dev/null || echo " Could not query"
|
||
|
|
|
||
|
|
echo -e "\n=========================================="
|
||
|
|
echo "Health Check Complete"
|
||
|
|
echo "=========================================="
|