#!/bin/bash # ============================================================================ # SigNoz Verification Script for Bakery IA # ============================================================================ # This script verifies that SigNoz is properly deployed and functioning # ============================================================================ set -e # Color codes for output RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' NC='\033[0m' # No Color # Function to display help show_help() { echo "Usage: $0 [OPTIONS] ENVIRONMENT" echo "" echo "Verify SigNoz deployment for Bakery IA" echo "" echo "Arguments: ENVIRONMENT Environment to verify (dev|prod)" echo "" echo "Options: -h, --help Show this help message -n, --namespace NAMESPACE Specify namespace (default: signoz)" echo "" echo "Examples: $0 dev # Verify development deployment $0 prod # Verify production deployment $0 --namespace monitoring dev # Verify with custom namespace" } # Parse command line arguments NAMESPACE="signoz" while [[ $# -gt 0 ]]; do case $1 in -h|--help) show_help exit 0 ;; -n|--namespace) NAMESPACE="$2" shift 2 ;; dev|prod) ENVIRONMENT="$1" shift ;; *) echo "Unknown argument: $1" show_help exit 1 ;; esac done # Validate environment if [[ -z "$ENVIRONMENT" ]]; then echo "Error: Environment not specified. Use 'dev' or 'prod'." show_help exit 1 fi if [[ "$ENVIRONMENT" != "dev" && "$ENVIRONMENT" != "prod" ]]; then echo "Error: Invalid environment. Use 'dev' or 'prod'." exit 1 fi # Function to check if kubectl is configured check_kubectl() { if ! kubectl cluster-info &> /dev/null; then echo "${RED}Error: kubectl is not configured or cannot connect to cluster.${NC}" echo "Please ensure you have access to a Kubernetes cluster." exit 1 fi } # Function to check namespace exists check_namespace() { if ! kubectl get namespace "$NAMESPACE" &> /dev/null; then echo "${RED}Error: Namespace $NAMESPACE does not exist.${NC}" echo "Please deploy SigNoz first using: ./deploy-signoz.sh $ENVIRONMENT" exit 1 fi } # Function to verify SigNoz deployment verify_deployment() { echo "${BLUE}" echo "==========================================" echo "🔍 Verifying SigNoz Deployment" echo "==========================================" echo "Environment: $ENVIRONMENT" echo "Namespace: $NAMESPACE" echo "${NC}" echo "" # Check if SigNoz helm release exists echo "${BLUE}1. Checking Helm release...${NC}" if helm list -n "$NAMESPACE" | grep -q signoz; then echo "${GREEN}✅ SigNoz Helm release found${NC}" else echo "${RED}❌ SigNoz Helm release not found${NC}" echo "Please deploy SigNoz first using: ./deploy-signoz.sh $ENVIRONMENT" exit 1 fi echo "" # Check pod status echo "${BLUE}2. Checking pod status...${NC}" local total_pods=$(kubectl get pods -n "$NAMESPACE" -l app.kubernetes.io/instance=signoz 2>/dev/null | grep -v "NAME" | wc -l | tr -d ' ' || echo "0") local running_pods=$(kubectl get pods -n "$NAMESPACE" -l app.kubernetes.io/instance=signoz --field-selector=status.phase=Running 2>/dev/null | grep -c "Running" || echo "0") local ready_pods=$(kubectl get pods -n "$NAMESPACE" -l app.kubernetes.io/instance=signoz 2>/dev/null | grep "Running" | grep "1/1" | wc -l | tr -d ' ' || echo "0") echo "Total pods: $total_pods" echo "Running pods: $running_pods" echo "Ready pods: $ready_pods" if [[ $total_pods -eq 0 ]]; then echo "${RED}❌ No SigNoz pods found${NC}" exit 1 fi if [[ $running_pods -eq $total_pods ]]; then echo "${GREEN}✅ All pods are running${NC}" else echo "${YELLOW}⚠️ Some pods are not running${NC}" fi if [[ $ready_pods -eq $total_pods ]]; then echo "${GREEN}✅ All pods are ready${NC}" else echo "${YELLOW}⚠️ Some pods are not ready${NC}" fi echo "" # Show pod details echo "${BLUE}Pod Details:${NC}" kubectl get pods -n "$NAMESPACE" -l app.kubernetes.io/instance=signoz echo "" # Check services echo "${BLUE}3. Checking services...${NC}" local service_count=$(kubectl get svc -n "$NAMESPACE" -l app.kubernetes.io/instance=signoz 2>/dev/null | grep -v "NAME" | wc -l | tr -d ' ' || echo "0") if [[ $service_count -gt 0 ]]; then echo "${GREEN}✅ Services found ($service_count services)${NC}" kubectl get svc -n "$NAMESPACE" -l app.kubernetes.io/instance=signoz else echo "${RED}❌ No services found${NC}" fi echo "" # Check ingress echo "${BLUE}4. Checking ingress...${NC}" local ingress_count=$(kubectl get ingress -n "$NAMESPACE" -l app.kubernetes.io/instance=signoz 2>/dev/null | grep -v "NAME" | wc -l | tr -d ' ' || echo "0") if [[ $ingress_count -gt 0 ]]; then echo "${GREEN}✅ Ingress found ($ingress_count ingress resources)${NC}" kubectl get ingress -n "$NAMESPACE" -l app.kubernetes.io/instance=signoz else echo "${YELLOW}⚠️ No ingress found (may be configured in main namespace)${NC}" fi echo "" # Check PVCs echo "${BLUE}5. Checking persistent volume claims...${NC}" local pvc_count=$(kubectl get pvc -n "$NAMESPACE" -l app.kubernetes.io/instance=signoz 2>/dev/null | grep -v "NAME" | wc -l | tr -d ' ' || echo "0") if [[ $pvc_count -gt 0 ]]; then echo "${GREEN}✅ PVCs found ($pvc_count PVCs)${NC}" kubectl get pvc -n "$NAMESPACE" -l app.kubernetes.io/instance=signoz else echo "${YELLOW}⚠️ No PVCs found (may not be required for all components)${NC}" fi echo "" # Check resource usage echo "${BLUE}6. Checking resource usage...${NC}" if command -v kubectl &> /dev/null && kubectl top pods -n "$NAMESPACE" &> /dev/null; then echo "${GREEN}✅ Resource usage:${NC}" kubectl top pods -n "$NAMESPACE" -l app.kubernetes.io/instance=signoz else echo "${YELLOW}⚠️ Metrics server not available or no resource usage data${NC}" fi echo "" # Check logs for errors echo "${BLUE}7. Checking for errors in logs...${NC}" local error_found=false # Check each pod for errors while IFS= read -r pod; do if [[ -n "$pod" ]]; then local pod_errors=$(kubectl logs -n "$NAMESPACE" "$pod" 2>/dev/null | grep -i "error\|exception\|fail\|crash" | wc -l || echo "0") if [[ $pod_errors -gt 0 ]]; then echo "${RED}❌ Errors found in pod $pod ($pod_errors errors)${NC}" error_found=true fi fi done < <(kubectl get pods -n "$NAMESPACE" -l app.kubernetes.io/instance=signoz -o name | sed 's|pod/||') if [[ "$error_found" == false ]]; then echo "${GREEN}✅ No errors found in logs${NC}" fi echo "" # Environment-specific checks if [[ "$ENVIRONMENT" == "dev" ]]; then verify_dev_specific else verify_prod_specific fi # Show access information show_access_info } # Function for development-specific verification verify_dev_specific() { echo "${BLUE}8. Development-specific checks...${NC}" # Check if localhost ingress is configured if kubectl get ingress -n "$NAMESPACE" | grep -q "localhost"; then echo "${GREEN}✅ Localhost ingress configured${NC}" else echo "${YELLOW}⚠️ Localhost ingress not found${NC}" fi # Check resource limits (should be lower for dev) local query_service=$(kubectl get deployment -n "$NAMESPACE" signoz-query-service -o jsonpath='{.spec.template.spec.containers[0].resources.limits.memory}' 2>/dev/null || echo "") if [[ -n "$query_service" && "$query_service" == "512Mi" ]]; then echo "${GREEN}✅ Development resource limits applied${NC}" else echo "${YELLOW}⚠️ Resource limits may not be optimized for development${NC}" fi echo "" } # Function for production-specific verification verify_prod_specific() { echo "${BLUE}8. Production-specific checks...${NC}" # Check if TLS is configured if kubectl get ingress -n "$NAMESPACE" | grep -q "signoz-tls-cert"; then echo "${GREEN}✅ TLS certificate configured${NC}" else echo "${YELLOW}⚠️ TLS certificate not found${NC}" fi # Check if multiple replicas are running local query_replicas=$(kubectl get deployment -n "$NAMESPACE" signoz-query-service -o jsonpath='{.spec.replicas}' 2>/dev/null || echo "1") if [[ $query_replicas -gt 1 ]]; then echo "${GREEN}✅ High availability configured ($query_replicas replicas)${NC}" else echo "${YELLOW}⚠️ Single replica detected (not highly available)${NC}" fi # Check resource limits (should be higher for prod) local query_service=$(kubectl get deployment -n "$NAMESPACE" signoz-query-service -o jsonpath='{.spec.template.spec.containers[0].resources.limits.memory}' 2>/dev/null || echo "") if [[ -n "$query_service" && "$query_service" == "2Gi" ]]; then echo "${GREEN}✅ Production resource limits applied${NC}" else echo "${YELLOW}⚠️ Resource limits may not be optimized for production${NC}" fi echo "" } # Function to show access information show_access_info() { echo "${BLUE}" echo "==========================================" echo "📋 Access Information" echo "==========================================" echo "${NC}" if [[ "$ENVIRONMENT" == "dev" ]]; then echo "SigNoz UI: https://localhost/signoz" echo "SigNoz API: https://localhost/signoz-api" echo "" echo "OpenTelemetry Collector:" echo " gRPC: localhost:4317" echo " HTTP: localhost:4318" echo " Metrics: localhost:8888" else echo "SigNoz UI: https://monitoring.bakewise.ai/signoz" echo "SigNoz API: https://monitoring.bakewise.ai/signoz-api" echo "SigNoz Alerts: https://monitoring.bakewise.ai/signoz-alerts" echo "" echo "OpenTelemetry Collector:" echo " gRPC: monitoring.bakewise.ai:4317" echo " HTTP: monitoring.bakewise.ai:4318" fi echo "" echo "Default Credentials:" echo " Username: admin" echo " Password: admin" echo "" # Show connection test commands echo "Connection Test Commands:" if [[ "$ENVIRONMENT" == "dev" ]]; then echo " curl -k https://localhost/signoz" echo " curl -k https://localhost/signoz-api/health" else echo " curl https://monitoring.bakewise.ai/signoz" echo " curl https://monitoring.bakewise.ai/signoz-api/health" fi echo "" } # Function to run connectivity tests run_connectivity_tests() { echo "${BLUE}" echo "==========================================" echo "🔗 Running Connectivity Tests" echo "==========================================" echo "${NC}" if [[ "$ENVIRONMENT" == "dev" ]]; then # Test frontend echo "Testing SigNoz frontend..." if curl -k -s -o /dev/null -w "%{http_code}" https://localhost/signoz | grep -q "200\|302"; then echo "${GREEN}✅ Frontend accessible${NC}" else echo "${RED}❌ Frontend not accessible${NC}" fi # Test API echo "Testing SigNoz API..." if curl -k -s -o /dev/null -w "%{http_code}" https://localhost/signoz-api/health | grep -q "200"; then echo "${GREEN}✅ API accessible${NC}" else echo "${RED}❌ API not accessible${NC}" fi # Test OTEL collector echo "Testing OpenTelemetry collector..." if curl -s -o /dev/null -w "%{http_code}" http://localhost:8888/metrics | grep -q "200"; then echo "${GREEN}✅ OTEL collector accessible${NC}" else echo "${YELLOW}⚠️ OTEL collector not accessible (may not be exposed)${NC}" fi else echo "${YELLOW}⚠️ Production connectivity tests require valid DNS and TLS${NC}" echo " Please ensure monitoring.bakewise.ai resolves to your cluster" fi echo "" } # Main execution main() { echo "${BLUE}" echo "==========================================" echo "🔍 SigNoz Verification for Bakery IA" echo "==========================================" echo "${NC}" # Check prerequisites check_kubectl check_namespace # Verify deployment verify_deployment # Run connectivity tests run_connectivity_tests echo "${GREEN}" echo "==========================================" echo "✅ Verification Complete" echo "==========================================" echo "${NC}" echo "Summary:" echo " Environment: $ENVIRONMENT" echo " Namespace: $NAMESPACE" echo "" echo "Next Steps:" echo " 1. Access SigNoz UI and verify dashboards" echo " 2. Configure alert rules for your services" echo " 3. Instrument your applications with OpenTelemetry" echo " 4. Set up custom dashboards for key metrics" echo "" } # Run main function main