From 23b8523b36f8ac01df5fd4d62937e34041968d72 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 2 Jan 2026 14:57:09 +0000 Subject: [PATCH] Add comprehensive Kubernetes migration guide from local to production This commit adds complete documentation and tooling for migrating from local development (Kind/Colima on macOS) to production deployment (MicroK8s on Ubuntu VPS at Clouding.io). Documentation added: - K8S-MIGRATION-GUIDE.md: Comprehensive step-by-step migration guide covering all phases from VPS setup to post-deployment operations - MIGRATION-CHECKLIST.md: Quick reference checklist for migration tasks - MIGRATION-SUMMARY.md: High-level overview and key changes summary Configuration updates: - Added storage-patch.yaml for MicroK8s storage class compatibility (changes from 'standard' to 'microk8s-hostpath') - Updated prod/kustomization.yaml to include storage patch Helper scripts: - deploy-production.sh: Interactive deployment script with validation - tag-and-push-images.sh: Automated image tagging and registry push - backup-databases.sh: Database backup script for production Key differences addressed: - Ingress: MicroK8s addon vs custom NGINX - Storage: MicroK8s hostpath vs Kind standard storage - Registry: Container registry configuration for production - SSL: Let's Encrypt production certificates - Domains: Real domain configuration vs localhost - Resources: Production-grade resource limits and scaling The migration guide covers: - VPS setup and MicroK8s installation - Configuration adaptations required - Container registry setup options - SSL certificate configuration - Monitoring and backup setup - Troubleshooting common issues - Security hardening checklist - Rollback procedures All existing Kubernetes manifests remain unchanged and compatible. --- docs/K8S-MIGRATION-GUIDE.md | 837 ++++++++++++++++++ docs/MIGRATION-CHECKLIST.md | 289 ++++++ docs/MIGRATION-SUMMARY.md | 275 ++++++ .../overlays/prod/kustomization.yaml | 3 + .../overlays/prod/storage-patch.yaml | 12 + scripts/backup-databases.sh | 161 ++++ scripts/deploy-production.sh | 190 ++++ scripts/tag-and-push-images.sh | 154 ++++ 8 files changed, 1921 insertions(+) create mode 100644 docs/K8S-MIGRATION-GUIDE.md create mode 100644 docs/MIGRATION-CHECKLIST.md create mode 100644 docs/MIGRATION-SUMMARY.md create mode 100644 infrastructure/kubernetes/overlays/prod/storage-patch.yaml create mode 100755 scripts/backup-databases.sh create mode 100755 scripts/deploy-production.sh create mode 100755 scripts/tag-and-push-images.sh diff --git a/docs/K8S-MIGRATION-GUIDE.md b/docs/K8S-MIGRATION-GUIDE.md new file mode 100644 index 00000000..497c15f6 --- /dev/null +++ b/docs/K8S-MIGRATION-GUIDE.md @@ -0,0 +1,837 @@ +# Kubernetes Migration Guide: Local Dev to Production (MicroK8s) + +## Overview + +This guide covers migrating the Bakery IA platform from local development environment to production on a Clouding.io VPS. + +**Current Setup (Local Development):** +- macOS with Colima +- Kind (Kubernetes in Docker) +- NGINX Ingress Controller +- Local storage +- Development domains (localhost, bakery-ia.local) + +**Target Setup (Production):** +- Ubuntu VPS (Clouding.io) +- MicroK8s +- MicroK8s NGINX Ingress +- Persistent storage +- Production domains (your actual domain) + +--- + +## Key Differences & Required Adaptations + +### 1. **Ingress Controller** +- **Local:** Custom NGINX installed via manifest +- **Production:** MicroK8s ingress addon +- **Action Required:** Enable MicroK8s ingress addon + +### 2. **Storage** +- **Local:** Kind uses `standard` storage class (hostPath) +- **Production:** MicroK8s uses `microk8s-hostpath` storage class +- **Action Required:** Update storage class in PVCs + +### 3. **Image Registry** +- **Local:** Images built locally, no push required +- **Production:** Need container registry (Docker Hub, GitHub Container Registry, or private registry) +- **Action Required:** Setup image registry and push images + +### 4. **Domain & SSL** +- **Local:** localhost with self-signed certs +- **Production:** Real domain with Let's Encrypt certificates +- **Action Required:** Configure DNS and update ingress + +### 5. **Resource Allocation** +- **Local:** Minimal resources (development mode) +- **Production:** Production-grade resources with HPA +- **Action Required:** Already configured in prod overlay + +### 6. **Build Process** +- **Local:** Skaffold with local build +- **Production:** CI/CD or manual build + push +- **Action Required:** Setup deployment pipeline + +--- + +## Pre-Migration Checklist + +### VPS Requirements +- [ ] Ubuntu 20.04 or later +- [ ] Minimum 8GB RAM (16GB+ recommended) +- [ ] Minimum 4 CPU cores (6+ recommended) +- [ ] 100GB+ disk space +- [ ] Public IP address +- [ ] Domain name configured + +### Access Requirements +- [ ] SSH access to VPS +- [ ] Domain DNS access +- [ ] Container registry credentials +- [ ] SSL certificate email address + +--- + +## Step-by-Step Migration Guide + +## Phase 1: VPS Setup + +### Step 1: Install MicroK8s on Ubuntu VPS + +```bash +# SSH into your VPS +ssh user@your-vps-ip + +# Update system +sudo apt update && sudo apt upgrade -y + +# Install MicroK8s +sudo snap install microk8s --classic --channel=1.28/stable + +# Add your user to microk8s group +sudo usermod -a -G microk8s $USER +sudo chown -f -R $USER ~/.kube + +# Restart session +newgrp microk8s + +# Verify installation +microk8s status --wait-ready + +# Enable required addons +microk8s enable dns +microk8s enable hostpath-storage +microk8s enable ingress +microk8s enable cert-manager +microk8s enable metrics-server +microk8s enable rbac + +# Optional but recommended +microk8s enable prometheus +microk8s enable registry # If you want local registry + +# Setup kubectl alias +echo "alias kubectl='microk8s kubectl'" >> ~/.bashrc +source ~/.bashrc + +# Verify +kubectl get nodes +kubectl get pods -A +``` + +### Step 2: Configure Firewall + +```bash +# Allow necessary ports +sudo ufw allow 22/tcp # SSH +sudo ufw allow 80/tcp # HTTP +sudo ufw allow 443/tcp # HTTPS +sudo ufw allow 16443/tcp # Kubernetes API (optional, for remote access) + +# Enable firewall +sudo ufw enable + +# Check status +sudo ufw status +``` + +--- + +## Phase 2: Configuration Adaptations + +### Step 3: Update Storage Class + +Create a production storage patch: + +```bash +# On your local machine +cat > infrastructure/kubernetes/overlays/prod/storage-patch.yaml < ~/.kube/config-merged +mv ~/.kube/config-merged ~/.kube/config + +# Deploy using skaffold +skaffold run -f skaffold-prod.yaml --kube-context=microk8s +``` + +### Step 10: Verify Deployment + +```bash +# Check all pods are running +kubectl get pods -n bakery-ia + +# Check services +kubectl get svc -n bakery-ia + +# Check ingress +kubectl get ingress -n bakery-ia + +# Check persistent volumes +kubectl get pvc -n bakery-ia + +# Check logs +kubectl logs -n bakery-ia deployment/gateway -f + +# Test database connectivity +kubectl exec -n bakery-ia deployment/auth-db -it -- psql -U postgres -c "\l" +``` + +--- + +## Phase 5: SSL Certificate Configuration + +### Step 11: Let's Encrypt SSL Certificates + +The cert-manager addon is already enabled. Configure production certificates: + +```bash +# Verify cert-manager is running +kubectl get pods -n cert-manager + +# Check cluster issuer +kubectl get clusterissuer + +# If letsencrypt-production issuer doesn't exist, create it: +cat < ~/backup-databases.sh <<'EOF' +#!/bin/bash +BACKUP_DIR="/backups/$(date +%Y-%m-%d)" +mkdir -p $BACKUP_DIR + +# Get all database pods +DBS=$(kubectl get pods -n bakery-ia -l app.kubernetes.io/component=database -o name) + +for db in $DBS; do + DB_NAME=$(echo $db | cut -d'/' -f2) + echo "Backing up $DB_NAME..." + + kubectl exec -n bakery-ia $db -- pg_dump -U postgres > "$BACKUP_DIR/${DB_NAME}.sql" +done + +# Compress backups +tar -czf "$BACKUP_DIR.tar.gz" "$BACKUP_DIR" +rm -rf "$BACKUP_DIR" + +# Keep only last 7 days +find /backups -name "*.tar.gz" -mtime +7 -delete + +echo "Backup completed: $BACKUP_DIR.tar.gz" +EOF + +chmod +x ~/backup-databases.sh + +# Setup daily cron job +(crontab -l 2>/dev/null; echo "0 2 * * * ~/backup-databases.sh") | crontab - +``` + +### Step 14: Setup Log Aggregation (Optional) + +```bash +# Enable Loki for log aggregation +microk8s enable observability + +# Or use external logging service like ELK, Datadog, etc. +``` + +--- + +## Phase 7: Post-Deployment Verification + +### Step 15: Health Checks + +```bash +# Test frontend +curl -k https://bakery.example.com + +# Test API +curl -k https://api.example.com/health + +# Test database connectivity +kubectl exec -n bakery-ia deployment/auth-service -- curl localhost:8000/health + +# Check all services are healthy +kubectl get pods -n bakery-ia -o wide + +# Check resource usage +kubectl top pods -n bakery-ia +kubectl top nodes +``` + +### Step 16: Performance Testing + +```bash +# Install hey (HTTP load testing tool) +go install github.com/rakyll/hey@latest + +# Test API endpoint +hey -n 1000 -c 10 https://api.example.com/health + +# Monitor during load test +kubectl top pods -n bakery-ia +``` + +--- + +## Ongoing Operations + +### Updating the Application + +```bash +# On local machine +# 1. Make code changes +# 2. Build and push new images +skaffold build -f skaffold-prod.yaml + +# 3. Update image tags in prod kustomization +# 4. Apply updates +kubectl apply -k infrastructure/kubernetes/overlays/prod + +# 5. Rolling update status +kubectl rollout status deployment/auth-service -n bakery-ia +``` + +### Scaling Services + +```bash +# Manual scaling +kubectl scale deployment auth-service -n bakery-ia --replicas=5 + +# Or update in kustomization.yaml and reapply +``` + +### Database Migrations + +```bash +# Run migration job +kubectl apply -f infrastructure/kubernetes/base/migrations/auth-migration-job.yaml + +# Check migration status +kubectl get jobs -n bakery-ia +kubectl logs -n bakery-ia job/auth-migration +``` + +--- + +## Troubleshooting Common Issues + +### Issue 1: Pods Not Starting + +```bash +# Check pod status +kubectl describe pod POD_NAME -n bakery-ia + +# Common causes: +# - Image pull errors: Check registry credentials +# - Resource limits: Check node resources +# - Volume mount issues: Check PVC status +``` + +### Issue 2: Ingress Not Working + +```bash +# Check ingress controller +kubectl get pods -n ingress + +# Check ingress resource +kubectl describe ingress bakery-ingress-prod -n bakery-ia + +# Check if port 80/443 are open +sudo netstat -tlnp | grep -E '(80|443)' + +# Check NGINX logs +kubectl logs -n ingress -l app.kubernetes.io/name=ingress-nginx +``` + +### Issue 3: SSL Certificate Issues + +```bash +# Check certificate status +kubectl describe certificate bakery-ia-prod-tls-cert -n bakery-ia + +# Check cert-manager logs +kubectl logs -n cert-manager deployment/cert-manager + +# Verify DNS +dig bakery.example.com + +# Manual certificate request +kubectl delete certificate bakery-ia-prod-tls-cert -n bakery-ia +kubectl apply -f infrastructure/kubernetes/overlays/prod/prod-ingress.yaml +``` + +### Issue 4: Database Connection Errors + +```bash +# Check database pod +kubectl get pods -n bakery-ia -l app.kubernetes.io/component=database + +# Check database logs +kubectl logs -n bakery-ia deployment/auth-db + +# Test connection from service pod +kubectl exec -n bakery-ia deployment/auth-service -- nc -zv auth-db 5432 +``` + +### Issue 5: Out of Resources + +```bash +# Check node resources +kubectl describe node + +# Check resource requests/limits +kubectl describe pod POD_NAME -n bakery-ia + +# Adjust resource limits in prod kustomization or scale down +``` + +--- + +## Security Hardening Checklist + +- [ ] Change all default passwords +- [ ] Enable pod security policies +- [ ] Setup network policies +- [ ] Enable audit logging +- [ ] Regular security updates +- [ ] Implement secrets rotation +- [ ] Setup intrusion detection +- [ ] Enable RBAC properly +- [ ] Regular backup testing +- [ ] Implement rate limiting +- [ ] Setup DDoS protection +- [ ] Enable security scanning + +--- + +## Performance Optimization + +### For VPS with Limited Resources + +If your VPS has limited resources, consider: + +```yaml +# Reduce replica counts in prod kustomization.yaml +replicas: + - name: auth-service + count: 2 # Instead of 3 + - name: gateway + count: 2 # Instead of 3 + +# Adjust resource limits +resources: + requests: + memory: "256Mi" # Reduced from 512Mi + cpu: "100m" # Reduced from 200m +``` + +### Database Optimization + +```bash +# Tune PostgreSQL for production +kubectl exec -n bakery-ia deployment/auth-db -it -- psql -U postgres + +# Inside PostgreSQL: +ALTER SYSTEM SET shared_buffers = '256MB'; +ALTER SYSTEM SET effective_cache_size = '1GB'; +ALTER SYSTEM SET maintenance_work_mem = '64MB'; +ALTER SYSTEM SET checkpoint_completion_target = '0.9'; +ALTER SYSTEM SET wal_buffers = '16MB'; +ALTER SYSTEM SET default_statistics_target = '100'; + +# Restart database pod +kubectl rollout restart deployment/auth-db -n bakery-ia +``` + +--- + +## Rollback Procedure + +If something goes wrong: + +```bash +# Rollback deployment +kubectl rollout undo deployment/DEPLOYMENT_NAME -n bakery-ia + +# Rollback to specific revision +kubectl rollout history deployment/DEPLOYMENT_NAME -n bakery-ia +kubectl rollout undo deployment/DEPLOYMENT_NAME --to-revision=2 -n bakery-ia + +# Restore from backup +tar -xzf /backups/2024-01-01.tar.gz +kubectl exec -n bakery-ia deployment/auth-db -- psql -U postgres < auth-db.sql +``` + +--- + +## Quick Reference + +### Useful Commands + +```bash +# View all resources +kubectl get all -n bakery-ia + +# Get pod logs +kubectl logs -f POD_NAME -n bakery-ia + +# Execute command in pod +kubectl exec -it POD_NAME -n bakery-ia -- /bin/bash + +# Port forward for debugging +kubectl port-forward svc/SERVICE_NAME 8000:8000 -n bakery-ia + +# Check events +kubectl get events -n bakery-ia --sort-by='.lastTimestamp' + +# Resource usage +kubectl top nodes +kubectl top pods -n bakery-ia + +# Restart deployment +kubectl rollout restart deployment/DEPLOYMENT_NAME -n bakery-ia + +# Scale deployment +kubectl scale deployment/DEPLOYMENT_NAME --replicas=3 -n bakery-ia +``` + +### Important File Locations on VPS + +``` +/var/snap/microk8s/current/credentials/ # Kubernetes credentials +/var/snap/microk8s/common/default-storage/ # Default storage location +~/kubernetes/ # Your manifests +/backups/ # Database backups +``` + +--- + +## Next Steps After Migration + +1. **Setup CI/CD Pipeline** + - GitHub Actions or GitLab CI + - Automated builds and deployments + - Automated testing + +2. **Implement Monitoring Dashboards** + - Setup Grafana dashboards + - Configure alerts + - Setup uptime monitoring + +3. **Disaster Recovery Plan** + - Document recovery procedures + - Test backup restoration + - Setup off-site backups + +4. **Cost Optimization** + - Monitor resource usage + - Right-size deployments + - Implement auto-scaling + +5. **Documentation** + - Document custom configurations + - Create runbooks for common tasks + - Train team members + +--- + +## Support and Resources + +- **MicroK8s Documentation:** https://microk8s.io/docs +- **Kubernetes Documentation:** https://kubernetes.io/docs +- **cert-manager Documentation:** https://cert-manager.io/docs +- **NGINX Ingress:** https://kubernetes.github.io/ingress-nginx + +## Conclusion + +This migration moves your application from a local development environment to a production-ready deployment. Remember to: + +- Test thoroughly before going live +- Have a rollback plan ready +- Monitor closely after deployment +- Keep regular backups +- Stay updated with security patches + +Good luck with your deployment! 🚀 diff --git a/docs/MIGRATION-CHECKLIST.md b/docs/MIGRATION-CHECKLIST.md new file mode 100644 index 00000000..e349f6b7 --- /dev/null +++ b/docs/MIGRATION-CHECKLIST.md @@ -0,0 +1,289 @@ +# Production Migration Quick Checklist + +This is a condensed checklist for migrating from local dev (Kind + Colima) to production (MicroK8s on Clouding.io VPS). + +## Pre-Migration (Do this BEFORE deployment) + +### 1. VPS Setup +- [ ] VPS provisioned (Ubuntu 20.04+, 8GB+ RAM, 4+ CPU cores, 100GB+ disk) +- [ ] SSH access configured +- [ ] Domain name registered +- [ ] DNS records configured (A records pointing to VPS IP) + +### 2. MicroK8s Installation +```bash +# Install MicroK8s +sudo snap install microk8s --classic --channel=1.28/stable +sudo usermod -a -G microk8s $USER +newgrp microk8s + +# Enable required addons +microk8s enable dns hostpath-storage ingress cert-manager metrics-server rbac + +# Setup kubectl alias +echo "alias kubectl='microk8s kubectl'" >> ~/.bashrc +source ~/.bashrc +``` + +### 3. Firewall Configuration +```bash +sudo ufw allow 22/tcp 80/tcp 443/tcp +sudo ufw enable +``` + +### 4. Configuration Updates + +#### Update Domain Names +Edit `infrastructure/kubernetes/overlays/prod/prod-ingress.yaml`: +- [ ] Replace `bakery.yourdomain.com` with your actual domain +- [ ] Replace `api.yourdomain.com` with your actual API domain +- [ ] Replace `monitoring.yourdomain.com` with your actual monitoring domain +- [ ] Update CORS origins with your domains +- [ ] Update cert-manager email address + +#### Update Production Secrets +Edit `infrastructure/kubernetes/base/secrets.yaml`: +- [ ] Generate strong passwords: `openssl rand -base64 32` +- [ ] Update all database passwords +- [ ] Update JWT secrets +- [ ] Update API keys +- [ ] **NEVER commit real secrets to git!** + +#### Configure Container Registry +Choose one option: + +**Option A: Docker Hub (Recommended)** +- [ ] Create Docker Hub account +- [ ] Login: `docker login` +- [ ] Update image names in `infrastructure/kubernetes/overlays/prod/kustomization.yaml` + +**Option B: MicroK8s Registry** +- [ ] Enable registry: `microk8s enable registry` +- [ ] Configure insecure registry in `/etc/docker/daemon.json` + +### 5. DNS Configuration +Point your domains to VPS IP: +``` +Type Host Value TTL +A bakery YOUR_VPS_IP 300 +A api YOUR_VPS_IP 300 +A monitoring YOUR_VPS_IP 300 +``` + +- [ ] DNS records configured +- [ ] Wait for DNS propagation (test with `nslookup bakery.yourdomain.com`) + +## Deployment Phase + +### 6. Build and Push Images + +**Using provided script:** +```bash +# Build all images +docker-compose build + +# Tag for your registry (Docker Hub example) +./scripts/tag-images.sh YOUR_DOCKERHUB_USERNAME + +# Push to registry +./scripts/push-images.sh YOUR_DOCKERHUB_USERNAME +``` + +**Manual:** +- [ ] Build all Docker images +- [ ] Tag with registry prefix +- [ ] Push to container registry + +### 7. Deploy to MicroK8s + +**Using provided script (on VPS):** +```bash +# Copy deployment script to VPS +scp scripts/deploy-production.sh user@YOUR_VPS_IP:~/ + +# SSH to VPS +ssh user@YOUR_VPS_IP + +# Clone your repository (or copy kubernetes manifests) +git clone YOUR_REPO_URL +cd bakery_ia + +# Run deployment script +./deploy-production.sh +``` + +**Manual deployment:** +```bash +# On VPS +kubectl apply -k infrastructure/kubernetes/overlays/prod +kubectl get pods -n bakery-ia -w +``` + +### 8. Verify Deployment + +- [ ] All pods running: `kubectl get pods -n bakery-ia` +- [ ] Services created: `kubectl get svc -n bakery-ia` +- [ ] Ingress configured: `kubectl get ingress -n bakery-ia` +- [ ] PVCs bound: `kubectl get pvc -n bakery-ia` +- [ ] Certificates issued: `kubectl get certificate -n bakery-ia` + +### 9. Test Application + +- [ ] Frontend accessible: `curl -k https://bakery.yourdomain.com` +- [ ] API responding: `curl -k https://api.yourdomain.com/health` +- [ ] SSL certificate valid (Let's Encrypt) +- [ ] Login functionality works +- [ ] Database connections working +- [ ] All microservices healthy + +### 10. Setup Monitoring & Backups + +**Monitoring:** +- [ ] Prometheus accessible +- [ ] Grafana accessible (if enabled) +- [ ] Set up alerts + +**Backups:** +```bash +# Copy backup script to VPS +scp scripts/backup-databases.sh user@YOUR_VPS_IP:~/ + +# Setup daily backups +crontab -e +# Add: 0 2 * * * ~/backup-databases.sh +``` + +- [ ] Backup script configured +- [ ] Test backup restoration +- [ ] Set up off-site backup storage + +## Post-Deployment + +### 11. Security Hardening +- [ ] Change all default passwords +- [ ] Review and update secrets regularly +- [ ] Enable pod security policies +- [ ] Configure network policies +- [ ] Set up monitoring and alerting +- [ ] Review firewall rules +- [ ] Enable audit logging + +### 12. Performance Tuning +- [ ] Monitor resource usage: `kubectl top pods -n bakery-ia` +- [ ] Adjust resource limits if needed +- [ ] Configure HPA (Horizontal Pod Autoscaling) +- [ ] Optimize database settings +- [ ] Set up CDN for frontend (optional) + +### 13. Documentation +- [ ] Document custom configurations +- [ ] Create runbooks for common operations +- [ ] Document recovery procedures +- [ ] Update team wiki/documentation + +## Key Differences from Local Dev + +| Aspect | Local (Kind) | Production (MicroK8s) | +|--------|--------------|----------------------| +| Ingress | Custom NGINX | MicroK8s ingress addon | +| Storage Class | `standard` | `microk8s-hostpath` | +| Image Pull | `Never` (local) | `Always` (from registry) | +| SSL Certs | Self-signed | Let's Encrypt | +| Domains | localhost | Real domains | +| Replicas | 1 per service | 2-3 per service | +| Resources | Minimal | Production-grade | +| Secrets | Dev secrets | Production secrets | + +## Troubleshooting Quick Reference + +### Pods Not Starting +```bash +kubectl describe pod POD_NAME -n bakery-ia +kubectl logs POD_NAME -n bakery-ia +``` + +### Ingress Not Working +```bash +kubectl describe ingress bakery-ingress-prod -n bakery-ia +kubectl logs -n ingress -l app.kubernetes.io/name=ingress-nginx +sudo netstat -tlnp | grep -E '(80|443)' +``` + +### SSL Certificate Issues +```bash +kubectl describe certificate bakery-ia-prod-tls-cert -n bakery-ia +kubectl logs -n cert-manager deployment/cert-manager +kubectl get challenges -n bakery-ia +``` + +### Database Connection Errors +```bash +kubectl get pods -n bakery-ia -l app.kubernetes.io/component=database +kubectl logs -n bakery-ia deployment/auth-db +kubectl exec -n bakery-ia deployment/auth-service -- nc -zv auth-db 5432 +``` + +## Rollback Procedure + +If deployment fails: +```bash +# Rollback specific deployment +kubectl rollout undo deployment/DEPLOYMENT_NAME -n bakery-ia + +# Check rollout history +kubectl rollout history deployment/DEPLOYMENT_NAME -n bakery-ia + +# Rollback to specific revision +kubectl rollout undo deployment/DEPLOYMENT_NAME --to-revision=2 -n bakery-ia +``` + +## Important Commands + +```bash +# View all resources +kubectl get all -n bakery-ia + +# Check logs +kubectl logs -f deployment/gateway -n bakery-ia + +# Check events +kubectl get events -n bakery-ia --sort-by='.lastTimestamp' + +# Resource usage +kubectl top nodes +kubectl top pods -n bakery-ia + +# Scale deployment +kubectl scale deployment/gateway --replicas=5 -n bakery-ia + +# Restart deployment +kubectl rollout restart deployment/gateway -n bakery-ia + +# Execute in pod +kubectl exec -it deployment/gateway -n bakery-ia -- /bin/bash +``` + +## Success Criteria + +Deployment is successful when: +- [ ] All pods are in Running state +- [ ] Application accessible via HTTPS +- [ ] SSL certificate is valid and auto-renewing +- [ ] Database migrations completed +- [ ] All health checks passing +- [ ] Monitoring and alerts configured +- [ ] Backups running successfully +- [ ] Team can access and operate the system +- [ ] Performance meets requirements +- [ ] No critical security issues + +## Support Resources + +- **Full Migration Guide:** See `docs/K8S-MIGRATION-GUIDE.md` +- **MicroK8s Docs:** https://microk8s.io/docs +- **Kubernetes Docs:** https://kubernetes.io/docs +- **Cert-Manager Docs:** https://cert-manager.io/docs + +--- + +**Note:** This is a condensed checklist. Refer to the full migration guide for detailed explanations and troubleshooting. diff --git a/docs/MIGRATION-SUMMARY.md b/docs/MIGRATION-SUMMARY.md new file mode 100644 index 00000000..914d59a7 --- /dev/null +++ b/docs/MIGRATION-SUMMARY.md @@ -0,0 +1,275 @@ +# Migration Summary: Local to Production + +## Quick Overview + +You're migrating from **Kind/Colima (macOS)** to **MicroK8s (Ubuntu VPS)**. + +Good news: **Most of your Kubernetes configuration is already production-ready!** Your infrastructure is well-structured with proper overlays for dev and prod environments. + +## What You Already Have ✅ + +Your configuration already includes: +- ✅ Separate dev and prod overlays +- ✅ Production ingress configuration +- ✅ Production ConfigMap with proper settings +- ✅ Resource scaling (2-3 replicas per service in prod) +- ✅ HorizontalPodAutoscalers for key services +- ✅ Security configurations (TLS, secrets, etc.) +- ✅ Database configurations +- ✅ Monitoring components (Prometheus, Grafana) + +## What Needs to Change 🔧 + +### Critical Changes (Must Do) + +1. **Domain Names** - Update in `infrastructure/kubernetes/overlays/prod/prod-ingress.yaml`: + - Replace `bakery.yourdomain.com` → your actual domain + - Replace `api.yourdomain.com` → your actual API domain + - Replace `monitoring.yourdomain.com` → your actual monitoring domain + - Update CORS origins + - Update cert-manager email + +2. **Storage Class** - Already patched in `storage-patch.yaml`: + - `standard` → `microk8s-hostpath` + +3. **Production Secrets** - Update in `infrastructure/kubernetes/base/secrets.yaml`: + - Generate strong passwords + - Update all sensitive values + - **Never commit real secrets to git!** + +4. **Container Registry** - Choose and configure: + - Docker Hub (easiest) + - GitHub Container Registry + - MicroK8s built-in registry + - Update image references in prod kustomization + +### Setup on VPS + +1. **Install MicroK8s**: + ```bash + sudo snap install microk8s --classic + microk8s enable dns hostpath-storage ingress cert-manager metrics-server + ``` + +2. **Configure Firewall**: + ```bash + sudo ufw allow 22/tcp 80/tcp 443/tcp + sudo ufw enable + ``` + +3. **DNS Configuration**: + Point your domains to VPS IP address + +## File Changes Summary + +### New Files Created +``` +docs/K8S-MIGRATION-GUIDE.md # Comprehensive guide +docs/MIGRATION-CHECKLIST.md # Quick checklist +docs/MIGRATION-SUMMARY.md # This file +infrastructure/kubernetes/overlays/prod/storage-patch.yaml # Storage fix +scripts/deploy-production.sh # Deployment helper +scripts/tag-and-push-images.sh # Image management +scripts/backup-databases.sh # Backup script +``` + +### Files to Modify + +1. **infrastructure/kubernetes/overlays/prod/prod-ingress.yaml** + - Update domain names (3 places) + - Update CORS origins + - Update cert-manager email + +2. **infrastructure/kubernetes/base/secrets.yaml** + - Update all secrets with production values + - Generate strong passwords + +3. **infrastructure/kubernetes/overlays/prod/kustomization.yaml** + - Update image registry prefixes if using external registry + - Already includes storage patch + +## Key Differences Table + +| Feature | Local (Kind) | Production (MicroK8s) | Action Required | +|---------|--------------|----------------------|-----------------| +| **Cluster** | Kind in Docker | Native MicroK8s | Install MicroK8s | +| **Ingress** | Custom NGINX | MicroK8s addon | Enable addon | +| **Storage** | `standard` | `microk8s-hostpath` | Use storage patch ✅ | +| **Images** | Local build | Registry push | Setup registry | +| **Domains** | localhost | Real domains | Update ingress | +| **SSL** | Self-signed | Let's Encrypt | Configure email | +| **Replicas** | 1 per service | 2-3 per service | Already configured ✅ | +| **Resources** | Minimal | Production limits | Already configured ✅ | +| **Secrets** | Dev secrets | Production secrets | Update values | +| **Monitoring** | Optional | Recommended | Already configured ✅ | + +## Deployment Steps (Quick Version) + +### Phase 1: Prepare (On Local Machine) +```bash +# 1. Update domain names +vim infrastructure/kubernetes/overlays/prod/prod-ingress.yaml + +# 2. Update secrets (use strong passwords!) +vim infrastructure/kubernetes/base/secrets.yaml + +# 3. Build and push images +docker login # or setup your registry +./scripts/tag-and-push-images.sh YOUR_USERNAME/bakery latest + +# 4. Update image references if using external registry +vim infrastructure/kubernetes/overlays/prod/kustomization.yaml +``` + +### Phase 2: Setup VPS +```bash +# SSH to VPS +ssh user@YOUR_VPS_IP + +# Install MicroK8s +sudo snap install microk8s --classic --channel=1.28/stable +sudo usermod -a -G microk8s $USER +newgrp microk8s + +# Enable addons +microk8s enable dns hostpath-storage ingress cert-manager metrics-server rbac + +# Setup kubectl +echo "alias kubectl='microk8s kubectl'" >> ~/.bashrc +source ~/.bashrc + +# Configure firewall +sudo ufw allow 22/tcp 80/tcp 443/tcp +sudo ufw enable +``` + +### Phase 3: Deploy +```bash +# On VPS - clone your repo or copy manifests +git clone YOUR_REPO_URL +cd bakery_ia + +# Deploy +kubectl apply -k infrastructure/kubernetes/overlays/prod + +# Monitor +kubectl get pods -n bakery-ia -w + +# Check everything +kubectl get all,ingress,pvc,certificate -n bakery-ia +``` + +### Phase 4: Verify +```bash +# Test access +curl -k https://bakery.yourdomain.com +curl -k https://api.yourdomain.com/health + +# Check SSL +kubectl get certificate -n bakery-ia + +# Check logs +kubectl logs -n bakery-ia deployment/gateway +``` + +## Common Pitfalls to Avoid + +1. **Forgot to update domain names** → Ingress won't work +2. **Using dev secrets in production** → Security risk +3. **DNS not propagated** → SSL certificate won't issue +4. **Firewall blocking ports 80/443** → Can't access application +5. **Images not in registry** → Pods fail with ImagePullBackOff +6. **Wrong storage class** → PVCs stay pending +7. **Insufficient VPS resources** → Pods get evicted + +## Resource Requirements + +### Minimum VPS Specs +- **CPU**: 4 cores (6+ recommended) +- **RAM**: 8GB (16GB+ recommended) +- **Disk**: 100GB (SSD preferred) +- **Network**: Public IP with ports 80/443 open + +### Resource Usage Estimates +With current prod configuration: +- ~20-30 pods running +- ~4-6GB memory used +- ~2-3 CPU cores used +- ~10-20GB disk for databases + +## Testing Strategy + +1. **Local Testing** (Before deploying): + - Build all images successfully + - Test with `skaffold build -f skaffold-prod.yaml` + - Validate kustomization: `kubectl kustomize infrastructure/kubernetes/overlays/prod` + +2. **Staging Deploy** (First deploy): + - Deploy to staging/test environment first + - Test all functionality + - Verify SSL certificates + - Load test + +3. **Production Deploy**: + - Deploy during low-traffic window + - Have rollback plan ready + - Monitor closely for first 24 hours + +## Rollback Plan + +If deployment fails: +```bash +# Quick rollback +kubectl rollout undo deployment/DEPLOYMENT_NAME -n bakery-ia + +# Or delete and redeploy previous version +kubectl delete -k infrastructure/kubernetes/overlays/prod +# Deploy previous version +``` + +Always have: +- Previous version images tagged +- Database backups +- Configuration backups + +## Post-Deployment Checklist + +- [ ] Application accessible via HTTPS +- [ ] SSL certificates valid +- [ ] All services healthy +- [ ] Database migrations completed +- [ ] Monitoring configured +- [ ] Backups scheduled +- [ ] Alerts configured +- [ ] Team has access +- [ ] Documentation updated +- [ ] Runbooks created + +## Getting Help + +- **Full Guide**: See `docs/K8S-MIGRATION-GUIDE.md` +- **Checklist**: See `docs/MIGRATION-CHECKLIST.md` +- **MicroK8s**: https://microk8s.io/docs +- **Kubernetes**: https://kubernetes.io/docs + +## Estimated Timeline + +- **VPS Setup**: 30-60 minutes +- **Configuration Updates**: 30-60 minutes +- **Image Build & Push**: 20-40 minutes +- **Deployment**: 15-30 minutes +- **Verification & Testing**: 30-60 minutes +- **Total**: 2-4 hours (first time) + +With experience: ~1 hour for updates/redeployments + +## Next Steps + +1. Read through the full migration guide +2. Provision your VPS +3. Update configuration files +4. Test locally first +5. Deploy to production +6. Monitor and optimize + +Good luck! 🚀 diff --git a/infrastructure/kubernetes/overlays/prod/kustomization.yaml b/infrastructure/kubernetes/overlays/prod/kustomization.yaml index 0dfa766e..3e839d0b 100644 --- a/infrastructure/kubernetes/overlays/prod/kustomization.yaml +++ b/infrastructure/kubernetes/overlays/prod/kustomization.yaml @@ -11,6 +11,9 @@ resources: - prod-ingress.yaml - prod-configmap.yaml +patchesStrategicMerge: + - storage-patch.yaml + labels: - includeSelectors: true pairs: diff --git a/infrastructure/kubernetes/overlays/prod/storage-patch.yaml b/infrastructure/kubernetes/overlays/prod/storage-patch.yaml new file mode 100644 index 00000000..0cc89883 --- /dev/null +++ b/infrastructure/kubernetes/overlays/prod/storage-patch.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: model-storage + namespace: bakery-ia +spec: + storageClassName: microk8s-hostpath # MicroK8s storage class + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 50Gi # Increased for production (adjust based on your needs) diff --git a/scripts/backup-databases.sh b/scripts/backup-databases.sh new file mode 100755 index 00000000..7ab8e90b --- /dev/null +++ b/scripts/backup-databases.sh @@ -0,0 +1,161 @@ +#!/bin/bash + +# Database Backup Script for Bakery IA +# This script backs up all PostgreSQL databases in the Kubernetes cluster +# Designed to run on the VPS via cron + +set -e + +# Configuration +BACKUP_ROOT="/backups" +NAMESPACE="bakery-ia" +RETENTION_DAYS=7 +DATE=$(date +%Y-%m-%d_%H-%M-%S) +BACKUP_DIR="${BACKUP_ROOT}/${DATE}" + +# Colors +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' + +# Logging +log() { + echo "[$(date +'%Y-%m-%d %H:%M:%S')] $1" +} + +log_error() { + echo -e "${RED}[$(date +'%Y-%m-%d %H:%M:%S')] ERROR: $1${NC}" +} + +log_success() { + echo -e "${GREEN}[$(date +'%Y-%m-%d %H:%M:%S')] SUCCESS: $1${NC}" +} + +log_warning() { + echo -e "${YELLOW}[$(date +'%Y-%m-%d %H:%M:%S')] WARNING: $1${NC}" +} + +# Create backup directory +mkdir -p "$BACKUP_DIR" + +log "Starting database backup to $BACKUP_DIR" + +# Get all database pods +DB_PODS=$(kubectl get pods -n "$NAMESPACE" -l app.kubernetes.io/component=database -o jsonpath='{.items[*].metadata.name}') + +if [ -z "$DB_PODS" ]; then + log_error "No database pods found in namespace $NAMESPACE" + exit 1 +fi + +log "Found database pods: $DB_PODS" + +# Backup counter +SUCCESS_COUNT=0 +FAILED_COUNT=0 +FAILED_DBS=() + +# Backup each database +for pod in $DB_PODS; do + log "Backing up database: $pod" + + # Get database name from pod labels + DB_NAME=$(kubectl get pod "$pod" -n "$NAMESPACE" -o jsonpath='{.metadata.labels.app\.kubernetes\.io/name}') + + if [ -z "$DB_NAME" ]; then + DB_NAME=$pod + fi + + BACKUP_FILE="${BACKUP_DIR}/${DB_NAME}.sql" + + # Perform backup + if kubectl exec -n "$NAMESPACE" "$pod" -- pg_dumpall -U postgres > "$BACKUP_FILE" 2>/dev/null; then + FILE_SIZE=$(du -h "$BACKUP_FILE" | cut -f1) + log_success "Backed up $DB_NAME ($FILE_SIZE)" + ((SUCCESS_COUNT++)) + else + log_error "Failed to backup $DB_NAME" + FAILED_DBS+=("$DB_NAME") + ((FAILED_COUNT++)) + rm -f "$BACKUP_FILE" # Remove partial backup + fi +done + +# Also backup Redis if present +REDIS_POD=$(kubectl get pods -n "$NAMESPACE" -l app.kubernetes.io/name=redis -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "") + +if [ -n "$REDIS_POD" ]; then + log "Backing up Redis: $REDIS_POD" + REDIS_BACKUP="${BACKUP_DIR}/redis.rdb" + + if kubectl exec -n "$NAMESPACE" "$REDIS_POD" -- redis-cli --rdb /tmp/dump.rdb SAVE > /dev/null 2>&1 && \ + kubectl cp "$NAMESPACE/$REDIS_POD:/tmp/dump.rdb" "$REDIS_BACKUP" > /dev/null 2>&1; then + FILE_SIZE=$(du -h "$REDIS_BACKUP" | cut -f1) + log_success "Backed up Redis ($FILE_SIZE)" + ((SUCCESS_COUNT++)) + else + log_warning "Failed to backup Redis (non-critical)" + fi +fi + +# Create backup metadata +cat > "${BACKUP_DIR}/backup-info.txt" < /dev/null; then + echo -e "${RED}Error: kubectl not found. Please install kubectl or setup microk8s alias.${NC}" + exit 1 +fi + +# Function to check if cluster is accessible +check_cluster() { + echo -e "${YELLOW}Checking cluster connectivity...${NC}" + if ! kubectl cluster-info &> /dev/null; then + echo -e "${RED}Error: Cannot connect to Kubernetes cluster.${NC}" + echo "Please ensure your kubeconfig is set correctly." + exit 1 + fi + echo -e "${GREEN}✓ Cluster connection successful${NC}" + echo "" +} + +# Function to check required addons +check_addons() { + echo -e "${YELLOW}Checking required MicroK8s addons...${NC}" + + # Check if this is MicroK8s + if command -v microk8s &> /dev/null; then + REQUIRED_ADDONS=("dns" "hostpath-storage" "ingress" "cert-manager" "metrics-server") + + for addon in "${REQUIRED_ADDONS[@]}"; do + if microk8s status | grep -q "$addon: enabled"; then + echo -e "${GREEN}✓ $addon enabled${NC}" + else + echo -e "${RED}✗ $addon not enabled${NC}" + echo -e "${YELLOW}Enable with: microk8s enable $addon${NC}" + exit 1 + fi + done + else + echo -e "${YELLOW}Not running on MicroK8s. Skipping addon check.${NC}" + fi + echo "" +} + +# Function to create namespace +create_namespace() { + echo -e "${YELLOW}Creating namespace...${NC}" + if kubectl get namespace $NAMESPACE &> /dev/null; then + echo -e "${GREEN}✓ Namespace $NAMESPACE already exists${NC}" + else + kubectl create namespace $NAMESPACE + echo -e "${GREEN}✓ Namespace $NAMESPACE created${NC}" + fi + echo "" +} + +# Function to apply secrets +apply_secrets() { + echo -e "${YELLOW}Applying secrets...${NC}" + echo -e "${RED}WARNING: Ensure production secrets are updated before deployment!${NC}" + read -p "Have you updated production secrets? (yes/no): " confirm + + if [ "$confirm" != "yes" ]; then + echo -e "${RED}Deployment cancelled. Please update secrets first.${NC}" + exit 1 + fi + + kubectl apply -f infrastructure/kubernetes/base/secrets.yaml + kubectl apply -f infrastructure/kubernetes/base/secrets/postgres-tls-secret.yaml + kubectl apply -f infrastructure/kubernetes/base/secrets/redis-tls-secret.yaml + kubectl apply -f infrastructure/kubernetes/base/secrets/demo-internal-api-key-secret.yaml + echo -e "${GREEN}✓ Secrets applied${NC}" + echo "" +} + +# Function to apply kustomization +deploy_application() { + echo -e "${YELLOW}Deploying application...${NC}" + kubectl apply -k $KUSTOMIZE_PATH + echo -e "${GREEN}✓ Application deployed${NC}" + echo "" +} + +# Function to wait for deployments +wait_for_deployments() { + echo -e "${YELLOW}Waiting for deployments to be ready...${NC}" + echo "This may take several minutes..." + + # Wait for all deployments + kubectl wait --for=condition=available --timeout=600s \ + deployment --all -n $NAMESPACE + + echo -e "${GREEN}✓ All deployments are ready${NC}" + echo "" +} + +# Function to check deployment status +check_status() { + echo -e "${YELLOW}Deployment Status:${NC}" + echo "" + + echo "Pods:" + kubectl get pods -n $NAMESPACE + echo "" + + echo "Services:" + kubectl get svc -n $NAMESPACE + echo "" + + echo "Ingress:" + kubectl get ingress -n $NAMESPACE + echo "" + + echo "Persistent Volume Claims:" + kubectl get pvc -n $NAMESPACE + echo "" + + echo "Certificates:" + kubectl get certificate -n $NAMESPACE + echo "" +} + +# Function to show access information +show_access_info() { + echo -e "${GREEN}========================================${NC}" + echo -e "${GREEN}Deployment Complete!${NC}" + echo -e "${GREEN}========================================${NC}" + echo "" + echo "Access your application at:" + + # Get ingress hosts + HOSTS=$(kubectl get ingress bakery-ingress-prod -n $NAMESPACE -o jsonpath='{.spec.rules[*].host}' 2>/dev/null || echo "") + + if [ -n "$HOSTS" ]; then + for host in $HOSTS; do + echo " https://$host" + done + else + echo " Configure your domain in prod-ingress.yaml" + fi + + echo "" + echo "Useful commands:" + echo " View logs: kubectl logs -f deployment/gateway -n $NAMESPACE" + echo " Check pods: kubectl get pods -n $NAMESPACE" + echo " Check events: kubectl get events -n $NAMESPACE --sort-by='.lastTimestamp'" + echo " Scale: kubectl scale deployment/gateway --replicas=5 -n $NAMESPACE" + echo "" +} + +# Main deployment flow +main() { + check_cluster + check_addons + create_namespace + apply_secrets + deploy_application + + echo -e "${YELLOW}Do you want to wait for deployments to be ready? (yes/no):${NC}" + read -p "> " wait_confirm + + if [ "$wait_confirm" = "yes" ]; then + wait_for_deployments + fi + + check_status + show_access_info +} + +# Run main function +main diff --git a/scripts/tag-and-push-images.sh b/scripts/tag-and-push-images.sh new file mode 100755 index 00000000..1a85ddbb --- /dev/null +++ b/scripts/tag-and-push-images.sh @@ -0,0 +1,154 @@ +#!/bin/bash + +# Script to tag and push all Bakery IA images to a container registry +# Usage: ./tag-and-push-images.sh [REGISTRY_PREFIX] [TAG] +# Example: ./tag-and-push-images.sh myuser/bakery v1.0.0 + +set -e + +# Colors +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' + +# Configuration +REGISTRY_PREFIX="${1:-}" +TAG="${2:-latest}" + +if [ -z "$REGISTRY_PREFIX" ]; then + echo -e "${RED}Error: Registry prefix required${NC}" + echo "Usage: $0 REGISTRY_PREFIX [TAG]" + echo "" + echo "Examples:" + echo " Docker Hub: $0 myusername/bakery v1.0.0" + echo " GitHub: $0 ghcr.io/myorg/bakery v1.0.0" + echo " MicroK8s: $0 YOUR_VPS_IP:32000/bakery v1.0.0" + exit 1 +fi + +# List of all services +SERVICES=( + "gateway" + "dashboard" + "auth-service" + "tenant-service" + "training-service" + "forecasting-service" + "sales-service" + "external-service" + "notification-service" + "inventory-service" + "recipes-service" + "suppliers-service" + "pos-service" + "orders-service" + "production-service" + "procurement-service" + "orchestrator-service" + "alert-processor" + "ai-insights-service" + "demo-session-service" + "distribution-service" +) + +echo -e "${GREEN}========================================${NC}" +echo -e "${GREEN}Bakery IA - Image Tagging and Push${NC}" +echo -e "${GREEN}========================================${NC}" +echo "" +echo "Registry: $REGISTRY_PREFIX" +echo "Tag: $TAG" +echo "" + +# Function to tag image +tag_image() { + local service=$1 + local local_name="bakery/${service}" + local remote_name="${REGISTRY_PREFIX}-${service}:${TAG}" + + echo -e "${YELLOW}Tagging ${local_name} -> ${remote_name}${NC}" + + if docker tag "$local_name" "$remote_name"; then + echo -e "${GREEN}✓ Tagged $service${NC}" + return 0 + else + echo -e "${RED}✗ Failed to tag $service${NC}" + return 1 + fi +} + +# Function to push image +push_image() { + local service=$1 + local remote_name="${REGISTRY_PREFIX}-${service}:${TAG}" + + echo -e "${YELLOW}Pushing ${remote_name}${NC}" + + if docker push "$remote_name"; then + echo -e "${GREEN}✓ Pushed $service${NC}" + return 0 + else + echo -e "${RED}✗ Failed to push $service${NC}" + return 1 + fi +} + +# Check if user is logged in to registry +echo -e "${YELLOW}Checking registry authentication...${NC}" +if ! docker info > /dev/null 2>&1; then + echo -e "${RED}Error: Docker daemon not running${NC}" + exit 1 +fi + +echo -e "${GREEN}✓ Docker is running${NC}" +echo "" + +# Ask for confirmation +echo -e "${YELLOW}This will tag and push ${#SERVICES[@]} images.${NC}" +read -p "Continue? (yes/no): " confirm + +if [ "$confirm" != "yes" ]; then + echo "Cancelled." + exit 0 +fi + +echo "" +echo -e "${GREEN}Starting image tagging and push...${NC}" +echo "" + +# Track success/failure +SUCCESS_COUNT=0 +FAILED_SERVICES=() + +# Tag and push all images +for service in "${SERVICES[@]}"; do + if tag_image "$service" && push_image "$service"; then + ((SUCCESS_COUNT++)) + else + FAILED_SERVICES+=("$service") + fi + echo "" +done + +# Summary +echo -e "${GREEN}========================================${NC}" +echo -e "${GREEN}Summary${NC}" +echo -e "${GREEN}========================================${NC}" +echo "" +echo "Successfully pushed: $SUCCESS_COUNT/${#SERVICES[@]}" + +if [ ${#FAILED_SERVICES[@]} -gt 0 ]; then + echo -e "${RED}Failed services:${NC}" + for service in "${FAILED_SERVICES[@]}"; do + echo -e "${RED} - $service${NC}" + done + exit 1 +else + echo -e "${GREEN}All images pushed successfully!${NC}" + echo "" + echo "Next steps:" + echo "1. Update image names in infrastructure/kubernetes/overlays/prod/kustomization.yaml" + echo "2. Deploy to production: kubectl apply -k infrastructure/kubernetes/overlays/prod" +fi + +echo ""