bakery-admin/bakery-ia
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add minio support and forntend analitycs

Browse Source
This commit is contained in:
Urtzi Alfaro
2026-01-17 22:42:40 +01:00
parent fbc670ddb3
commit 3c4b5c2a06
53 changed files with 3485 additions and 437 deletions
Download Patch File Download Diff File Expand all files Collapse all files

263
Tiltfile
View File

@@ -16,22 +16,142 @@
# - Gateway only rebuilds when gateway/ or shared/ code changes
# =============================================================================
# =============================================================================
# TILT CONFIGURATION
# =============================================================================
# Update settings
update_settings(
max_parallel_updates=2, # Reduce parallel updates to avoid resource exhaustion
k8s_upsert_timeout_secs=120 # Increase timeout for slower local builds
)
# Ensure we're running in the correct context
allow_k8s_contexts('kind-bakery-ia-local')
# =============================================================================
# DISK SPACE MANAGEMENT & CLEANUP CONFIGURATION
# =============================================================================
# Disk space management settings
disk_cleanup_enabled = True # Default to True, can be disabled with TILT_DISABLE_CLEANUP=true
if 'TILT_DISABLE_CLEANUP' in os.environ:
disk_cleanup_enabled = os.environ['TILT_DISABLE_CLEANUP'].lower() != 'true'
disk_space_threshold_gb = '10'
if 'TILT_DISK_THRESHOLD_GB' in os.environ:
disk_space_threshold_gb = os.environ['TILT_DISK_THRESHOLD_GB']
disk_cleanup_frequency_minutes = '30'
if 'TILT_CLEANUP_FREQUENCY' in os.environ:
disk_cleanup_frequency_minutes = os.environ['TILT_CLEANUP_FREQUENCY']
print("""
DISK SPACE MANAGEMENT CONFIGURATION
======================================
Cleanup Enabled: {}
Free Space Threshold: {}GB
Cleanup Frequency: Every {} minutes
To disable cleanup: export TILT_DISABLE_CLEANUP=true
To change threshold: export TILT_DISK_THRESHOLD_GB=20
To change frequency: export TILT_CLEANUP_FREQUENCY=60
""".format(
'YES' if disk_cleanup_enabled else 'NO (TILT_DISABLE_CLEANUP=true)',
disk_space_threshold_gb,
disk_cleanup_frequency_minutes
))
# Automatic cleanup scheduler (informational only - actual scheduling done externally)
if disk_cleanup_enabled:
local_resource(
'automatic-disk-cleanup-info',
cmd='''
echo "Automatic disk cleanup is ENABLED"
echo "Settings:"
echo " - Threshold: ''' + disk_space_threshold_gb + ''' GB free space"
echo " - Frequency: Every ''' + disk_cleanup_frequency_minutes + ''' minutes"
echo ""
echo "Note: Actual cleanup runs via external scheduling (cron job or similar)"
echo "To run cleanup now: tilt trigger manual-disk-cleanup"
''',
labels=['99-cleanup'],
auto_init=True,
allow_parallel=False
)
# Manual cleanup trigger (can be run on demand)
local_resource(
'manual-disk-cleanup',
cmd='''
echo "Starting manual disk cleanup..."
python3 scripts/cleanup_disk_space.py --manual --verbose
''',
labels=['99-cleanup'],
auto_init=False,
allow_parallel=False
)
# Disk space monitoring resource
local_resource(
'disk-space-monitor',
cmd='''
echo "DISK SPACE MONITORING"
echo "======================================"
# Get disk usage
df -h / | grep -v Filesystem | awk '{{print "Total: " $2 " | Used: " $3 " | Free: " $4 " | Usage: " $5}}'
# Get Docker disk usage
echo ""
echo "DOCKER DISK USAGE:"
docker system df
# Get Kubernetes disk usage (if available)
echo ""
echo "KUBERNETES DISK USAGE:"
kubectl get pvc -n bakery-ia --no-headers 2>/dev/null | awk '{{print "PVC: " $1 " | Status: " $2 " | Capacity: " $3 " | Used: " $4}}' || echo " Kubernetes PVCs not available"
echo ""
echo "Cleanup Status:"
if [ "{disk_cleanup_enabled}" = "True" ]; then
echo " Automatic cleanup: ENABLED (every {disk_cleanup_frequency_minutes} minutes)"
echo " Threshold: {disk_space_threshold_gb}GB free space"
else
echo " Automatic cleanup: DISABLED"
echo " To enable: unset TILT_DISABLE_CLEANUP or set TILT_DISABLE_CLEANUP=false"
fi
echo ""
echo "Manual cleanup commands:"
echo " tilt trigger manual-disk-cleanup # Run cleanup now"
echo " docker system prune -a # Manual Docker cleanup"
echo " kubectl delete jobs --all # Clean up completed jobs"
''',
labels=['99-cleanup'],
auto_init=False,
allow_parallel=False
)
# =============================================================================
# DOCKER REGISTRY CONFIGURATION
# =============================================================================
# Docker registry configuration
# Set USE_DOCKERHUB=true environment variable to push images to Docker Hub
# Otherwise, uses local registry for faster builds and deployments
use_dockerhub = os.getenv('USE_DOCKERHUB', 'false').lower() == 'true'
dockerhub_username = os.getenv('DOCKERHUB_USERNAME', 'uals')
use_dockerhub = False # Default to False
if 'USE_DOCKERHUB' in os.environ:
use_dockerhub = os.environ['USE_DOCKERHUB'].lower() == 'true'
dockerhub_username = 'uals' # Default username
if 'DOCKERHUB_USERNAME' in os.environ:
dockerhub_username = os.environ['DOCKERHUB_USERNAME']
if use_dockerhub:
print("""
🐳 DOCKER HUB MODE ENABLED
DOCKER HUB MODE ENABLED
Images will be pushed to Docker Hub: docker.io/%s
Make sure you're logged in: docker login
To disable: unset USE_DOCKERHUB or set USE_DOCKERHUB=false
@@ -39,7 +159,7 @@ if use_dockerhub:
default_registry('docker.io/%s' % dockerhub_username)
else:
print("""
🏠 LOCAL REGISTRY MODE
LOCAL REGISTRY MODE
Using local registry for faster builds: localhost:5001
This registry is created by kubernetes_restart.sh script
To use Docker Hub: export USE_DOCKERHUB=true
@@ -52,20 +172,21 @@ else:
print("""
======================================
🔐 Bakery IA Secure Development Mode
Bakery IA Secure Development Mode
======================================
Security Features:
✅ TLS encryption for PostgreSQL and Redis
✅ Strong 32-character passwords
✅ PersistentVolumeClaims (no data loss)
✅ pgcrypto extension for encryption
✅ PostgreSQL audit logging
TLS encryption for PostgreSQL and Redis
Strong 32-character passwords
PersistentVolumeClaims (no data loss)
Column encryption: pgcrypto extension
Audit logging: PostgreSQL query logging
Object storage: MinIO with TLS for ML models
Monitoring:
📊 Service metrics available at /metrics endpoints
🔍 Telemetry ready (traces, metrics, logs)
ℹ️ SigNoz deployment optional for local dev (see signoz-info resource)
Service metrics available at /metrics endpoints
Telemetry ready (traces, metrics, logs)
SigNoz deployment optional for local dev (see signoz-info resource)
Applying security configurations...
""")
@@ -74,7 +195,7 @@ Applying security configurations...
local_resource(
'dockerhub-secret',
cmd='''
echo "🐳 Setting up Docker Hub image pull secret..."
echo "Setting up Docker Hub image pull secret..."
# Check if Docker Hub credentials are available
if [ -n "$DOCKERHUB_USERNAME" ] && [ -n "$DOCKERHUB_PASSWORD" ]; then
@@ -84,7 +205,7 @@ local_resource(
echo " Attempting to use Docker CLI credentials..."
./infrastructure/kubernetes/create-dockerhub-secret.sh
else
echo " ⚠️ Docker Hub credentials not found"
echo " Docker Hub credentials not found"
echo " To enable automatic Docker Hub authentication:"
echo " 1. Run 'docker login', OR"
echo " 2. Set environment variables:"
@@ -103,13 +224,13 @@ local_resource(
local_resource(
'security-setup',
cmd='''
echo "📦 Applying security secrets and configurations..."
echo "Applying security secrets and configurations..."
kubectl apply -f infrastructure/kubernetes/base/secrets.yaml
kubectl apply -f infrastructure/kubernetes/base/secrets/postgres-tls-secret.yaml
kubectl apply -f infrastructure/kubernetes/base/secrets/redis-tls-secret.yaml
kubectl apply -f infrastructure/kubernetes/base/configs/postgres-init-config.yaml
kubectl apply -f infrastructure/kubernetes/base/configmaps/postgres-logging-config.yaml
echo "✅ Security configurations applied"
echo "Security configurations applied"
''',
resource_deps=['dockerhub-secret'],
labels=['00-security'],
@@ -120,7 +241,7 @@ local_resource(
local_resource(
'verify-tls',
cmd='''
echo "🔍 Verifying TLS configuration..."
echo "Verifying TLS configuration..."
sleep 5 # Wait for pods to be ready
# Check if auth-db pod exists and has TLS certs
@@ -129,8 +250,8 @@ local_resource(
if [ -n "$AUTH_POD" ]; then
echo " Checking PostgreSQL TLS certificates..."
kubectl exec -n bakery-ia "$AUTH_POD" -- ls -la /tls/ 2>/dev/null && \
echo " ✅ PostgreSQL TLS certificates mounted" || \
echo " ⚠️ PostgreSQL TLS certificates not found (pods may still be starting)"
echo " PostgreSQL TLS certificates mounted" || \
echo " PostgreSQL TLS certificates not found (pods may still be starting)"
fi
# Check if redis pod exists and has TLS certs
@@ -139,15 +260,14 @@ local_resource(
if [ -n "$REDIS_POD" ]; then
echo " Checking Redis TLS certificates..."
kubectl exec -n bakery-ia "$REDIS_POD" -- ls -la /tls/ 2>/dev/null && \
echo " ✅ Redis TLS certificates mounted" || \
echo " ⚠️ Redis TLS certificates not found (pods may still be starting)"
echo " Redis TLS certificates mounted" || \
echo " Redis TLS certificates not found (pods may still be starting)"
fi
echo "✅ TLS verification complete"
echo "TLS verification complete"
''',
resource_deps=['auth-db', 'redis'],
auto_init=True,
trigger_mode=TRIGGER_MODE_MANUAL,
labels=['00-security']
)
@@ -155,15 +275,14 @@ local_resource(
local_resource(
'verify-pvcs',
cmd='''
echo "🔍 Verifying PersistentVolumeClaims..."
kubectl get pvc -n bakery-ia | grep -E "NAME|db-pvc" || echo " ⚠️ PVCs not yet bound"
echo "Verifying PersistentVolumeClaims..."
kubectl get pvc -n bakery-ia | grep -E "NAME|db-pvc" || echo " PVCs not yet bound"
PVC_COUNT=$(kubectl get pvc -n bakery-ia -o json | jq '.items | length')
echo " Found $PVC_COUNT PVCs"
echo "✅ PVC verification complete"
echo "PVC verification complete"
''',
resource_deps=['auth-db'],
auto_init=True,
trigger_mode=TRIGGER_MODE_MANUAL,
labels=['00-security']
)
@@ -171,11 +290,11 @@ local_resource(
local_resource(
'cert-manager-install',
cmd='''
echo "📦 Installing cert-manager..."
echo "Installing cert-manager..."
# Check if cert-manager CRDs already exist
if kubectl get crd certificates.cert-manager.io >/dev/null 2>&1; then
echo " ✅ cert-manager CRDs already installed"
echo " cert-manager CRDs already installed"
else
echo " Installing cert-manager v1.13.2..."
kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.13.2/cert-manager.yaml
@@ -184,10 +303,10 @@ local_resource(
kubectl wait --for=condition=available --timeout=120s deployment/cert-manager -n cert-manager
kubectl wait --for=condition=available --timeout=120s deployment/cert-manager-webhook -n cert-manager
echo " ✅ cert-manager installed and ready"
echo " cert-manager installed and ready"
fi
echo "✅ cert-manager verification complete"
echo "cert-manager verification complete"
''',
labels=['00-security'],
auto_init=True
@@ -265,19 +384,21 @@ def build_python_service(service_name, service_path):
# =============================================================================
# Frontend (React + Vite)
frontend_debug_env = os.getenv('FRONTEND_DEBUG', 'false')
frontend_debug_env = 'false' # Default to false
if 'FRONTEND_DEBUG' in os.environ:
frontend_debug_env = os.environ['FRONTEND_DEBUG']
frontend_debug = frontend_debug_env.lower() == 'true'
if frontend_debug:
print("""
🐛 FRONTEND DEBUG MODE ENABLED
FRONTEND DEBUG MODE ENABLED
Building frontend with NO minification for easier debugging.
Full React error messages will be displayed.
To disable: unset FRONTEND_DEBUG or set FRONTEND_DEBUG=false
""")
else:
print("""
📦 FRONTEND PRODUCTION MODE
FRONTEND PRODUCTION MODE
Building frontend with minification for optimized performance.
To enable debug mode: export FRONTEND_DEBUG=true
""")
@@ -384,6 +505,10 @@ k8s_resource('redis', resource_deps=['security-setup'], labels=['01-infrastructu
k8s_resource('rabbitmq', labels=['01-infrastructure'])
k8s_resource('nominatim', labels=['01-infrastructure'])
# MinIO Storage
k8s_resource('minio', resource_deps=['security-setup'], labels=['01-infrastructure'])
k8s_resource('minio-bucket-init', resource_deps=['minio'], labels=['01-infrastructure'])
# =============================================================================
# MONITORING RESOURCES - SigNoz (Unified Observability)
# =============================================================================
@@ -392,25 +517,25 @@ k8s_resource('nominatim', labels=['01-infrastructure'])
local_resource(
'signoz-deploy',
cmd='''
echo "📊 Deploying SigNoz Monitoring Stack..."
echo "Deploying SigNoz Monitoring Stack..."
echo ""
# Ensure Docker Hub secret exists in bakery-ia namespace
echo "🔐 Ensuring Docker Hub secret exists in bakery-ia namespace..."
echo "Ensuring Docker Hub secret exists in bakery-ia namespace..."
if ! kubectl get secret dockerhub-creds -n bakery-ia &>/dev/null; then
echo " ⚠️ Docker Hub secret not found, attempting to create..."
echo " Docker Hub secret not found, attempting to create..."
./infrastructure/kubernetes/create-dockerhub-secret.sh || echo " Continuing without Docker Hub authentication..."
else
echo " ✅ Docker Hub secret exists"
echo " Docker Hub secret exists"
fi
echo ""
# Check if SigNoz is already deployed
if helm list -n bakery-ia | grep -q signoz; then
echo "✅ SigNoz already deployed, checking status..."
echo "SigNoz already deployed, checking status..."
helm status signoz -n bakery-ia
else
echo "🚀 Installing SigNoz..."
echo "Installing SigNoz..."
# Add SigNoz Helm repository if not already added
helm repo add signoz https://charts.signoz.io 2>/dev/null || true
@@ -424,25 +549,23 @@ local_resource(
--wait
echo ""
echo "✅ SigNoz deployment completed"
echo "SigNoz deployment completed"
fi
echo ""
echo "📈 SigNoz Access Information:"
echo "SigNoz Access Information:"
echo " URL: https://monitoring.bakery-ia.local"
echo " Username: admin"
echo " Password: admin"
echo ""
echo "🔧 OpenTelemetry Collector Endpoints:"
echo "OpenTelemetry Collector Endpoints:"
echo " gRPC: localhost:4317"
echo " HTTP: localhost:4318"
echo ""
echo "💡 To check pod status: kubectl get pods -n signoz"
echo "To check pod status: kubectl get pods -n signoz"
''',
labels=['05-monitoring'],
auto_init=False,
trigger_mode=TRIGGER_MODE_MANUAL,
allow_parallel=False
)
# Track SigNoz pods in Tilt UI using workload tracking
@@ -450,7 +573,7 @@ local_resource(
local_resource(
'signoz-status',
cmd='''
echo "📊 SigNoz Status Check"
echo "SigNoz Status Check"
echo ""
# Check pod status
@@ -470,19 +593,17 @@ local_resource(
echo "Pod Status: $READY_PODS/$TOTAL_PODS ready"
if [ "$READY_PODS" -eq "$TOTAL_PODS" ]; then
echo "✅ All SigNoz pods are running!"
echo "All SigNoz pods are running!"
echo ""
echo "Access SigNoz at: https://monitoring.bakery-ia.local"
echo "Credentials: admin / admin"
else
echo "⏳ Waiting for pods to become ready..."
echo "Waiting for pods to become ready..."
fi
fi
''',
labels=['05-monitoring'],
resource_deps=['signoz-deploy'],
auto_init=False,
trigger_mode=TRIGGER_MODE_MANUAL
)
# Optional exporters (in monitoring namespace) - DISABLED since using SigNoz
@@ -566,7 +687,6 @@ k8s_resource('demo-session-migration', resource_deps=['demo-session-db'], labels
k8s_resource('external-data-init', resource_deps=['external-migration', 'redis'], labels=['08-data-init'])
k8s_resource('nominatim-init', labels=['08-data-init'])
# =============================================================================
# =============================================================================
# APPLICATION SERVICES
# =============================================================================
@@ -618,15 +738,9 @@ k8s_resource('demo-session-cleanup', resource_deps=['demo-session-service'], lab
k8s_resource('external-data-rotation', resource_deps=['external-service'], labels=['16-cronjobs'])
# =============================================================================
# TILT CONFIGURATION
# WATCH SETTINGS
# =============================================================================
# Update settings
update_settings(
max_parallel_updates=2, # Reduce parallel updates to avoid resource exhaustion
k8s_upsert_timeout_secs=120 # Increase timeout for slower local builds
)
# Watch settings
watch_settings(
ignore=[
@@ -665,18 +779,19 @@ watch_settings(
# =============================================================================
print("""
✅ Security setup complete!
Security setup complete!
Database Security Features Active:
🔐 TLS encryption: PostgreSQL and Redis
🔑 Strong passwords: 32-character cryptographic
💾 Persistent storage: PVCs for all databases
🔒 Column encryption: pgcrypto extension
📋 Audit logging: PostgreSQL query logging
TLS encryption: PostgreSQL and Redis
Strong passwords: 32-character cryptographic
Persistent storage: PVCs for all databases
Column encryption: pgcrypto extension
Audit logging: PostgreSQL query logging
Internal Schedulers Active:
⏰ Alert Priority Recalculation: Hourly @ :15 (alert-processor)
⏰ Usage Tracking: Daily @ 2:00 AM UTC (tenant-service)
Alert Priority Recalculation: Hourly @ :15 (alert-processor)
Usage Tracking: Daily @ 2:00 AM UTC (tenant-service)
Disk Cleanup: Every {disk_cleanup_frequency_minutes} minutes (threshold: {disk_space_threshold_gb}GB)
Access your application:
Main Application: https://bakery-ia.local
@@ -708,11 +823,11 @@ Documentation:
docs/DATABASE_SECURITY_ANALYSIS_REPORT.md
Build Optimization Active:
✅ Services only rebuild when their code changes
✅ Shared folder changes trigger ALL services (as expected)
✅ Reduces unnecessary rebuilds and disk usage
💡 Edit service code: only that service rebuilds
💡 Edit shared/ code: all services rebuild (required)
Services only rebuild when their code changes
Shared folder changes trigger ALL services (as expected)
Reduces unnecessary rebuilds and disk usage
Edit service code: only that service rebuilds
Edit shared/ code: all services rebuild (required)
Useful Commands:
# Work on specific services only
@@ -730,4 +845,4 @@ DNS Configuration:
# 127.0.0.1 monitoring.bakery-ia.local
======================================
""")
""")