Add migration services
This commit is contained in:
Urtzi Alfaro
231
scripts/dev-reset-database.sh
Executable file
231
scripts/dev-reset-database.sh
Executable file
@@ -0,0 +1,231 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Development Database Reset Script
|
||||
#
|
||||
# This script helps developers reset their databases to a clean slate.
|
||||
# It can reset individual services or all services at once.
|
||||
|
||||
set -e
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Configuration
|
||||
NAMESPACE="bakery-ia"
|
||||
SERVICES=("alert-processor" "auth" "external" "forecasting" "inventory" "notification" "orders" "pos" "production" "recipes" "sales" "suppliers" "tenant" "training")
|
||||
|
||||
print_banner() {
|
||||
echo -e "${BLUE}"
|
||||
echo "╔═══════════════════════════════════════════════════════════════╗"
|
||||
echo "║ Bakery-IA Development Database Reset ║"
|
||||
echo "║ ║"
|
||||
echo "║ This script will reset database(s) to a clean slate ║"
|
||||
echo "║ WARNING: This will delete all existing data! ║"
|
||||
echo "╚═══════════════════════════════════════════════════════════════╝"
|
||||
echo -e "${NC}"
|
||||
}
|
||||
|
||||
show_usage() {
|
||||
echo "Usage: $0 [OPTIONS] [SERVICE]"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " -a, --all Reset all services"
|
||||
echo " -s, --service NAME Reset specific service"
|
||||
echo " -l, --list List available services"
|
||||
echo " -y, --yes Skip confirmation prompts"
|
||||
echo " -h, --help Show this help"
|
||||
echo ""
|
||||
echo "Examples:"
|
||||
echo " $0 --service auth # Reset only auth service"
|
||||
echo " $0 --all # Reset all services"
|
||||
echo " $0 auth # Reset auth service (short form)"
|
||||
}
|
||||
|
||||
list_services() {
|
||||
echo -e "${YELLOW}Available services:${NC}"
|
||||
for service in "${SERVICES[@]}"; do
|
||||
echo " - $service"
|
||||
done
|
||||
}
|
||||
|
||||
confirm_action() {
|
||||
local service="$1"
|
||||
local message="${2:-Are you sure you want to reset}"
|
||||
|
||||
if [[ "$SKIP_CONFIRM" == "true" ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
echo -e "${YELLOW}$message the database for service: ${RED}$service${YELLOW}?${NC}"
|
||||
echo -e "${RED}This will delete ALL existing data!${NC}"
|
||||
read -p "Type 'yes' to continue: " confirmation
|
||||
|
||||
if [[ "$confirmation" != "yes" ]]; then
|
||||
echo -e "${YELLOW}Operation cancelled.${NC}"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
enable_force_recreate() {
|
||||
echo -e "${BLUE}Enabling force recreate mode...${NC}"
|
||||
|
||||
# Update the development config
|
||||
kubectl patch configmap development-config -n "$NAMESPACE" \
|
||||
--patch='{"data":{"DB_FORCE_RECREATE":"true"}}' 2>/dev/null || \
|
||||
kubectl create configmap development-config -n "$NAMESPACE" \
|
||||
--from-literal=DB_FORCE_RECREATE=true \
|
||||
--from-literal=DEVELOPMENT_MODE=true \
|
||||
--from-literal=DEBUG_LOGGING=true || true
|
||||
}
|
||||
|
||||
disable_force_recreate() {
|
||||
echo -e "${BLUE}Disabling force recreate mode...${NC}"
|
||||
|
||||
kubectl patch configmap development-config -n "$NAMESPACE" \
|
||||
--patch='{"data":{"DB_FORCE_RECREATE":"false"}}' 2>/dev/null || true
|
||||
}
|
||||
|
||||
reset_service() {
|
||||
local service="$1"
|
||||
echo -e "${BLUE}Resetting database for service: $service${NC}"
|
||||
|
||||
# Delete existing migration job if it exists
|
||||
kubectl delete job "${service}-migration" -n "$NAMESPACE" 2>/dev/null || true
|
||||
|
||||
# Wait a moment for cleanup
|
||||
sleep 2
|
||||
|
||||
# Create new migration job
|
||||
echo -e "${YELLOW}Creating migration job for $service...${NC}"
|
||||
kubectl apply -f "infrastructure/kubernetes/base/migrations/${service}-migration-job.yaml"
|
||||
|
||||
# Wait for job to complete
|
||||
echo -e "${YELLOW}Waiting for migration to complete...${NC}"
|
||||
kubectl wait --for=condition=complete job/"${service}-migration" -n "$NAMESPACE" --timeout=300s
|
||||
|
||||
# Check job status
|
||||
if kubectl get job "${service}-migration" -n "$NAMESPACE" -o jsonpath='{.status.succeeded}' | grep -q "1"; then
|
||||
echo -e "${GREEN}✓ Database reset completed successfully for $service${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ Database reset failed for $service${NC}"
|
||||
echo "Check logs with: kubectl logs -l job-name=${service}-migration -n $NAMESPACE"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
reset_all_services() {
|
||||
echo -e "${BLUE}Resetting databases for all services...${NC}"
|
||||
|
||||
local failed_services=()
|
||||
|
||||
for service in "${SERVICES[@]}"; do
|
||||
echo -e "\n${BLUE}Processing $service...${NC}"
|
||||
if ! reset_service "$service"; then
|
||||
failed_services+=("$service")
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ ${#failed_services[@]} -eq 0 ]]; then
|
||||
echo -e "\n${GREEN}✓ All services reset successfully!${NC}"
|
||||
else
|
||||
echo -e "\n${RED}✗ Some services failed to reset:${NC}"
|
||||
for service in "${failed_services[@]}"; do
|
||||
echo -e " ${RED}- $service${NC}"
|
||||
done
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
cleanup_migration_jobs() {
|
||||
echo -e "${BLUE}Cleaning up migration jobs...${NC}"
|
||||
kubectl delete jobs -l app.kubernetes.io/component=migration -n "$NAMESPACE" 2>/dev/null || true
|
||||
}
|
||||
|
||||
main() {
|
||||
local action=""
|
||||
local target_service=""
|
||||
|
||||
# Parse arguments
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
-a|--all)
|
||||
action="all"
|
||||
shift
|
||||
;;
|
||||
-s|--service)
|
||||
action="service"
|
||||
target_service="$2"
|
||||
shift 2
|
||||
;;
|
||||
-l|--list)
|
||||
list_services
|
||||
exit 0
|
||||
;;
|
||||
-y|--yes)
|
||||
SKIP_CONFIRM="true"
|
||||
shift
|
||||
;;
|
||||
-h|--help)
|
||||
show_usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
if [[ -z "$action" && -z "$target_service" ]]; then
|
||||
action="service"
|
||||
target_service="$1"
|
||||
fi
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
print_banner
|
||||
|
||||
# Validate arguments
|
||||
if [[ -z "$action" ]]; then
|
||||
echo -e "${RED}Error: No action specified${NC}"
|
||||
show_usage
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ "$action" == "service" && -z "$target_service" ]]; then
|
||||
echo -e "${RED}Error: Service name required${NC}"
|
||||
show_usage
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ "$action" == "service" ]]; then
|
||||
# Validate service name
|
||||
if [[ ! " ${SERVICES[*]} " =~ " ${target_service} " ]]; then
|
||||
echo -e "${RED}Error: Invalid service name: $target_service${NC}"
|
||||
list_services
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# Execute action
|
||||
case "$action" in
|
||||
"all")
|
||||
if confirm_action "ALL SERVICES" "Are you sure you want to reset ALL databases? This will affect"; then
|
||||
enable_force_recreate
|
||||
trap disable_force_recreate EXIT
|
||||
reset_all_services
|
||||
fi
|
||||
;;
|
||||
"service")
|
||||
if confirm_action "$target_service"; then
|
||||
enable_force_recreate
|
||||
trap disable_force_recreate EXIT
|
||||
reset_service "$target_service"
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
# Run main function
|
||||
main "$@"
|
||||
235
scripts/dev-workflow.sh
Executable file
235
scripts/dev-workflow.sh
Executable file
@@ -0,0 +1,235 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Development Workflow Script for Bakery-IA
|
||||
#
|
||||
# This script provides common development workflows with database management
|
||||
|
||||
set -e
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
show_usage() {
|
||||
echo "Development Workflow Script for Bakery-IA"
|
||||
echo ""
|
||||
echo "Usage: $0 [COMMAND] [OPTIONS]"
|
||||
echo ""
|
||||
echo "Commands:"
|
||||
echo " start Start development environment"
|
||||
echo " reset Reset database(s) and restart"
|
||||
echo " clean Clean start (drop all data)"
|
||||
echo " migrate Run migrations only"
|
||||
echo " logs Show service logs"
|
||||
echo " status Show deployment status"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " --service NAME Target specific service (default: all)"
|
||||
echo " --profile NAME Use specific Skaffold profile (minimal, full, dev)"
|
||||
echo " --clean-slate Force recreate all tables"
|
||||
echo " --help Show this help"
|
||||
echo ""
|
||||
echo "Examples:"
|
||||
echo " $0 start --profile minimal # Start with minimal services"
|
||||
echo " $0 reset --service auth # Reset auth service only"
|
||||
echo " $0 clean --profile dev # Clean start with dev profile"
|
||||
}
|
||||
|
||||
start_development() {
|
||||
local profile="${1:-dev}"
|
||||
local clean_slate="${2:-false}"
|
||||
|
||||
echo -e "${BLUE}Starting development environment with profile: $profile${NC}"
|
||||
|
||||
if [[ "$clean_slate" == "true" ]]; then
|
||||
echo -e "${YELLOW}Enabling clean slate mode...${NC}"
|
||||
kubectl create configmap development-config --dry-run=client -o yaml \
|
||||
--from-literal=DB_FORCE_RECREATE=true \
|
||||
--from-literal=DEVELOPMENT_MODE=true \
|
||||
--from-literal=DEBUG_LOGGING=true | \
|
||||
kubectl apply -f -
|
||||
fi
|
||||
|
||||
# Start with Skaffold
|
||||
echo -e "${BLUE}Starting Skaffold with profile: $profile${NC}"
|
||||
skaffold dev --profile="$profile" --port-forward
|
||||
}
|
||||
|
||||
reset_service_and_restart() {
|
||||
local service="$1"
|
||||
local profile="${2:-dev}"
|
||||
|
||||
echo -e "${BLUE}Resetting service: $service${NC}"
|
||||
|
||||
# Reset the database
|
||||
./scripts/dev-reset-database.sh --service "$service" --yes
|
||||
|
||||
# Restart the deployment
|
||||
kubectl rollout restart deployment "${service}-service" -n bakery-ia 2>/dev/null || \
|
||||
kubectl rollout restart deployment "$service" -n bakery-ia 2>/dev/null || true
|
||||
|
||||
echo -e "${GREEN}Service $service reset and restarted${NC}"
|
||||
}
|
||||
|
||||
clean_start() {
|
||||
local profile="${1:-dev}"
|
||||
|
||||
echo -e "${YELLOW}Performing clean start...${NC}"
|
||||
|
||||
# Stop existing Skaffold process
|
||||
pkill -f "skaffold" || true
|
||||
|
||||
# Clean up all deployments
|
||||
kubectl delete jobs -l app.kubernetes.io/component=migration -n bakery-ia 2>/dev/null || true
|
||||
|
||||
# Wait a moment
|
||||
sleep 2
|
||||
|
||||
# Start with clean slate
|
||||
start_development "$profile" "true"
|
||||
}
|
||||
|
||||
run_migrations() {
|
||||
local service="$1"
|
||||
|
||||
if [[ -n "$service" ]]; then
|
||||
echo -e "${BLUE}Running migration for service: $service${NC}"
|
||||
kubectl delete job "${service}-migration" -n bakery-ia 2>/dev/null || true
|
||||
kubectl apply -f "infrastructure/kubernetes/base/migrations/${service}-migration-job.yaml"
|
||||
kubectl wait --for=condition=complete job/"${service}-migration" -n bakery-ia --timeout=300s
|
||||
else
|
||||
echo -e "${BLUE}Running migrations for all services${NC}"
|
||||
kubectl delete jobs -l app.kubernetes.io/component=migration -n bakery-ia 2>/dev/null || true
|
||||
kubectl apply -f infrastructure/kubernetes/base/migrations/
|
||||
# Wait for all migration jobs
|
||||
for job in $(kubectl get jobs -l app.kubernetes.io/component=migration -n bakery-ia -o name); do
|
||||
kubectl wait --for=condition=complete "$job" -n bakery-ia --timeout=300s
|
||||
done
|
||||
fi
|
||||
|
||||
echo -e "${GREEN}Migrations completed${NC}"
|
||||
}
|
||||
|
||||
show_logs() {
|
||||
local service="$1"
|
||||
|
||||
if [[ -n "$service" ]]; then
|
||||
echo -e "${BLUE}Showing logs for service: $service${NC}"
|
||||
kubectl logs -l app.kubernetes.io/name="${service}" -n bakery-ia --tail=100 -f
|
||||
else
|
||||
echo -e "${BLUE}Available services for logs:${NC}"
|
||||
kubectl get deployments -n bakery-ia -o custom-columns="NAME:.metadata.name"
|
||||
fi
|
||||
}
|
||||
|
||||
show_status() {
|
||||
echo -e "${BLUE}Deployment Status:${NC}"
|
||||
echo ""
|
||||
|
||||
echo -e "${YELLOW}Pods:${NC}"
|
||||
kubectl get pods -n bakery-ia
|
||||
|
||||
echo ""
|
||||
echo -e "${YELLOW}Services:${NC}"
|
||||
kubectl get services -n bakery-ia
|
||||
|
||||
echo ""
|
||||
echo -e "${YELLOW}Jobs:${NC}"
|
||||
kubectl get jobs -n bakery-ia
|
||||
|
||||
echo ""
|
||||
echo -e "${YELLOW}ConfigMaps:${NC}"
|
||||
kubectl get configmaps -n bakery-ia
|
||||
}
|
||||
|
||||
main() {
|
||||
local command=""
|
||||
local service=""
|
||||
local profile="dev"
|
||||
local clean_slate="false"
|
||||
|
||||
# Parse arguments
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
start|reset|clean|migrate|logs|status)
|
||||
command="$1"
|
||||
shift
|
||||
;;
|
||||
--service)
|
||||
service="$2"
|
||||
shift 2
|
||||
;;
|
||||
--profile)
|
||||
profile="$2"
|
||||
shift 2
|
||||
;;
|
||||
--clean-slate)
|
||||
clean_slate="true"
|
||||
shift
|
||||
;;
|
||||
--help)
|
||||
show_usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
if [[ -z "$command" ]]; then
|
||||
command="$1"
|
||||
fi
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ -z "$command" ]]; then
|
||||
show_usage
|
||||
exit 1
|
||||
fi
|
||||
|
||||
case "$command" in
|
||||
"start")
|
||||
start_development "$profile" "$clean_slate"
|
||||
;;
|
||||
"reset")
|
||||
if [[ -n "$service" ]]; then
|
||||
reset_service_and_restart "$service" "$profile"
|
||||
else
|
||||
echo -e "${RED}Error: --service required for reset command${NC}"
|
||||
exit 1
|
||||
fi
|
||||
;;
|
||||
"clean")
|
||||
clean_start "$profile"
|
||||
;;
|
||||
"migrate")
|
||||
run_migrations "$service"
|
||||
;;
|
||||
"logs")
|
||||
show_logs "$service"
|
||||
;;
|
||||
"status")
|
||||
show_status
|
||||
;;
|
||||
*)
|
||||
echo -e "${RED}Error: Unknown command: $command${NC}"
|
||||
show_usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
# Check if kubectl and skaffold are available
|
||||
if ! command -v kubectl &> /dev/null; then
|
||||
echo -e "${RED}Error: kubectl is not installed or not in PATH${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v skaffold &> /dev/null; then
|
||||
echo -e "${RED}Error: skaffold is not installed or not in PATH${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Run main function
|
||||
main "$@"
|
||||
134
scripts/run_migrations.py
Executable file
134
scripts/run_migrations.py
Executable file
@@ -0,0 +1,134 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Enhanced Migration Runner
|
||||
|
||||
Handles automatic table creation and Alembic migrations for Kubernetes deployments.
|
||||
Supports both first-time deployments and incremental migrations.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import asyncio
|
||||
import argparse
|
||||
import structlog
|
||||
from pathlib import Path
|
||||
|
||||
# Add the project root to the Python path
|
||||
project_root = Path(__file__).parent.parent
|
||||
sys.path.insert(0, str(project_root))
|
||||
|
||||
from shared.database.base import DatabaseManager
|
||||
from shared.database.init_manager import initialize_service_database
|
||||
|
||||
# Configure logging
|
||||
structlog.configure(
|
||||
processors=[
|
||||
structlog.stdlib.filter_by_level,
|
||||
structlog.stdlib.add_logger_name,
|
||||
structlog.stdlib.add_log_level,
|
||||
structlog.stdlib.PositionalArgumentsFormatter(),
|
||||
structlog.processors.TimeStamper(fmt="iso"),
|
||||
structlog.processors.StackInfoRenderer(),
|
||||
structlog.processors.format_exc_info,
|
||||
structlog.processors.JSONRenderer()
|
||||
],
|
||||
context_class=dict,
|
||||
logger_factory=structlog.stdlib.LoggerFactory(),
|
||||
wrapper_class=structlog.stdlib.BoundLogger,
|
||||
cache_logger_on_first_use=True,
|
||||
)
|
||||
|
||||
logger = structlog.get_logger()
|
||||
|
||||
|
||||
async def run_service_migration(service_name: str, force_recreate: bool = False) -> bool:
|
||||
"""
|
||||
Run migration for a specific service
|
||||
|
||||
Args:
|
||||
service_name: Name of the service (e.g., 'auth', 'inventory')
|
||||
force_recreate: Whether to force recreate tables (development mode)
|
||||
|
||||
Returns:
|
||||
True if successful, False otherwise
|
||||
"""
|
||||
logger.info("Starting migration for service", service=service_name, force_recreate=force_recreate)
|
||||
|
||||
try:
|
||||
# Get database URL from environment (try both constructed and direct approaches)
|
||||
db_url_key = f"{service_name.upper().replace('-', '_')}_DATABASE_URL"
|
||||
database_url = os.getenv(db_url_key) or os.getenv("DATABASE_URL")
|
||||
|
||||
# If no direct URL, construct from components
|
||||
if not database_url:
|
||||
host = os.getenv("POSTGRES_HOST")
|
||||
port = os.getenv("POSTGRES_PORT")
|
||||
db_name = os.getenv("POSTGRES_DB")
|
||||
user = os.getenv("POSTGRES_USER")
|
||||
password = os.getenv("POSTGRES_PASSWORD")
|
||||
|
||||
if all([host, port, db_name, user, password]):
|
||||
database_url = f"postgresql+asyncpg://{user}:{password}@{host}:{port}/{db_name}"
|
||||
logger.info("Constructed database URL from components", host=host, port=port, db=db_name)
|
||||
else:
|
||||
logger.error("Database connection details not found",
|
||||
db_url_key=db_url_key,
|
||||
host=bool(host),
|
||||
port=bool(port),
|
||||
db=bool(db_name),
|
||||
user=bool(user),
|
||||
password=bool(password))
|
||||
return False
|
||||
|
||||
# Create database manager
|
||||
db_manager = DatabaseManager(database_url=database_url)
|
||||
|
||||
# Initialize the database
|
||||
result = await initialize_service_database(
|
||||
database_manager=db_manager,
|
||||
service_name=service_name,
|
||||
force_recreate=force_recreate
|
||||
)
|
||||
|
||||
logger.info("Migration completed successfully", service=service_name, result=result)
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Migration failed", service=service_name, error=str(e))
|
||||
return False
|
||||
|
||||
finally:
|
||||
# Cleanup database connections
|
||||
try:
|
||||
await db_manager.close_connections()
|
||||
except:
|
||||
pass
|
||||
|
||||
|
||||
async def main():
|
||||
"""Main migration runner"""
|
||||
parser = argparse.ArgumentParser(description="Enhanced Migration Runner")
|
||||
parser.add_argument("service", help="Service name (e.g., auth, inventory)")
|
||||
parser.add_argument("--force-recreate", action="store_true",
|
||||
help="Force recreate tables (development mode)")
|
||||
parser.add_argument("--verbose", "-v", action="store_true", help="Verbose logging")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.verbose:
|
||||
logger.info("Starting migration runner", service=args.service,
|
||||
force_recreate=args.force_recreate)
|
||||
|
||||
# Run the migration
|
||||
success = await run_service_migration(args.service, args.force_recreate)
|
||||
|
||||
if success:
|
||||
logger.info("Migration runner completed successfully")
|
||||
sys.exit(0)
|
||||
else:
|
||||
logger.error("Migration runner failed")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
Reference in New Issue
Block a user