Add new infra architecture
This commit is contained in:
Urtzi Alfaro
193
infrastructure/platform/storage/minio/minio-bucket-init-job.yaml
Normal file
193
infrastructure/platform/storage/minio/minio-bucket-init-job.yaml
Normal file
@@ -0,0 +1,193 @@
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: minio-bucket-init
|
||||
namespace: bakery-ia
|
||||
labels:
|
||||
app.kubernetes.io/name: minio-bucket-init
|
||||
app.kubernetes.io/component: storage-init
|
||||
app.kubernetes.io/part-of: bakery-ia
|
||||
spec:
|
||||
ttlSecondsAfterFinished: 300
|
||||
backoffLimit: 3
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: minio-bucket-init
|
||||
app.kubernetes.io/component: storage-init
|
||||
spec:
|
||||
restartPolicy: OnFailure
|
||||
initContainers:
|
||||
# Wait for MinIO to be ready
|
||||
- name: wait-for-minio
|
||||
image: busybox:1.36
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
echo "Waiting for MinIO to be ready..."
|
||||
until nc -z minio.bakery-ia.svc.cluster.local 9000; do
|
||||
echo "MinIO not ready, waiting..."
|
||||
sleep 5
|
||||
done
|
||||
echo "MinIO is ready!"
|
||||
containers:
|
||||
- name: bucket-init
|
||||
image: minio/mc:RELEASE.2024-11-17T19-35-25Z
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- |
|
||||
set -e
|
||||
|
||||
echo "Configuring MinIO client..."
|
||||
|
||||
# Configure mc alias with TLS (skip cert verification for self-signed)
|
||||
mc alias set myminio https://minio.bakery-ia.svc.cluster.local:9000 \
|
||||
${MINIO_ROOT_USER} ${MINIO_ROOT_PASSWORD} --insecure
|
||||
|
||||
echo "Creating buckets..."
|
||||
|
||||
# Create training-models bucket if not exists
|
||||
if ! mc ls myminio/training-models --insecure 2>/dev/null; then
|
||||
mc mb myminio/training-models --insecure
|
||||
echo "Created bucket: training-models"
|
||||
else
|
||||
echo "Bucket already exists: training-models"
|
||||
fi
|
||||
|
||||
# Set bucket policy (private by default)
|
||||
mc anonymous set none myminio/training-models --insecure
|
||||
|
||||
# Enable versioning for model backups
|
||||
mc version enable myminio/training-models --insecure
|
||||
echo "Enabled versioning on training-models bucket"
|
||||
|
||||
# Set lifecycle policy to expire old versions after 90 days
|
||||
cat > /tmp/lifecycle.json << 'EOF'
|
||||
{
|
||||
"Rules": [
|
||||
{
|
||||
"ID": "expire-old-versions",
|
||||
"Status": "Enabled",
|
||||
"Filter": {
|
||||
"Prefix": "models/"
|
||||
},
|
||||
"NoncurrentVersionExpiration": {
|
||||
"NoncurrentDays": 90
|
||||
}
|
||||
},
|
||||
{
|
||||
"ID": "expire-old-metadata",
|
||||
"Status": "Enabled",
|
||||
"Filter": {
|
||||
"Prefix": "models/"
|
||||
},
|
||||
"Expiration": {
|
||||
"ExpiredObjectDeleteMarker": true
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
EOF
|
||||
mc ilm import myminio/training-models < /tmp/lifecycle.json --insecure || true
|
||||
echo "Lifecycle policy configured"
|
||||
|
||||
# Create service accounts with limited permissions
|
||||
echo "Creating service accounts..."
|
||||
|
||||
# Training service policy (read/write models)
|
||||
cat > /tmp/training-policy.json << 'EOF'
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:GetObject",
|
||||
"s3:PutObject",
|
||||
"s3:DeleteObject",
|
||||
"s3:ListBucket",
|
||||
"s3:GetBucketLocation",
|
||||
"s3:ListBucketMultipartUploads"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::training-models",
|
||||
"arn:aws:s3:::training-models/*"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
EOF
|
||||
|
||||
# Forecasting service policy (read-only models)
|
||||
cat > /tmp/forecasting-policy.json << 'EOF'
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:GetObject",
|
||||
"s3:ListBucket"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::training-models",
|
||||
"arn:aws:s3:::training-models/*"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
EOF
|
||||
|
||||
# Create service accounts using credentials from secrets
|
||||
echo "Creating service accounts..."
|
||||
mc admin user add myminio ${TRAINING_MINIO_USER} ${TRAINING_MINIO_PASSWORD} --insecure 2>/dev/null || true
|
||||
mc admin user add myminio ${FORECASTING_MINIO_USER} ${FORECASTING_MINIO_PASSWORD} --insecure 2>/dev/null || true
|
||||
|
||||
# Apply policies (ignore errors if already exists)
|
||||
mc admin policy create myminio training-policy /tmp/training-policy.json --insecure 2>/dev/null || true
|
||||
mc admin policy attach myminio training-policy --user=${TRAINING_MINIO_USER} --insecure 2>/dev/null || true
|
||||
|
||||
mc admin policy create myminio forecasting-policy /tmp/forecasting-policy.json --insecure 2>/dev/null || true
|
||||
mc admin policy attach myminio forecasting-policy --user=${FORECASTING_MINIO_USER} --insecure 2>/dev/null || true
|
||||
|
||||
echo "MinIO bucket initialization complete!"
|
||||
|
||||
# List buckets for verification
|
||||
echo "Current buckets:"
|
||||
mc ls myminio --insecure
|
||||
|
||||
env:
|
||||
- name: MINIO_ROOT_USER
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: minio-secrets
|
||||
key: MINIO_ROOT_USER
|
||||
- name: MINIO_ROOT_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: minio-secrets
|
||||
key: MINIO_ROOT_PASSWORD
|
||||
# Training service MinIO credentials
|
||||
- name: TRAINING_MINIO_USER
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: minio-secrets
|
||||
key: MINIO_ACCESS_KEY
|
||||
- name: TRAINING_MINIO_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: minio-secrets
|
||||
key: MINIO_SECRET_KEY
|
||||
# Forecasting service MinIO credentials
|
||||
- name: FORECASTING_MINIO_USER
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: minio-secrets
|
||||
key: FORECASTING_MINIO_ACCESS_KEY
|
||||
- name: FORECASTING_MINIO_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: minio-secrets
|
||||
key: FORECASTING_MINIO_SECRET_KEY
|
||||
Reference in New Issue
Block a user