From 851be67a3a25a2b890256b6c765fe343c937f466 Mon Sep 17 00:00:00 2001
From: Bakery Admin <admin@bakewise.ai>
Date: Fri, 23 Jan 2026 05:55:43 +0100
Subject: [PATCH] Fix resources isues 32

---
 .../templates/task-kaniko-build.yaml          | 101 +++++++++++++++---
 1 file changed, 86 insertions(+), 15 deletions(-)

diff --git a/infrastructure/cicd/tekton-helm/templates/task-kaniko-build.yaml b/infrastructure/cicd/tekton-helm/templates/task-kaniko-build.yaml
index e45facf1..ebdfe507 100644
--- a/infrastructure/cicd/tekton-helm/templates/task-kaniko-build.yaml
+++ b/infrastructure/cicd/tekton-helm/templates/task-kaniko-build.yaml
@@ -128,13 +128,24 @@ spec:
           SERVICES="$SERVICES_PARAM"
         fi
 
-        # Build each service
-        echo "$SERVICES" | tr ',' '\n' | while read service; do
+        # Build each service SEQUENTIALLY to avoid registry upload conflicts
+        # Track build results using files (variables don't persist across subshells)
+        BUILD_STATUS_FILE="/tmp/build_status"
+        echo "0" > "$BUILD_STATUS_FILE.success"
+        echo "0" > "$BUILD_STATUS_FILE.failed"
+        echo "" > "$BUILD_STATUS_FILE.failed_services"
+
+        # Convert comma-separated to newline-separated and iterate
+        # Using for loop instead of pipe to avoid subshell issues
+        SERVICES_LIST=$(echo "$SERVICES" | tr ',' ' ')
+
+        for service in $SERVICES_LIST; do
           service=$(echo "$service" | tr -d ' ')  # Trim whitespace
           if [ -n "$service" ] && [ "$service" != "none" ] && [ "$service" != "infrastructure" ] && [ "$service" != "shared" ]; then
             echo ""
+            echo "==================================================================="
             echo "Building service: $service"
-            echo "-------------------------------------------------------------------"
+            echo "==================================================================="
 
             # Determine Dockerfile path (services vs gateway vs frontend)
             if [ "$service" = "gateway" ]; then
@@ -151,25 +162,85 @@ spec:
               continue
             fi
 
-            /kaniko/executor \
-              --dockerfile="$DOCKERFILE_PATH" \
-              --destination="$(params.registry)/$service:$(params.git-revision)" \
-              --context="$WORKSPACE" \
-              --build-arg="BASE_REGISTRY=$(params.base-registry)" \
-              --build-arg="PYTHON_IMAGE=$(params.python-image)" \
-              --cache=true \
-              --cache-repo="$(params.registry)/cache" \
-              --cache-ttl=168h
+            # Build with retry logic to handle transient registry errors
+            RETRY_COUNT=0
+            MAX_RETRIES=2
+            BUILD_SUCCESS=false
 
-            echo "Successfully built: $(params.registry)/$service:$(params.git-revision)"
+            while [ "$RETRY_COUNT" -le "$MAX_RETRIES" ] && [ "$BUILD_SUCCESS" = "false" ]; do
+              if [ "$RETRY_COUNT" -gt 0 ]; then
+                echo "Retry $RETRY_COUNT/$MAX_RETRIES for $service..."
+                # Wait before retry to let registry recover
+                sleep 10
+              fi
+
+              if /kaniko/executor \
+                --dockerfile="$DOCKERFILE_PATH" \
+                --destination="$(params.registry)/$service:$(params.git-revision)" \
+                --context="$WORKSPACE" \
+                --build-arg="BASE_REGISTRY=$(params.base-registry)" \
+                --build-arg="PYTHON_IMAGE=$(params.python-image)" \
+                --cache=true \
+                --cache-repo="$(params.registry)/cache" \
+                --cache-ttl=168h \
+                --push-retry=3 \
+                --image-fs-extract-retry=3; then
+                BUILD_SUCCESS=true
+                echo "Successfully built and pushed: $(params.registry)/$service:$(params.git-revision)"
+                # Increment success count
+                COUNT=$(cat "$BUILD_STATUS_FILE.success")
+                echo $((COUNT + 1)) > "$BUILD_STATUS_FILE.success"
+              else
+                RETRY_COUNT=$((RETRY_COUNT + 1))
+                echo "Build/push failed for $service (attempt $RETRY_COUNT)"
+              fi
+            done
+
+            if [ "$BUILD_SUCCESS" = "false" ]; then
+              echo "ERROR: Failed to build $service after $MAX_RETRIES retries"
+              # Increment failed count and record service name
+              COUNT=$(cat "$BUILD_STATUS_FILE.failed")
+              echo $((COUNT + 1)) > "$BUILD_STATUS_FILE.failed"
+              echo "$service" >> "$BUILD_STATUS_FILE.failed_services"
+            fi
+
+            # Small delay between services to let registry settle
+            # This prevents "offset mismatch" errors from concurrent uploads
+            echo "Waiting 5s before next build to let registry settle..."
+            sleep 5
           fi
         done
 
+        # Read final counts
+        SUCCESS_COUNT=$(cat "$BUILD_STATUS_FILE.success")
+        FAILED_COUNT=$(cat "$BUILD_STATUS_FILE.failed")
+        FAILED_SERVICES=$(cat "$BUILD_STATUS_FILE.failed_services" | tr '\n' ',' | sed 's/,$//')
+
         echo ""
         echo "==================================================================="
-        echo "Build completed successfully!"
+        echo "Build Summary"
         echo "==================================================================="
-        echo "success" > $(results.build-status.path)
+        echo "Successful builds: $SUCCESS_COUNT"
+        echo "Failed builds: $FAILED_COUNT"
+        if [ -n "$FAILED_SERVICES" ]; then
+          echo "Failed services: $FAILED_SERVICES"
+        fi
+        echo "==================================================================="
+
+        # Set result based on outcome
+        if [ "$FAILED_COUNT" -gt 0 ]; then
+          if [ "$SUCCESS_COUNT" -gt 0 ]; then
+            echo "partial" > $(results.build-status.path)
+            echo "Build completed with some failures"
+          else
+            echo "failed" > $(results.build-status.path)
+            echo "All builds failed!"
+            exit 1
+          fi
+        else
+          echo "success" > $(results.build-status.path)
+          echo "All builds completed successfully!"
+        fi
       resources:
         limits:
           cpu: 2000m