Fix resources isues 32

infrastructure/cicd/tekton-helm/templates/task-kaniko-build.yaml

@@ -128,13 +128,24 @@ spec:
           SERVICES="$SERVICES_PARAM"
         fi
 
-        # Build each service
-        echo "$SERVICES" | tr ',' '\n' | while read service; do
+        # Build each service SEQUENTIALLY to avoid registry upload conflicts
+        # Track build results using files (variables don't persist across subshells)
+        BUILD_STATUS_FILE="/tmp/build_status"
+        echo "0" > "$BUILD_STATUS_FILE.success"
+        echo "0" > "$BUILD_STATUS_FILE.failed"
+        echo "" > "$BUILD_STATUS_FILE.failed_services"
+
+        # Convert comma-separated to newline-separated and iterate
+        # Using for loop instead of pipe to avoid subshell issues
+        SERVICES_LIST=$(echo "$SERVICES" | tr ',' ' ')
+
+        for service in $SERVICES_LIST; do
           service=$(echo "$service" | tr -d ' ')  # Trim whitespace
           if [ -n "$service" ] && [ "$service" != "none" ] && [ "$service" != "infrastructure" ] && [ "$service" != "shared" ]; then
             echo ""
+            echo "==================================================================="
             echo "Building service: $service"
-            echo "-------------------------------------------------------------------"
+            echo "==================================================================="
 
             # Determine Dockerfile path (services vs gateway vs frontend)
             if [ "$service" = "gateway" ]; then
@@ -151,25 +162,85 @@ spec:
               continue
             fi
 
-            /kaniko/executor \
-              --dockerfile="$DOCKERFILE_PATH" \
-              --destination="$(params.registry)/$service:$(params.git-revision)" \
-              --context="$WORKSPACE" \
-              --build-arg="BASE_REGISTRY=$(params.base-registry)" \
-              --build-arg="PYTHON_IMAGE=$(params.python-image)" \
-              --cache=true \
-              --cache-repo="$(params.registry)/cache" \
-              --cache-ttl=168h
+            # Build with retry logic to handle transient registry errors
+            RETRY_COUNT=0
+            MAX_RETRIES=2
+            BUILD_SUCCESS=false
 
-            echo "Successfully built: $(params.registry)/$service:$(params.git-revision)"
+            while [ "$RETRY_COUNT" -le "$MAX_RETRIES" ] && [ "$BUILD_SUCCESS" = "false" ]; do
+              if [ "$RETRY_COUNT" -gt 0 ]; then
+                echo "Retry $RETRY_COUNT/$MAX_RETRIES for $service..."
+                # Wait before retry to let registry recover
+                sleep 10
+              fi
+
+              if /kaniko/executor \
+                --dockerfile="$DOCKERFILE_PATH" \
+                --destination="$(params.registry)/$service:$(params.git-revision)" \
+                --context="$WORKSPACE" \
+                --build-arg="BASE_REGISTRY=$(params.base-registry)" \
+                --build-arg="PYTHON_IMAGE=$(params.python-image)" \
+                --cache=true \
+                --cache-repo="$(params.registry)/cache" \
+                --cache-ttl=168h \
+                --push-retry=3 \
+                --image-fs-extract-retry=3; then
+                BUILD_SUCCESS=true
+                echo "Successfully built and pushed: $(params.registry)/$service:$(params.git-revision)"
+                # Increment success count
+                COUNT=$(cat "$BUILD_STATUS_FILE.success")
+                echo $((COUNT + 1)) > "$BUILD_STATUS_FILE.success"
+              else
+                RETRY_COUNT=$((RETRY_COUNT + 1))
+                echo "Build/push failed for $service (attempt $RETRY_COUNT)"
+              fi
+            done
+
+            if [ "$BUILD_SUCCESS" = "false" ]; then
+              echo "ERROR: Failed to build $service after $MAX_RETRIES retries"
+              # Increment failed count and record service name
+              COUNT=$(cat "$BUILD_STATUS_FILE.failed")
+              echo $((COUNT + 1)) > "$BUILD_STATUS_FILE.failed"
+              echo "$service" >> "$BUILD_STATUS_FILE.failed_services"
+            fi
+
+            # Small delay between services to let registry settle
+            # This prevents "offset mismatch" errors from concurrent uploads
+            echo "Waiting 5s before next build to let registry settle..."
+            sleep 5
           fi
         done
 
+        # Read final counts
+        SUCCESS_COUNT=$(cat "$BUILD_STATUS_FILE.success")
+        FAILED_COUNT=$(cat "$BUILD_STATUS_FILE.failed")
+        FAILED_SERVICES=$(cat "$BUILD_STATUS_FILE.failed_services" | tr '\n' ',' | sed 's/,$//')
+
         echo ""
         echo "==================================================================="
-        echo "Build completed successfully!"
+        echo "Build Summary"
         echo "==================================================================="
-        echo "success" > $(results.build-status.path)
+        echo "Successful builds: $SUCCESS_COUNT"
+        echo "Failed builds: $FAILED_COUNT"
+        if [ -n "$FAILED_SERVICES" ]; then
+          echo "Failed services: $FAILED_SERVICES"
+        fi
+        echo "==================================================================="
+
+        # Set result based on outcome
+        if [ "$FAILED_COUNT" -gt 0 ]; then
+          if [ "$SUCCESS_COUNT" -gt 0 ]; then
+            echo "partial" > $(results.build-status.path)
+            echo "Build completed with some failures"
+          else
+            echo "failed" > $(results.build-status.path)
+            echo "All builds failed!"
+            exit 1
+          fi
+        else
+          echo "success" > $(results.build-status.path)
+          echo "All builds completed successfully!"
+        fi
       resources:
         limits:
           cpu: 2000m