From 4bfdcad3eabd4483f07ce8bdfd2df0a25c89209e Mon Sep 17 00:00:00 2001
From: Matthew Grossman <mgrossman@nvidia.com>
Date: Wed, 14 Oct 2026 16:16:27 -0700
Subject: [PATCH] ci(helm): retry chart pushes when GHCR briefly misses a new
 manifest

helm push uploads the chart manifest by digest, then fetches it back to
apply the version tag. GHCR sometimes returns 404 for a manifest it just
accepted, failing the push with 'failed to perform "Tag" on
destination: <digest>: not found'. This failed Release Dev on Oct 5 and
the v0.1.5-pre.2 Release Tag run after qualification passed.

Retry each chart push up to four times with a short backoff. A retry
finds the blobs and manifest already present and only applies the tag.

Closes #4577

Signed-off-by: Matthew Grossman <mgrossman@nvidia.com>
---
 .github/actions/release-helm-oci/action.yml | 34 +++++++++++++++++++--
 1 file changed, 32 insertions(+), 2 deletions(-)

diff --git a/.github/actions/release-helm-oci/action.yml b/.github/actions/release-helm-oci/action.yml
index 2d1a1b86b2..a296c9e4ba 100644
--- a/.github/actions/release-helm-oci/action.yml
+++ b/.github/actions/release-helm-oci/action.yml
@@ -94,8 +94,24 @@ runs:
       shell: bash
       run: |
         set -euo pipefail
+        # Helm pushes the manifest by digest, then fetches it back to tag it.
+        # GHCR can briefly return 404 for a manifest it just accepted, so
+        # retry. Re-pushing the same archive is idempotent.
+        push_chart() {
+          local attempt
+          for attempt in 1 2 3 4; do
+            if helm push "$1" oci://ghcr.io/nvidia/openshell; then
+              return 0
+            fi
+            if [ "${attempt}" -lt 4 ]; then
+              echo "::warning::helm push of $(basename "$1") failed (attempt ${attempt}); retrying in $((attempt * 10))s."
+              sleep $((attempt * 10))
+            fi
+          done
+          return 1
+        }
         for archive in /tmp/helm-charts/*.tgz; do
-          helm push "${archive}" oci://ghcr.io/nvidia/openshell
+          push_chart "${archive}"
         done
 
     - name: Push SHA-pinned chart
@@ -107,6 +123,20 @@ runs:
       shell: bash
       run: |
         set -euo pipefail
+        # See "Push Helm chart to GHCR OCI" for why pushes retry.
+        push_chart() {
+          local attempt
+          for attempt in 1 2 3 4; do
+            if helm push "$1" oci://ghcr.io/nvidia/openshell; then
+              return 0
+            fi
+            if [ "${attempt}" -lt 4 ]; then
+              echo "::warning::helm push of $(basename "$1") failed (attempt ${attempt}); retrying in $((attempt * 10))s."
+              sleep $((attempt * 10))
+            fi
+          done
+          return 1
+        }
         mkdir -p /tmp/sha-pin
         for source_dir in "${GATEWAY_CHART_DIR}" "${WORKSPACE_CHART_DIR}"; do
           chart_name="$(basename "${source_dir}")"
@@ -117,5 +147,5 @@ runs:
           helm package "${sha_chart_dir}" --destination /tmp/sha-pin
         done
         for archive in /tmp/sha-pin/*.tgz; do
-          helm push "${archive}" oci://ghcr.io/nvidia/openshell
+          push_chart "${archive}"
         done