Improve CI stability: make kubeconform schema validation resilient to transient download failures

.github/workflows/pulsar-helm-chart-ci.yaml

@@ -132,6 +132,16 @@ jobs:
             --validate-maintainers=false \
             --target-branch ${{ github.event.repository.default_branch }}
 
+      - name: Restore kubeconform schema cache
+        id: kubeconform-schema-cache
+        if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
+        uses: actions/cache/restore@v4
+        with:
+          path: output/kubeconform-cache
+          key: kubeconform-schema-cache-${{ hashFiles('.github/workflows/pulsar-helm-chart-ci.yaml') }}
+          restore-keys: |
+            kubeconform-schema-cache-
+
       - name: Run kubeconform check for helm template with every major k8s version 1.25.0-1.34.0
         if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
         run: |
@@ -152,8 +162,10 @@ jobs:
             else
               echo ""
             fi
-            helm template charts/pulsar --set victoria-metrics-k8s-stack.enabled=false --set components.pulsar_manager=true --kube-version $kube_version "$@" | \
-              kubeconform -schema-location default -schema-location 'https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json' -strict -kubernetes-version $kube_version -summary
+            # render to a file so that validation can be retried without re-running helm
+            local rendered_manifests="$RUNNER_TEMP/kubeconform-input.yaml"
+            helm template charts/pulsar --set victoria-metrics-k8s-stack.enabled=false --set components.pulsar_manager=true --kube-version $kube_version "$@" > "$rendered_manifests"
+            hack::kubeconform_with_retries -schema-location default -schema-location 'https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json' -strict -kubernetes-version $kube_version -summary "$rendered_manifests"
           }
           set -o pipefail
           for k8s_version_part in {25..34}; do
@@ -200,7 +212,7 @@ jobs:
           echo "::endgroup::"
 
           echo "::group::kubeconform validate (base)"
-          kubeconform -schema-location default -schema-location "$schema_crd_url" \
+          hack::kubeconform_with_retries -schema-location default -schema-location "$schema_crd_url" \
             -strict -kubernetes-version $kube_version -summary "$render_dir"
           echo "::endgroup::"
 
@@ -213,10 +225,19 @@ jobs:
           echo "::endgroup::"
 
           echo "::group::kubeconform validate (patch1)"
-          kubeconform -schema-location default -schema-location "$schema_crd_url" \
+          hack::kubeconform_with_retries -schema-location default -schema-location "$schema_crd_url" \
             -strict -kubernetes-version $kube_version -summary "$render_dir_patch1"
           echo "::endgroup::"
 
+      - name: Save kubeconform schema cache
+        # save the cache even when validation failed so that already downloaded
+        # schemas don't get re-downloaded on the next attempt
+        if: ${{ always() && steps.check_changes.outputs.docs_only != 'true' && steps.kubeconform-schema-cache.outputs.cache-hit != 'true' }}
+        uses: actions/cache/save@v4
+        with:
+          path: output/kubeconform-cache
+          key: kubeconform-schema-cache-${{ hashFiles('.github/workflows/pulsar-helm-chart-ci.yaml') }}
+
       - name: Validate kustomize yaml for extra new lines in pulsar-init commands
         if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
         run: |

hack/common.sh

@@ -35,6 +35,7 @@ CR_BIN=$OUTPUT_BIN/cr
 : "${CR_VERSION:=1.7.0}"
 KUBECONFORM_BIN=$OUTPUT_BIN/kubeconform
 : "${KUBECONFORM_VERSION:=0.6.7}"
+: "${KUBECONFORM_CACHE_DIR:=$OUTPUT/kubeconform-cache}"
 export PATH="$OUTPUT_BIN:$PATH"
 
 test -d "$OUTPUT_BIN" || mkdir -p "$OUTPUT_BIN"
@@ -146,3 +147,25 @@ function hack::ensure_kubeconform() {
     curl -s --retry 10 -L https://github.com/yannh/kubeconform/releases/download/v"${KUBECONFORM_VERSION}"/kubeconform-"${OS}"-"${ARCH}".tar.gz | tar -xzO kubeconform > "$KUBECONFORM_BIN"
     chmod +x "$KUBECONFORM_BIN"
 }
+
+# Runs kubeconform with a persistent schema cache, retrying only when the
+# failure is a schema download error (raw.githubusercontent.com throttles
+# GitHub Actions runners intermittently). Schemas already in the cache are
+# not re-downloaded on retry, so each retry only fetches the missing ones.
+function hack::kubeconform_with_retries() {
+    test -d "$KUBECONFORM_CACHE_DIR" || mkdir -p "$KUBECONFORM_CACHE_DIR"
+    local max_attempts=5 attempt exit_code output
+    for ((attempt = 1; attempt <= max_attempts; attempt++)); do
+        exit_code=0
+        output=$(kubeconform -cache "$KUBECONFORM_CACHE_DIR" "$@" 2>&1) || exit_code=$?
+        echo "$output"
+        if [ $exit_code -eq 0 ] || ! grep -q "failed downloading schema" <<< "$output"; then
+            return $exit_code
+        fi
+        if [ $attempt -lt $max_attempts ]; then
+            echo "kubeconform failed to download schemas (attempt $attempt/$max_attempts), retrying in 5 seconds..."
+            sleep 5
+        fi
+    done
+    return $exit_code
+}