From 727ae97986668da6beab61c04db12cde6afc9911 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Mon, 6 Jul 2026 15:20:31 +0300 Subject: [PATCH] Improve CI stability: make kubeconform schema validation resilient to transient download failures The chart-testing lint job intermittently fails when kubeconform cannot download schemas from raw.githubusercontent.com (throttling of GitHub Actions runners). Each kubeconform invocation re-downloaded the same schemas, ~21 values files x 10 k8s versions per run. - cache downloaded schemas with kubeconform's -cache option so each schema is downloaded at most once per run - persist the schema cache across runs with actions/cache - retry kubeconform up to 5 times when the failure is a schema download error; cached schemas are not re-downloaded on retry, and genuine validation failures fail immediately without retrying - render helm template output to a file so validation can be retried without re-running helm --- .github/workflows/pulsar-helm-chart-ci.yaml | 29 ++++++++++++++++++--- hack/common.sh | 23 ++++++++++++++++ 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pulsar-helm-chart-ci.yaml b/.github/workflows/pulsar-helm-chart-ci.yaml index 6507fc10..7a9d2a90 100644 --- a/.github/workflows/pulsar-helm-chart-ci.yaml +++ b/.github/workflows/pulsar-helm-chart-ci.yaml @@ -132,6 +132,16 @@ jobs: --validate-maintainers=false \ --target-branch ${{ github.event.repository.default_branch }} + - name: Restore kubeconform schema cache + id: kubeconform-schema-cache + if: ${{ steps.check_changes.outputs.docs_only != 'true' }} + uses: actions/cache/restore@v4 + with: + path: output/kubeconform-cache + key: kubeconform-schema-cache-${{ hashFiles('.github/workflows/pulsar-helm-chart-ci.yaml') }} + restore-keys: | + kubeconform-schema-cache- + - name: Run kubeconform check for helm template with every major k8s version 1.25.0-1.34.0 if: ${{ steps.check_changes.outputs.docs_only != 'true' }} run: | @@ -152,8 +162,10 @@ jobs: else echo "" fi - helm template charts/pulsar --set victoria-metrics-k8s-stack.enabled=false --set components.pulsar_manager=true --kube-version $kube_version "$@" | \ - kubeconform -schema-location default -schema-location 'https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json' -strict -kubernetes-version $kube_version -summary + # render to a file so that validation can be retried without re-running helm + local rendered_manifests="$RUNNER_TEMP/kubeconform-input.yaml" + helm template charts/pulsar --set victoria-metrics-k8s-stack.enabled=false --set components.pulsar_manager=true --kube-version $kube_version "$@" > "$rendered_manifests" + hack::kubeconform_with_retries -schema-location default -schema-location 'https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json' -strict -kubernetes-version $kube_version -summary "$rendered_manifests" } set -o pipefail for k8s_version_part in {25..34}; do @@ -200,7 +212,7 @@ jobs: echo "::endgroup::" echo "::group::kubeconform validate (base)" - kubeconform -schema-location default -schema-location "$schema_crd_url" \ + hack::kubeconform_with_retries -schema-location default -schema-location "$schema_crd_url" \ -strict -kubernetes-version $kube_version -summary "$render_dir" echo "::endgroup::" @@ -213,10 +225,19 @@ jobs: echo "::endgroup::" echo "::group::kubeconform validate (patch1)" - kubeconform -schema-location default -schema-location "$schema_crd_url" \ + hack::kubeconform_with_retries -schema-location default -schema-location "$schema_crd_url" \ -strict -kubernetes-version $kube_version -summary "$render_dir_patch1" echo "::endgroup::" + - name: Save kubeconform schema cache + # save the cache even when validation failed so that already downloaded + # schemas don't get re-downloaded on the next attempt + if: ${{ always() && steps.check_changes.outputs.docs_only != 'true' && steps.kubeconform-schema-cache.outputs.cache-hit != 'true' }} + uses: actions/cache/save@v4 + with: + path: output/kubeconform-cache + key: kubeconform-schema-cache-${{ hashFiles('.github/workflows/pulsar-helm-chart-ci.yaml') }} + - name: Validate kustomize yaml for extra new lines in pulsar-init commands if: ${{ steps.check_changes.outputs.docs_only != 'true' }} run: | diff --git a/hack/common.sh b/hack/common.sh index 5f4f4ffa..8f5b930d 100755 --- a/hack/common.sh +++ b/hack/common.sh @@ -35,6 +35,7 @@ CR_BIN=$OUTPUT_BIN/cr : "${CR_VERSION:=1.7.0}" KUBECONFORM_BIN=$OUTPUT_BIN/kubeconform : "${KUBECONFORM_VERSION:=0.6.7}" +: "${KUBECONFORM_CACHE_DIR:=$OUTPUT/kubeconform-cache}" export PATH="$OUTPUT_BIN:$PATH" test -d "$OUTPUT_BIN" || mkdir -p "$OUTPUT_BIN" @@ -146,3 +147,25 @@ function hack::ensure_kubeconform() { curl -s --retry 10 -L https://github.com/yannh/kubeconform/releases/download/v"${KUBECONFORM_VERSION}"/kubeconform-"${OS}"-"${ARCH}".tar.gz | tar -xzO kubeconform > "$KUBECONFORM_BIN" chmod +x "$KUBECONFORM_BIN" } + +# Runs kubeconform with a persistent schema cache, retrying only when the +# failure is a schema download error (raw.githubusercontent.com throttles +# GitHub Actions runners intermittently). Schemas already in the cache are +# not re-downloaded on retry, so each retry only fetches the missing ones. +function hack::kubeconform_with_retries() { + test -d "$KUBECONFORM_CACHE_DIR" || mkdir -p "$KUBECONFORM_CACHE_DIR" + local max_attempts=5 attempt exit_code output + for ((attempt = 1; attempt <= max_attempts; attempt++)); do + exit_code=0 + output=$(kubeconform -cache "$KUBECONFORM_CACHE_DIR" "$@" 2>&1) || exit_code=$? + echo "$output" + if [ $exit_code -eq 0 ] || ! grep -q "failed downloading schema" <<< "$output"; then + return $exit_code + fi + if [ $attempt -lt $max_attempts ]; then + echo "kubeconform failed to download schemas (attempt $attempt/$max_attempts), retrying in 5 seconds..." + sleep 5 + fi + done + return $exit_code +}