diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index acb539df2d0..06773a77b41 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,6 +13,7 @@ on: - 'Makefile' - 'actions/setup/**' - '.github/workflows/ci.yml' + - '.github/workflows/*.md' - '.github/aw/releases.json' - '.github/aw/releases.schema.json' - '.github/aw/compat.json' @@ -408,6 +409,35 @@ jobs: - name: Compile pkg/cli/workflows run: make compile-cli-workflows + check-workflow-drift: + name: Check workflow markdown/lock drift + if: ${{ needs.changes.outputs.has_changes == 'true' }} + needs: + - changes + - update + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + concurrency: + group: ci-${{ github.ref }}-check-workflow-drift + cancel-in-progress: true + steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Download gh-aw binary artifact + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4 + with: + name: gh-aw-linux-amd64 + path: . + + - name: Prepare gh-aw binary + run: chmod +x ./gh-aw + + - name: Check workflow drift + run: bash scripts/check-workflow-drift.sh ./gh-aw + js-integration-live-api: if: ${{ needs.changes.outputs.has_changes == 'true' && (github.ref == 'refs/heads/main') }} needs: diff --git a/Makefile b/Makefile index e63fe8ec838..99dfa1c477c 100644 --- a/Makefile +++ b/Makefile @@ -775,6 +775,17 @@ lint-lock: build @echo "Linting committed lock files with gh aw lint..." ./$(BINARY_NAME) lint +# Check for drift between workflow markdown sources and generated lock files. +# Compiles all .github/workflows/*.md files and fails if any .lock.yml would +# change, reminding contributors to run 'make recompile' before committing. +.PHONY: check-workflow-drift +check-workflow-drift: + @if [ ! -x "./$(BINARY_NAME)" ]; then \ + echo "./$(BINARY_NAME) not found; building it first..."; \ + $(MAKE) build; \ + fi + @bash scripts/check-workflow-drift.sh ./$(BINARY_NAME) + # Format code .PHONY: fmt fmt: fmt-go fmt-cjs fmt-json @@ -1083,10 +1094,10 @@ agent-finish: deps-dev fmt lint build build-wasm test-all validate-otel-contract # Lightweight pre-PR gate — run before every report_progress / create_pull_request call. # Includes formatting + lint validation to prevent lint-fix PR churn: -# build + fmt + lint + test-unit. +# build + fmt + lint + test-unit + workflow drift check. .PHONY: agent-report-progress -agent-report-progress: build fmt lint test-unit - @echo "Pre-PR validation passed (zero lint errors). Safe to call report_progress." +agent-report-progress: build fmt lint test-unit check-workflow-drift + @echo "Pre-PR validation passed (zero lint errors, lock files in sync). Safe to call report_progress." # Extended pre-PR gate with lock-file-only linting. .PHONY: agent-report-progress-lint @@ -1157,6 +1168,7 @@ help: @echo " actionlint - Validate workflows with actionlint (depends on build)" @echo " lint-lock - Run lock-file-only lint with gh aw lint (depends on build)" @echo " validate-workflows - Validate compiled workflow lock files (depends on build)" + @echo " check-workflow-drift - Check for drift between .md sources and .lock.yml files (builds binary if missing)" @echo " install - Install binary locally" @echo " sync-action-pins - Sync actions-lock.json from .github/aw to pkg/actionpins/data and pkg/workflow/data (runs automatically during build)" @echo " sync-action-scripts - Sync install-gh-aw.sh to actions/setup-cli/install.sh (runs automatically during build)" @@ -1176,7 +1188,7 @@ help: @echo " clean-docs - Clean documentation artifacts (dist, node_modules, .astro)" @echo " agent-finish - Complete validation sequence (build, test, fix, recompile, fmt, lint, security-scan)" - @echo " agent-report-progress - Lightweight pre-PR gate: build + fmt + lint + test-unit" + @echo " agent-report-progress - Lightweight pre-PR gate: build + fmt + lint + test-unit + check-workflow-drift" @echo " agent-report-progress-lint - Pre-PR gate + gh aw lint lock-file check" @echo " sbom - Generate SBOM in SPDX and CycloneDX formats (requires syft)" @echo " help - Show this help message" diff --git a/scripts/check-workflow-drift.sh b/scripts/check-workflow-drift.sh new file mode 100755 index 00000000000..0e19a3de369 --- /dev/null +++ b/scripts/check-workflow-drift.sh @@ -0,0 +1,148 @@ +#!/bin/bash +set +o histexpand + +# check-workflow-drift.sh - Detect drift between workflow markdown sources and generated lock files +# +# Compiles all .github/workflows/*.md files and then checks whether the resulting +# .lock.yml files match what is already on disk. Exits non-zero and prints a clear +# remediation message if any drift is detected. +# +# Usage: +# check-workflow-drift.sh +# +# Arguments: +# Path to the gh-aw binary. +# +# Exit codes: +# 0 - Lock files are up to date with their markdown sources +# 1 - Drift detected (lock files need to be regenerated) + +set -euo pipefail + +# Disable colors when not connected to a TTY, when NO_COLOR is set, or when +# TERM=dumb — this keeps output readable when captured into CI step summaries. +if [ -t 1 ] && [ -z "${NO_COLOR:-}" ] && [ "${TERM:-}" != "dumb" ]; then + RED='\033[0;31m' + GREEN='\033[0;32m' + YELLOW='\033[1;33m' + NC='\033[0m' +else + RED='' + GREEN='' + YELLOW='' + NC='' +fi + +BINARY="${1:?Usage: check-workflow-drift.sh }" + +if [ ! -e "$BINARY" ]; then + echo -e "${RED}ERROR${NC}: gh-aw binary not found at '$BINARY'." + echo "" + echo "Build or download the binary first, then rerun:" + echo "" + echo " make build" + echo " bash scripts/check-workflow-drift.sh ./gh-aw" + exit 1 +fi + +if [ ! -x "$BINARY" ]; then + echo -e "${RED}ERROR${NC}: gh-aw binary at '$BINARY' is not executable." + echo "" + echo "Mark it executable (or rebuild it), then rerun:" + echo "" + echo " chmod +x '$BINARY'" + echo " bash scripts/check-workflow-drift.sh '$BINARY'" + exit 1 +fi + +SNAPSHOT_DIR=$(mktemp -d) +PRE_LOCKFILES_FILE="$SNAPSHOT_DIR/pre-lockfiles.txt" +POST_LOCKFILES_FILE="$SNAPSHOT_DIR/post-lockfiles.txt" +SNAPSHOT_ROOT="$SNAPSHOT_DIR/original" + +restore_lockfiles() { + local file + + find .github/workflows -maxdepth 1 -type f -name '*.lock.yml' -delete + + while IFS= read -r file; do + [ -n "$file" ] || continue + mkdir -p "$(dirname "$file")" + cp "$SNAPSHOT_ROOT/$file" "$file" + done < "$PRE_LOCKFILES_FILE" +} + +cleanup() { + if [ -d "$SNAPSHOT_DIR" ]; then + restore_lockfiles + rm -rf "$SNAPSHOT_DIR" + fi +} +trap cleanup EXIT + +find .github/workflows -maxdepth 1 -type f -name '*.lock.yml' | LC_ALL=C sort > "$PRE_LOCKFILES_FILE" +while IFS= read -r file; do + [ -n "$file" ] || continue + mkdir -p "$SNAPSHOT_ROOT/$(dirname "$file")" + cp "$file" "$SNAPSHOT_ROOT/$file" +done < "$PRE_LOCKFILES_FILE" + +echo "Checking for workflow markdown/lock file drift..." +echo "" + +# Compile all workflow markdown files, regenerating lock files in place. +# --validate: enforce schema validation so compilation errors surface clearly +# --no-check-update: skip version-update check (CI-safe, avoids network calls) +# --purge: remove orphaned .lock.yml files whose .md source was deleted +# +# Snapshot and restore the current lock files so the check never leaves the +# caller's working tree dirty, even when compilation rewrites or purges files. +if ! "$BINARY" compile --validate --no-check-update --purge 2>&1; then + echo "" + echo -e "${RED}ERROR${NC}: Workflow compilation failed — fix the errors above, then run:" + echo "" + echo " make recompile" + echo "" + exit 1 +fi + +# Collect lock files that changed, appeared, or were deleted relative to the +# pre-compilation snapshot. +find .github/workflows -maxdepth 1 -type f -name '*.lock.yml' | LC_ALL=C sort > "$POST_LOCKFILES_FILE" +all_drift=$( + cat "$PRE_LOCKFILES_FILE" "$POST_LOCKFILES_FILE" \ + | sed '/^$/d' \ + | LC_ALL=C sort -u \ + | while IFS= read -r file; do + if ! grep -Fxq "$file" "$PRE_LOCKFILES_FILE"; then + printf '%s\n' "$file" + elif ! grep -Fxq "$file" "$POST_LOCKFILES_FILE"; then + printf '%s\n' "$file" + elif ! cmp -s "$SNAPSHOT_ROOT/$file" "$file"; then + printf '%s\n' "$file" + fi + done +) + +if [ -z "$all_drift" ]; then + echo -e "${GREEN}✓ All workflow lock files are in sync with their markdown sources.${NC}" + exit 0 +fi + +echo "" +echo -e "${RED}ERROR: Workflow lock files are out of sync with their markdown sources.${NC}" +echo "" +echo "The following lock files differ from what would be generated by compilation:" +echo "" +while IFS= read -r file; do + echo " $file" +done <<< "$all_drift" +echo "" +echo -e "${YELLOW}Fix:${NC} Regenerate and stage the lock files, then use report_progress:" +echo "" +echo " make recompile" +echo " git add .github/workflows/*.lock.yml" +echo " # then call report_progress to commit and push via the pre-PR gate" +echo "" +echo "Lock files must always be committed together with their .md sources." +exit 1 diff --git a/scripts/check-workflow-drift_test.sh b/scripts/check-workflow-drift_test.sh new file mode 100644 index 00000000000..104464d75fb --- /dev/null +++ b/scripts/check-workflow-drift_test.sh @@ -0,0 +1,117 @@ +#!/bin/bash +set +o histexpand + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +DRIFT_SCRIPT="$SCRIPT_DIR/check-workflow-drift.sh" + +TESTS_PASSED=0 +TESTS_FAILED=0 + +pass() { echo "PASS: $1"; TESTS_PASSED=$((TESTS_PASSED + 1)); } +fail() { echo "FAIL: $1"; echo " $2"; TESTS_FAILED=$((TESTS_FAILED + 1)); } + +create_fixture_repo() { + local repo_dir="$1" + + mkdir -p "$repo_dir/.github/workflows" + cat > "$repo_dir/.github/workflows/example.md" <<'EOF' +# Example workflow +EOF + cat > "$repo_dir/.github/workflows/example.lock.yml" <<'EOF' +lock: original +EOF +} + +create_fake_binary() { + local path="$1" + cat > "$path" <<'EOF' +#!/bin/bash +set -euo pipefail + +if [ "${1:-}" != "compile" ]; then + echo "unexpected command: ${1:-}" >&2 + exit 1 +fi + +case "${FAKE_COMPILE_MODE:-stable}" in + stable) + ;; + mutate) + cat > .github/workflows/example.lock.yml <<'OUT' +lock: mutated +OUT + ;; + fail) + echo "compile failed" >&2 + exit 1 + ;; + *) + echo "unknown FAKE_COMPILE_MODE: ${FAKE_COMPILE_MODE:-}" >&2 + exit 1 + ;; +esac +EOF + chmod +x "$path" +} + +echo "Running check-workflow-drift.sh tests..." +echo + +TMP_ROOT=$(mktemp -d) +trap 'rm -rf "$TMP_ROOT"' EXIT +TEST1_OUTPUT="$TMP_ROOT/test1-output.txt" +TEST2_OUTPUT="$TMP_ROOT/test2-output.txt" +TEST3_OUTPUT="$TMP_ROOT/test3-output.txt" + +# Test 1: matching lock file exits 0. +echo "Test 1: matching lock file exits 0..." +TEST_REPO="$TMP_ROOT/stable" +mkdir -p "$TEST_REPO" +create_fixture_repo "$TEST_REPO" +create_fake_binary "$TEST_REPO/fake-gh-aw" +if (cd "$TEST_REPO" && FAKE_COMPILE_MODE=stable bash "$DRIFT_SCRIPT" "$TEST_REPO/fake-gh-aw" >"$TEST1_OUTPUT" 2>&1); then + pass "matching lock file exits 0" +else + fail "matching lock file should exit 0" "$(cat "$TEST1_OUTPUT")" +fi + +# Test 2: drift is reported and the original file is restored afterwards. +echo "Test 2: drift is reported without leaving the repo dirty..." +TEST_REPO="$TMP_ROOT/mutate" +mkdir -p "$TEST_REPO" +create_fixture_repo "$TEST_REPO" +create_fake_binary "$TEST_REPO/fake-gh-aw" +if (cd "$TEST_REPO" && FAKE_COMPILE_MODE=mutate bash "$DRIFT_SCRIPT" "$TEST_REPO/fake-gh-aw" >"$TEST2_OUTPUT" 2>&1); then + fail "drift should exit 1" "$(cat "$TEST2_OUTPUT")" +elif grep -q ".github/workflows/example.lock.yml" "$TEST2_OUTPUT" \ + && grep -q "report_progress" "$TEST2_OUTPUT" \ + && grep -q "^lock: original$" "$TEST_REPO/.github/workflows/example.lock.yml"; then + pass "drift is reported and the original file is restored" +else + fail "drift output or restoration was incorrect" "$(cat "$TEST2_OUTPUT"; echo; cat "$TEST_REPO/.github/workflows/example.lock.yml")" +fi + +# Test 3: missing binary gets a targeted error. +echo "Test 3: missing binary path gets a targeted error..." +TEST_REPO="$TMP_ROOT/missing-binary" +mkdir -p "$TEST_REPO" +create_fixture_repo "$TEST_REPO" +if (cd "$TEST_REPO" && bash "$DRIFT_SCRIPT" "$TEST_REPO/does-not-exist" >"$TEST3_OUTPUT" 2>&1); then + fail "missing binary should exit 1" "$(cat "$TEST3_OUTPUT")" +elif grep -q "binary not found" "$TEST3_OUTPUT"; then + pass "missing binary reports a targeted error" +else + fail "missing binary error message was incorrect" "$(cat "$TEST3_OUTPUT")" +fi + +echo +echo "Tests passed: $TESTS_PASSED" +echo "Tests failed: $TESTS_FAILED" + +if [ "$TESTS_FAILED" -gt 0 ]; then + exit 1 +fi + +echo "✓ All tests passed!"