diff --git a/.github/workflows/live.yml b/.github/workflows/live.yml new file mode 100644 index 00000000..814842ef --- /dev/null +++ b/.github/workflows/live.yml @@ -0,0 +1,213 @@ +name: Live Test Suite + +env: + TURBO_TELEMETRY_DISABLED: 1 + +# The live suite runs the live-ready categories against a real local stack +# (node + indexer + proof-server via `make env-up`) with real ZK proofs, so a +# run takes hours, not minutes. It is deliberately separate from the regular +# PR checks and never required: +# - nightly on main, as the regression safety net (failures are mirrored +# into a `live-nightly` tracking issue) +# - on demand via workflow_dispatch (optionally scoped to one category or +# file filter) +# - on a PR, opt-in, by applying the `live-tests` label (re-apply to re-run) +# +# A plan job derives the live-ready list from the runner (`test:live --list`, +# backed by LIVE_READY in scripts/test-live.ts) and fans out one matrix job +# per category, so each category gets its own runner, stack, and 6-hour +# job budget. +on: + schedule: + - cron: "0 2 * * *" + workflow_dispatch: + inputs: + category: + description: "Live-ready category to run" + type: choice + # Mirror LIVE_READY in scripts/test-live.ts as categories join. A stale + # entry fails loudly: the runner rejects non-live-ready names (exit 2). + options: + - all + - multisig + default: all + filter: + description: "Vitest file filter within the category (e.g. Forwarder)" + required: false + default: "" + pull_request: + types: [labeled] + +concurrency: + group: live-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + plan: + name: Plan categories + if: github.event_name != 'pull_request' || github.event.label.name == 'live-tests' + runs-on: ubuntu-24.04 + permissions: + contents: read + timeout-minutes: 5 + outputs: + categories: ${{ steps.list.outputs.categories }} + + steps: + - name: Harden Runner + uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4 + with: + egress-policy: audit + + - name: Check out code + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + + # The list script only uses node builtins — no yarn install needed. + - name: Setup Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version-file: ".nvmrc" + + - name: List live-ready categories + id: list + env: + CATEGORY: ${{ inputs.category }} + run: | + if [ -n "$CATEGORY" ] && [ "$CATEGORY" != "all" ]; then + echo "categories=[\"$CATEGORY\"]" >> "$GITHUB_OUTPUT" + else + echo "categories=$(node scripts/test-live.ts --list)" >> "$GITHUB_OUTPUT" + fi + + run-live-suite: + name: live-${{ matrix.category }} + needs: plan + strategy: + fail-fast: false # one category failing should not cancel the others + matrix: + category: ${{ fromJSON(needs.plan.outputs.categories) }} + runs-on: ubuntu-24.04 + permissions: + contents: read + # Per-category budget: the runner compiles (real ZK keys), resets the + # stack, and verifies failures with a second round — budget generously but + # stay under the 6h hosted-runner cap. + timeout-minutes: 350 + + steps: + - name: Harden Runner + uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4 + with: + egress-policy: audit + + - name: Check out code + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + fetch-depth: 2 # Recommended by turbo team + + - name: Setup Environment + uses: ./.github/actions/setup + + - name: Run live suite + env: + CATEGORY: ${{ matrix.category }} + FILTER: ${{ inputs.filter }} + run: yarn test:live "$CATEGORY" $FILTER + + - name: Stop local stack + if: always() + run: make env-down + + # The JSON verdict reports are tiny and useful on green runs too (flaky + # trends); the service/worker logs can reach hundreds of MB after a + # multi-hour run, so those only upload when something failed. + - name: Upload live reports + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: live-reports-${{ matrix.category }} + if-no-files-found: ignore + retention-days: 14 + path: logs/live-*.json + + - name: Upload live logs + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: live-logs-${{ matrix.category }} + if-no-files-found: ignore + retention-days: 7 + path: | + logs/live-harness-*.log + logs/node.log + logs/indexer.log + logs/proof-server.log + + # A failed scheduled run only emails the workflow author, which is easy to + # miss — mirror the nightly state into a `live-nightly` tracking issue + # instead: opened (or commented) on failure, closed on the next green run. + report-nightly: + name: Report nightly result + needs: [plan, run-live-suite] + if: always() && github.event_name == 'schedule' + runs-on: ubuntu-24.04 + permissions: + contents: read + issues: write + timeout-minutes: 5 + + steps: + - name: Harden Runner + uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4 + with: + egress-policy: audit + + - name: Open, update, or close the tracking issue + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + SUITE_RESULT: ${{ needs.run-live-suite.result }} + PLAN_RESULT: ${{ needs.plan.result }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + # The suite is `skipped` when the plan job died before it — that is + # still a failed nightly. A cancelled run (superseded) reports nothing. + if [ "$SUITE_RESULT" = "skipped" ] && [ "$PLAN_RESULT" = "failure" ]; then + SUITE_RESULT="failure" + fi + + issue=$(gh issue list --repo "$REPO" --label live-nightly --state open \ + --json number --jq '.[0].number') + + case "$SUITE_RESULT" in + success) + if [ -n "$issue" ]; then + gh issue close "$issue" --repo "$REPO" \ + --comment "Nightly live run is green again: $RUN_URL" + fi + ;; + failure) + if [ -n "$issue" ]; then + gh issue comment "$issue" --repo "$REPO" \ + --body "Nightly live run failed again: $RUN_URL" + else + gh label create live-nightly --repo "$REPO" --force \ + --description "Tracks the nightly live test run" --color B60205 + body=$(cat < > The file stores ANSI color codes, so render them rather than reading them raw. In VS Code, an ANSI extension such as [`iliazeus.vscode-ansi`](https://marketplace.visualstudio.com/items?itemName=iliazeus.vscode-ansi) renders a `.ansi` file via **"ANSI Text: Open Preview"**. In a terminal, use `less -R logs/live-multisig.ansi`. On Linux, prefix `systemd-inhibit --why="live tests"` for a long run. +#### Live tests in CI + +The live suite is too slow for the regular PR checks (hours, not minutes), so [`live.yml`](./.github/workflows/live.yml) runs it separately and is never a required check: + +* **Nightly** on `main` — the regression safety net. A failed nightly opens (or comments on) a `live-nightly` tracking issue, which closes automatically on the next green run. +* **On demand** — trigger `Live Test Suite` from the Actions tab (or `gh workflow run live.yml`), optionally scoped with the `category` / `filter` inputs. +* **On a PR** — apply the `live-tests` label. Re-apply it (or re-run the workflow) for a fresh run after new pushes. + +A plan job reads the live-ready list from the runner (`yarn test:live --list`) and fans out one job per category (`live-multisig`, ...), so each category gets its own runner, stack, and 6-hour job budget. Every job runs the same `yarn test:live` runner as a local run, so the two-round flake semantics apply unchanged. Each job's verdict (with any flaky files) lands in its GitHub job summary, the JSON verdict reports upload as a `live-reports-` artifact on every run, and service/worker logs as `live-logs-` on failure. + ## Styleguides ### TypeScript Styleguide