OpenZeppelin · 0xisk · Jul 15, 2026
@@ -0,0 +1,213 @@
+name: Live Test Suite
+
+env:
+  TURBO_TELEMETRY_DISABLED: 1
+
+# The live suite runs the live-ready categories against a real local stack
+# (node + indexer + proof-server via `make env-up`) with real ZK proofs, so a
+# run takes hours, not minutes. It is deliberately separate from the regular
+# PR checks and never required:
+#   - nightly on main, as the regression safety net (failures are mirrored
+#     into a `live-nightly` tracking issue)
+#   - on demand via workflow_dispatch (optionally scoped to one category or
+#     file filter)
+#   - on a PR, opt-in, by applying the `live-tests` label (re-apply to re-run)
+#
+# A plan job derives the live-ready list from the runner (`test:live --list`,
+# backed by LIVE_READY in scripts/test-live.ts) and fans out one matrix job
+# per category, so each category gets its own runner, stack, and 6-hour
+# job budget.
+on:
+  schedule:
+    - cron: "0 2 * * *"
+  workflow_dispatch:
+    inputs:
+      category:
+        description: "Live-ready category to run"
+        type: choice
+        # Mirror LIVE_READY in scripts/test-live.ts as categories join. A stale
+        # entry fails loudly: the runner rejects non-live-ready names (exit 2).
+        options:
+          - all
+          - multisig
+        default: all
+      filter:
+        description: "Vitest file filter within the category (e.g. Forwarder)"
+        required: false
+        default: ""
+  pull_request:
+    types: [labeled]
+
+concurrency:
+  group: live-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  plan:
+    name: Plan categories
+    if: github.event_name != 'pull_request' || github.event.label.name == 'live-tests'
+    runs-on: ubuntu-24.04
+    permissions:
+      contents: read
+    timeout-minutes: 5
+    outputs:
+      categories: ${{ steps.list.outputs.categories }}
+
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4
+        with:
+          egress-policy: audit
+
+      - name: Check out code
+        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+
+      # The list script only uses node builtins — no yarn install needed.
+      - name: Setup Node.js
+        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
+        with:
+          node-version-file: ".nvmrc"
+
+      - name: List live-ready categories
+        id: list
+        env:
+          CATEGORY: ${{ inputs.category }}
+        run: |
+          if [ -n "$CATEGORY" ] && [ "$CATEGORY" != "all" ]; then
+            echo "categories=[\"$CATEGORY\"]" >> "$GITHUB_OUTPUT"
+          else
+            echo "categories=$(node scripts/test-live.ts --list)" >> "$GITHUB_OUTPUT"
+          fi
+
+  run-live-suite:
+    name: live-${{ matrix.category }}
+    needs: plan
+    strategy:
+      fail-fast: false # one category failing should not cancel the others
+      matrix:
+        category: ${{ fromJSON(needs.plan.outputs.categories) }}
+    runs-on: ubuntu-24.04
+    permissions:
+      contents: read
+    # Per-category budget: the runner compiles (real ZK keys), resets the
+    # stack, and verifies failures with a second round — budget generously but
+    # stay under the 6h hosted-runner cap.
+    timeout-minutes: 350
+
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4
+        with:
+          egress-policy: audit
+
+      - name: Check out code
+        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+        with:
+          fetch-depth: 2 # Recommended by turbo team
+
+      - name: Setup Environment
+        uses: ./.github/actions/setup
+
+      - name: Run live suite
+        env:
+          CATEGORY: ${{ matrix.category }}
+          FILTER: ${{ inputs.filter }}
+        run: yarn test:live "$CATEGORY" $FILTER
+
+      - name: Stop local stack
+        if: always()
+        run: make env-down
+
+      # The JSON verdict reports are tiny and useful on green runs too (flaky
+      # trends); the service/worker logs can reach hundreds of MB after a
+      # multi-hour run, so those only upload when something failed.
+      - name: Upload live reports
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: live-reports-${{ matrix.category }}
+          if-no-files-found: ignore
+          retention-days: 14
+          path: logs/live-*.json
+
+      - name: Upload live logs
+        if: failure()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: live-logs-${{ matrix.category }}
+          if-no-files-found: ignore
+          retention-days: 7
+          path: |
+            logs/live-harness-*.log
+            logs/node.log
+            logs/indexer.log
+            logs/proof-server.log
+
+  # A failed scheduled run only emails the workflow author, which is easy to
+  # miss — mirror the nightly state into a `live-nightly` tracking issue
+  # instead: opened (or commented) on failure, closed on the next green run.
+  report-nightly:
+    name: Report nightly result
+    needs: [plan, run-live-suite]
+    if: always() && github.event_name == 'schedule'
+    runs-on: ubuntu-24.04
+    permissions:
+      contents: read
+      issues: write
+    timeout-minutes: 5
+
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4
+        with:
+          egress-policy: audit
+
+      - name: Open, update, or close the tracking issue
+        env:
+          GH_TOKEN: ${{ github.token }}
+          REPO: ${{ github.repository }}
+          SUITE_RESULT: ${{ needs.run-live-suite.result }}
+          PLAN_RESULT: ${{ needs.plan.result }}
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        run: |
+          # The suite is `skipped` when the plan job died before it — that is
+          # still a failed nightly. A cancelled run (superseded) reports nothing.
+          if [ "$SUITE_RESULT" = "skipped" ] && [ "$PLAN_RESULT" = "failure" ]; then
+            SUITE_RESULT="failure"
+          fi
+
+          issue=$(gh issue list --repo "$REPO" --label live-nightly --state open \
+            --json number --jq '.[0].number')
+
+          case "$SUITE_RESULT" in
+            success)
+              if [ -n "$issue" ]; then
+                gh issue close "$issue" --repo "$REPO" \
+                  --comment "Nightly live run is green again: $RUN_URL"
+              fi
+              ;;
+            failure)
+              if [ -n "$issue" ]; then
+                gh issue comment "$issue" --repo "$REPO" \
+                  --body "Nightly live run failed again: $RUN_URL"
+              else
+                gh label create live-nightly --repo "$REPO" --force \
+                  --description "Tracks the nightly live test run" --color B60205
+                body=$(cat <<EOF
+          The scheduled live test run failed.
+
+          * Run: $RUN_URL
+          * Per-category verdicts are in each job's summary; JSON reports and service logs are attached as artifacts.
+
+          Managed by \`live.yml\`: each failing nightly adds a comment here, and the issue closes automatically on the next green run.
+          EOF
+          )
+                gh issue create --repo "$REPO" --label live-nightly \
+                  --title "Nightly live test run is failing" \
+                  --body "$body"
+              fi
+              ;;
+            *)
+              echo "suite result: $SUITE_RESULT — nothing to report."
+              ;;
+          esac
@@ -227,6 +227,16 @@ Environment knobs:
 >
 > The file stores ANSI color codes, so render them rather than reading them raw. In VS Code, an ANSI extension such as [`iliazeus.vscode-ansi`](https://marketplace.visualstudio.com/items?itemName=iliazeus.vscode-ansi) renders a `.ansi` file via **"ANSI Text: Open Preview"**. In a terminal, use `less -R logs/live-multisig.ansi`. On Linux, prefix `systemd-inhibit --why="live tests"` for a long run.
 
+#### Live tests in CI
+
+The live suite is too slow for the regular PR checks (hours, not minutes), so [`live.yml`](./.github/workflows/live.yml) runs it separately and is never a required check:
+
+* **Nightly** on `main` — the regression safety net. A failed nightly opens (or comments on) a `live-nightly` tracking issue, which closes automatically on the next green run.
+* **On demand** — trigger `Live Test Suite` from the Actions tab (or `gh workflow run live.yml`), optionally scoped with the `category` / `filter` inputs.
+* **On a PR** — apply the `live-tests` label. Re-apply it (or re-run the workflow) for a fresh run after new pushes.
+
+A plan job reads the live-ready list from the runner (`yarn test:live --list`) and fans out one job per category (`live-multisig`, ...), so each category gets its own runner, stack, and 6-hour job budget. Every job runs the same `yarn test:live` runner as a local run, so the two-round flake semantics apply unchanged. Each job's verdict (with any flaky files) lands in its GitHub job summary, the JSON verdict reports upload as a `live-reports-<category>` artifact on every run, and service/worker logs as `live-logs-<category>` on failure.
+
 ## Styleguides
 
 ### TypeScript Styleguide