From 02533780fd96d76681093831ee92226bc7d97cda Mon Sep 17 00:00:00 2001 From: 0xisk <0xisk@proton.me> Date: Mon, 27 Jul 2026 15:30:38 +0200 Subject: [PATCH 01/16] build(turbo): scope compile inputs to .compact sources The `compile` task declared only `dependsOn`, so turbo fell back to hashing every tracked file in the contracts package. Editing a spec or a vitest config was therefore a cache miss on a task that recompiles all 70 contracts with real keygen, which defeats the intent that TS-only edits skip recompilation. The six per-category tasks were cache hits while the top-level task rebuilt everything anyway. Declare what the script actually reads, mirroring the per-category tasks. Measured with `turbo run compile --dry=json`, the resolved input set drops from 214 files to 71, and excludes `src/archive` to match the script's own `--exclude`. Also declare MIDNIGHT_LIVE_KEEP_ENV as a global passthrough: the live orchestrator reads it to skip stack teardown, the same way it already reads GITHUB_ACTIONS and GITHUB_STEP_SUMMARY. Note that declaring SKIP_ZK means `compile` now honours an ambient value where strict env mode previously discarded it. The orchestrator clears it for both compiles so a live run cannot be handed keyless artifacts. --- turbo.json | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/turbo.json b/turbo.json index 98e1a776..1607f5f3 100644 --- a/turbo.json +++ b/turbo.json @@ -1,6 +1,10 @@ { "$schema": "https://turbo.build/schema.json", - "globalPassThroughEnv": ["GITHUB_ACTIONS", "GITHUB_STEP_SUMMARY"], + "globalPassThroughEnv": [ + "GITHUB_ACTIONS", + "GITHUB_STEP_SUMMARY", + "MIDNIGHT_LIVE_KEEP_ENV" + ], "tasks": { "compile:crypto": { "dependsOn": ["^build"], @@ -66,7 +70,11 @@ "compile:access", "compile:multisig", "compile:token" - ] + ], + "env": ["COMPACT_HOME", "SKIP_ZK"], + "inputs": ["src/**/*.compact", "!src/archive/**"], + "outputLogs": "new-only", + "outputs": ["artifacts/**/"] }, "test": { "dependsOn": ["compile"], From 9aa2d7e144296fdb183f0899fdcb47327f1e2fea Mon Sep 17 00:00:00 2001 From: 0xisk <0xisk@proton.me> Date: Mon, 27 Jul 2026 15:30:54 +0200 Subject: [PATCH 02/16] refactor(scripts): scope the key scan by source roots `emptyKeyArtifacts` took a single optional source root and skipped any artifact directory with no matching `.compact` under it, so stale orphan directories could not false-positive. That filter also skipped anything compiled from outside `src`. The integration mocks compile into the same `artifacts/` tree but live under `test/integration/_mocks`, so a `src`-only scan never looked at them. A truncated key in a mock would therefore go undetected, the live deploy would fail in `beforeAll`, and vitest would turn that into a silent whole-suite skip: exactly the failure this check exists to prevent. Take variadic source roots and union the contract names across them. One call site, one pass over `artifacts/`, and the stale-orphan filter still works. The standalone CLI now passes both roots too. --- scripts/keyIntegrity.ts | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/scripts/keyIntegrity.ts b/scripts/keyIntegrity.ts index 389b32ef..329526dd 100644 --- a/scripts/keyIntegrity.ts +++ b/scripts/keyIntegrity.ts @@ -2,12 +2,11 @@ import { existsSync, readdirSync, statSync } from 'node:fs'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; -/** Recursively collect `` basenames of every `.compact` under `root`. +/** Recursively collect `` basenames of every `.compact` under `roots`. * The compiler names each artifact dir after the source file's basename, so this * is the set of contract names the current tree can legitimately produce. */ -function compactContractNames(root: string): Set { +function compactContractNames(...roots: string[]): Set { const names = new Set(); - if (!existsSync(root)) return names; const walk = (dir: string): void => { for (const entry of readdirSync(dir, { withFileTypes: true })) { const p = path.join(dir, entry.name); @@ -16,7 +15,7 @@ function compactContractNames(root: string): Set { names.add(entry.name.slice(0, -'.compact'.length)); } }; - walk(root); + for (const root of roots) if (existsSync(root)) walk(root); return names; } @@ -44,20 +43,27 @@ function collectEmptyKeys(dir: string, out: string[]): void { * real deploy fail in `beforeAll`, which vitest turns into a silent whole-suite * skip. Callers check this before starting the live stack. * - * When `sourceRoot` is given, only contracts that still have a `.compact` source - * under it are checked, so stale orphan artifact dirs (source deleted, keys never - * rebuilt) do not false-positive. Omit it to scan every contract dir. + * When `sourceRoots` are given, only contracts that still have a `.compact` + * source under one of them are checked, so stale orphan artifact dirs (source + * deleted, keys never rebuilt) do not false-positive. Pass none to scan every + * contract dir. + * + * More than one root matters because sources outside `src/` also compile into the + * same `artifacts/` tree: the integration mocks live under + * `test/integration/_mocks`, so a `src`-only scan silently skips them — the live + * integration target passes both roots. * * @param artifactsRoot - artifact tree to scan (e.g. `contracts/artifacts`) - * @param sourceRoot - optional source tree to scope by (e.g. `contracts/src`) + * @param sourceRoots - source trees to scope by (e.g. `contracts/src`) * @returns absolute paths of empty key files; empty array means all good */ export function emptyKeyArtifacts( artifactsRoot: string, - sourceRoot?: string, + ...sourceRoots: string[] ): string[] { if (!existsSync(artifactsRoot)) return []; - const live = sourceRoot ? compactContractNames(sourceRoot) : undefined; + const live = + sourceRoots.length > 0 ? compactContractNames(...sourceRoots) : undefined; const empty: string[] = []; for (const contract of readdirSync(artifactsRoot, { withFileTypes: true })) { if (!contract.isDirectory()) continue; @@ -68,7 +74,8 @@ export function emptyKeyArtifacts( } // Standalone CLI: `node scripts/keyIntegrity.ts` checks the repo's artifacts -// against its sources and exits 1 if any live contract has a truncated key. +// against its sources — `src` plus the integration mocks, the two trees that +// compile into `artifacts/` — and exits 1 if any has a truncated key. const selfPath = fileURLToPath(import.meta.url); if (process.argv[1] && path.resolve(process.argv[1]) === selfPath) { const repoRoot = path.resolve(path.dirname(selfPath), '..'); @@ -76,6 +83,7 @@ if (process.argv[1] && path.resolve(process.argv[1]) === selfPath) { const bad = emptyKeyArtifacts( path.join(contracts, 'artifacts'), path.join(contracts, 'src'), + path.join(contracts, 'test/integration/_mocks'), ); if (bad.length === 0) { console.log('ZK keys OK — no truncated (0-byte) .verifier/.prover files.'); From f262aac90e5368fbaa0e2467d01bce5f02d919b6 Mon Sep 17 00:00:00 2001 From: 0xisk <0xisk@proton.me> Date: Mon, 27 Jul 2026 15:30:54 +0200 Subject: [PATCH 03/16] test(harness): reject two live projects per vitest run Every live project derives its wallets from `walletSeedsFor(VITEST_POOL_ID)`, so worker 1 of `unit-live` and worker 1 of `integration-live` resolve to the same genesis deployer seed. Two pools would balance transactions against one deployer's UTXO snapshot and reintroduce the stale-UTXO race that `WalletPool.ensureReady`'s serial build exists to prevent, and both would write `logs/live-harness-w1.log`. The run lock cannot catch this. It is deliberately reentrant for our own pid, precisely so one process can run several live globalSetups. So claim the process separately. The guard lives in globalSetup rather than in `live.setup`: vitest gives each project its own worker processes, so a per-worker guard never sees the other project, while globalSetup runs once per project in the single main process. The claim is parked on `globalThis` because each project loads the file through its own module runner. It runs before the lock and before the freshness scan, so a mis-invocation fails in milliseconds without touching the node. `assertSoleLiveProject` is exported as a pure function, matching the convention already used for `lockHolderState` and `countCoinEvents`. --- .../test-utils/harness/live.globalSetup.ts | 59 ++++++++++++++++++- 1 file changed, 57 insertions(+), 2 deletions(-) diff --git a/contracts/test-utils/harness/live.globalSetup.ts b/contracts/test-utils/harness/live.globalSetup.ts index e9b35859..049f149d 100644 --- a/contracts/test-utils/harness/live.globalSetup.ts +++ b/contracts/test-utils/harness/live.globalSetup.ts @@ -9,7 +9,11 @@ import { fetchCoinEvents, indexerHead } from './ledgerEvents.js'; * of after a slow wallet build. Gated on `MIDNIGHT_BACKEND === 'live'` so a dry * `vitest run` that happens to glob the live tests is a no-op. * - * It guards two things: + * It guards three things: + * - **One live project per invocation.** Every live project derives its + * wallets from the same `VITEST_POOL_ID` partition, so two of them in one + * vitest run would spend the same genesis deployer's coins. See + * {@link assertSoleLiveProject}. * - **Freshness.** The live tests are not isolated from one another: they all * run against the same node, so shielded-coin state left by an earlier run * changes a later run's outcome (a coin re-spent against stale state is @@ -49,6 +53,50 @@ const ENV_UP_HINT = "run 'yarn env:up' to reset the local stack"; const sleep = (ms: number): Promise => new Promise((resolve) => setTimeout(resolve, ms)); +// --- one live project per invocation --------------------------------------- + +// Parked on `globalThis`, not in a module-level binding: each project loads this +// file through its OWN module runner, so a module variable would not be shared +// between two projects in the same process. +const PROJECT_CLAIM = '__midnightLiveProject'; + +/** + * Reject a second live project in the same vitest invocation. + * + * Every live project builds its wallets from `walletSeedsFor(VITEST_POOL_ID)`, + * so worker 1 of `unit-live` and worker 1 of `integration-live` resolve to the + * SAME genesis deployer seed. Two pools would then balance transactions against + * one deployer's UTXO snapshot — the stale-UTXO race (node `Custom error: 103`) + * that `WalletPool.ensureReady`'s serial build exists to prevent — and both + * would write `logs/live-harness-w1.log`, interleaving two runs' diagnostics. + * + * The run lock below cannot catch this: it is deliberately reentrant for our own + * pid, precisely so one process CAN run several live globalSetups. So the claim + * is tracked separately here. + * + * @param project - the project whose globalSetup is running + * @param claimed - the project that already claimed this process, if any + */ +export function assertSoleLiveProject( + project: string, + claimed: string | undefined, +): void { + if (claimed === undefined || claimed === project) return; + throw new Error( + `two live projects in one vitest run ('${claimed}' and '${project}'): ` + + 'both derive their wallets from the same VITEST_POOL_ID partition, so ' + + "each project's worker 1 would spend the same genesis deployer's coins " + + '(node "Custom error: 103"). Pass one --project per invocation.', + ); +} + +/** Claim this process for `project`, or throw if another live project holds it. */ +function claimLiveProject(project: string): void { + const registry = globalThis as Record; + assertSoleLiveProject(project, registry[PROJECT_CLAIM] as string | undefined); + registry[PROJECT_CLAIM] = project; +} + // --- lock ------------------------------------------------------------------ export interface LockInfo { @@ -206,8 +254,15 @@ async function assertFreshNode(): Promise { } } -export default async function setup(): Promise<() => void> { +/** + * Vitest calls this once per project, in the main process, passing that project. + * Typed structurally so this file keeps importing nothing but `node:` builtins. + */ +export default async function setup(project?: { + readonly name?: string; +}): Promise<() => void> { if (process.env.MIDNIGHT_BACKEND !== 'live') return () => {}; + claimLiveProject(project?.name ?? '(unnamed project)'); const { reentrant } = acquireLock(); try { if (process.env.MIDNIGHT_LIVE_ALLOW_DIRTY !== '1') await assertFreshNode(); From 5058963aaec426dc20e3104ba96edf0363398b89 Mon Sep 17 00:00:00 2001 From: 0xisk <0xisk@proton.me> Date: Mon, 27 Jul 2026 15:31:10 +0200 Subject: [PATCH 04/16] refactor(scripts): split the live orchestrator `test-live.ts` had grown to 478 lines carrying eleven responsibilities: paths, process spawning, console output, category discovery, target resolution, vitest invocation, report parsing, the run lock, compile verification, verdict reporting, and the two-round loop. Split one service per concern under `scripts/live/`, moving each rationale comment to sit with the code it explains. The entry point is now a 99-line main that resolves a plan, wires the services, and runs under the lock. Behaviour is preserved: every console string, all exit codes, and the order of operations are unchanged. Two behavioural additions come with it. The orchestrator now owns the whole stack lifecycle. `env-up` was already automatic and itself resets, so a leaked stack never corrupted the next run, but containers and the `docker compose logs -f` streamers each bring-up backgrounds outlived the process. Teardown runs on every exit path, before the lock release so no other run can start against a half-stopped stack, and from a signal handler because Node runs no `finally` on a signal. MIDNIGHT_LIVE_KEEP_ENV=1 opts out. `run` is asynchronous. With spawnSync the event loop was blocked for a child's whole lifetime, and a compile phase is one long synchronous chain, so a queued SIGINT handler could not run until the phase finished. Ctrl-C during a compile was ignored, the interrupted compile looked successful because turbo exits 0 on its own graceful shutdown, and the truncated keys the interruption had just created were then read as a poisoned cache: draining it and starting a pointless serial recompile. Awaiting the child keeps the loop live. Teardown stays synchronous, since the handler exits as soon as it returns. The integration target is added as a first-class target here: it routes to `--project integration-live`, compiles the mocks with real keys, and is emitted by `--list` so CI can spawn a job for it. It is matched before the category branch because it is not a `src/` directory and would otherwise fall through to the unscoped path, silently running every live-ready unit category instead. --- scripts/live/ArtifactCompiler.ts | 109 +++++++ scripts/live/LiveOrchestrator.ts | 243 +++++++++++++++ scripts/live/LiveStack.ts | 60 ++++ scripts/live/Reporter.ts | 87 ++++++ scripts/live/RunLock.ts | 75 +++++ scripts/live/VitestRunner.ts | 67 ++++ scripts/live/paths.ts | 53 ++++ scripts/live/shell.ts | 94 ++++++ scripts/live/targets.ts | 128 ++++++++ scripts/test-live.ts | 515 ++++--------------------------- 10 files changed, 984 insertions(+), 447 deletions(-) create mode 100644 scripts/live/ArtifactCompiler.ts create mode 100644 scripts/live/LiveOrchestrator.ts create mode 100644 scripts/live/LiveStack.ts create mode 100644 scripts/live/Reporter.ts create mode 100644 scripts/live/RunLock.ts create mode 100644 scripts/live/VitestRunner.ts create mode 100644 scripts/live/paths.ts create mode 100644 scripts/live/shell.ts create mode 100644 scripts/live/targets.ts diff --git a/scripts/live/ArtifactCompiler.ts b/scripts/live/ArtifactCompiler.ts new file mode 100644 index 00000000..8271c021 --- /dev/null +++ b/scripts/live/ArtifactCompiler.ts @@ -0,0 +1,109 @@ +import { rmSync } from 'node:fs'; +import { emptyKeyArtifacts } from '../keyIntegrity.ts'; +import { + ARTIFACTS, + INTEGRATION_MOCKS, + rel, + SRC, + TURBO_CACHE, +} from './paths.ts'; +import { run } from './shell.ts'; + +/** + * Builds the artifacts a live run will deploy, and refuses to hand the run a + * poisoned artifact tree. + * + * A killed compile (or machine crash) can poison the turbo cache so every later + * cache hit re-extracts a truncated key, and a concurrent compile racing over the + * shared `artifacts/` tree can truncate keys directly + * (OpenZeppelin/compact-contracts#675). A 0-byte `.prover` makes the deploy fail + * in `beforeAll`, which vitest turns into a silent whole-suite skip — the failure + * mode this check exists to prevent. Both repairs are mechanical, so self-heal + * once (drain the cache, recompile serially — a parallel recompile can re-poison + * it) and only abort if keys are still truncated afterwards. + */ +export class ArtifactCompiler { + /** Whether this run also needs the integration mocks built with proving keys. */ + readonly #integration: boolean; + + constructor(integration: boolean) { + this.#integration = integration; + } + + /** + * Compile, verify, self-heal once, verify again. + * + * @returns `true` when the artifact tree is safe to deploy from + */ + async compileVerified(): Promise { + if (!(await this.#compileAll([]))) { + console.log('compile failed — a compile error is real, not a flake.'); + return false; + } + const empty = this.#truncatedKeys(); + if (empty.length === 0) return true; + + console.log( + '\ncompile reported success but left truncated (0-byte) ZK key(s):', + ); + for (const k of empty) console.log(` ✗ ${rel(k)}`); + console.log( + '\nPoisoned turbo cache or artifact tree ' + + '(OpenZeppelin/compact-contracts#675) — draining the cache and ' + + 'recompiling serially...', + ); + rmSync(TURBO_CACHE, { recursive: true, force: true }); + if (!(await this.#compileAll(['--concurrency=1']))) { + console.log('serial recompile failed.'); + return false; + } + const stillEmpty = this.#truncatedKeys(); + if (stillEmpty.length === 0) { + console.log('recovered — ZK keys intact after the serial recompile.'); + return true; + } + console.log( + '\nstill truncated after a serial recompile — needs investigation:', + ); + for (const k of stillEmpty) console.log(` ✗ ${rel(k)}`); + return false; + } + + /** + * `src` first, integration mocks second. + * + * Artifact directories are keyed on the source basename, so basenames must stay + * unique across `src/` and `test/integration/_mocks/` — two files sharing one + * would overwrite each other's `artifacts//`. The composed mock is named + * `ComposedConfidentialFungibleTokenPublicSupply.compact` for exactly this + * reason. src-first order is kept as a convention; it is no longer a + * correctness requirement. + * + * BOTH compiles clear `SKIP_ZK` rather than trusting the ambient value: a live + * run always needs real proving keys, and the dry `test:integration` path + * exports `SKIP_ZK=true`. Clearing it here means an ambient value can never + * hand the live path keyless artifacts, whatever turbo's env mode does. turbo + * keys both tasks on `SKIP_ZK`, so dry and full-key builds cache apart. + */ + async #compileAll(extraArgs: string[]): Promise { + const { SKIP_ZK: _skipZk, ...fullKeyEnv } = process.env; + if ((await run('yarn', ['compile', ...extraArgs], fullKeyEnv)) !== 0) { + return false; + } + if (!this.#integration) return true; + return ( + (await run('yarn', ['compile:integration', ...extraArgs], fullKeyEnv)) === + 0 + ); + } + + /** Scoped to the source roots this run deploys from, so a stale orphan + * artifact directory cannot false-positive. */ + #truncatedKeys(): string[] { + return emptyKeyArtifacts( + ARTIFACTS, + SRC, + ...(this.#integration ? [INTEGRATION_MOCKS] : []), + ); + } +} diff --git a/scripts/live/LiveOrchestrator.ts b/scripts/live/LiveOrchestrator.ts new file mode 100644 index 00000000..86a0f7d6 --- /dev/null +++ b/scripts/live/LiveOrchestrator.ts @@ -0,0 +1,243 @@ +import { existsSync, readdirSync, rmSync } from 'node:fs'; +import path from 'node:path'; +import type { ArtifactCompiler } from './ArtifactCompiler.ts'; +import type { LiveStack } from './LiveStack.ts'; +import { + LOGS, + ROUND2_REPORT_PREFIX, + rel, + round1Report, + round2Report, +} from './paths.ts'; +import type { Reporter } from './Reporter.ts'; +import { banner, run } from './shell.ts'; +import type { LivePlan, LiveTarget } from './targets.ts'; +import type { VitestRunner } from './VitestRunner.ts'; + +/** Exit code for an infrastructure abort, as opposed to a test failure (1). */ +export const INFRA_ABORT = 2; + +interface FailedFile { + readonly file: string; + /** The target that ran it, so round 2 re-runs it under the same project. */ + readonly target: LiveTarget; +} + +/** + * Split round-1 failures into flakes and real failures. + * + * Only an explicit round-2 pass demotes a failure to FLAKY; a file that failed + * again — or never reported (crashed) — stays REAL. + */ +export function classify( + files: readonly string[], + round2: ReadonlyMap, +): { flaky: string[]; real: string[] } { + return { + flaky: files.filter((f) => round2.get(f) === 'passed'), + real: files.filter((f) => round2.get(f) !== 'passed'), + }; +} + +/** + * Runs the two-round live verification. + * + * The live specs are not isolated from one another: they all run against one + * shared node, so state left by an earlier test can make a later one fail (a coin + * re-spent against stale node state is rejected with node "Custom error: 103"). + * A file that fails during a busy full run may therefore pass in isolation on a + * fresh node. Hence two rounds: + * + * Round 1: compile + harness smoke once, then per target: reset the stack and + * run that target's files (parallel workers where the project allows + * it). Collect the files that failed from the JSON reporter. + * Round 2: for each failed file, reset the stack and re-run just that file on + * its own (one worker), so no earlier round-2 file can dirty the node + * under a later one. + * + * A file that fails round 1 but passes round 2 is FLAKY (an environment + * artifact); one that fails both — or never reports in round 2 — is a REAL + * failure. The run exits 0 unless there is a real failure, so an env flake never + * turns the build red, but it is reported loudly. + * + * Anything that prevents classification (no report written, a non-zero exit with + * no failing files, a stack that will not come up) aborts with + * {@link INFRA_ABORT} rather than being guessed at. + */ +export class LiveOrchestrator { + readonly #plan: LivePlan; + readonly #stack: LiveStack; + readonly #compiler: ArtifactCompiler; + readonly #runner: VitestRunner; + readonly #reporter: Reporter; + + constructor(deps: { + readonly plan: LivePlan; + readonly stack: LiveStack; + readonly compiler: ArtifactCompiler; + readonly runner: VitestRunner; + readonly reporter: Reporter; + }) { + this.#plan = deps.plan; + this.#stack = deps.stack; + this.#compiler = deps.compiler; + this.#runner = deps.runner; + this.#reporter = deps.reporter; + } + + /** @returns the process exit code */ + async run(): Promise { + this.#clearStaleReports(); + + const { targets, skipped, fileFilters } = this.#plan; + banner( + `ROUND 1 — targets: ${targets.map((t) => t.name).join(', ')}` + + (fileFilters.length ? ` (filter: ${fileFilters.join(' ')})` : ''), + ); + if (skipped.length > 0) { + console.log(`skipped (not yet live-ready): ${skipped.join(', ')}`); + } + + if (!(await this.#compiler.compileVerified())) return INFRA_ABORT; + if ((await this.#stack.up()) !== 0) { + console.log('env-up failed — cannot start the live stack.'); + return INFRA_ABORT; + } + if ((await run('yarn', ['test:harness:live'])) !== 0) { + console.log( + '\nlive harness smoke failed — this is an infrastructure problem, ' + + 'not a spec flake. Fix the stack and retry.', + ); + return INFRA_ABORT; + } + + const failed = await this.#round1(); + if (failed === undefined) return INFRA_ABORT; + if (failed.length === 0) return this.#reporter.firstRunGreen(); + + banner(`ROUND 1 found ${failed.length} failing file(s)`); + for (const f of failed) console.log(` ✗ ${rel(f.file)}`); + + const round2 = await this.#round2(failed); + if (round2 === undefined) return INFRA_ABORT; + + const { flaky, real } = classify( + failed.map((f) => f.file), + round2, + ); + return this.#reporter.verdict(flaky, real); + } + + /** Drop reports from previous runs, so a stale file can never be read as this + * run's result. Round-2 names depend on which files fail, so clear them all. */ + #clearStaleReports(): void { + for (const t of this.#plan.targets) { + rmSync(round1Report(t.name), { force: true }); + } + if (!existsSync(LOGS)) return; + for (const f of readdirSync(LOGS)) { + if (f.startsWith(ROUND2_REPORT_PREFIX) && f.endsWith('.json')) { + rmSync(path.join(LOGS, f), { force: true }); + } + } + } + + /** + * Run every target once. + * + * Each target gets a freshly reset node: smaller coin tree, no cross-target + * state interactions. The harness smoke already validated the stack, and its + * only on-chain footprint (NIGHT/dust) does not trip the freshness guard — so + * the first target reuses the node the smoke ran against. + * + * @returns the failing files, or `undefined` on an infrastructure abort + */ + async #round1(): Promise { + const { targets, fileFilters } = this.#plan; + const failed: FailedFile[] = []; + + for (const [i, target] of targets.entries()) { + banner(`ROUND 1 · ${target.name} (${i + 1}/${targets.length})`); + if (i > 0 && (await this.#stack.up()) !== 0) { + console.log(`env-up failed before '${target.name}'.`); + return undefined; + } + + // vitest ORs positional filters, so passing the target dir *and* a name + // filter would match the whole target (every file is under the dir). Use + // the name filters when given — they scope to the matching files; + // otherwise the target's own filters run the whole set (for integration: + // none, so the project's include glob decides). + const filters = + fileFilters.length > 0 ? fileFilters : target.defaultFilters; + const reportPath = round1Report(target.name); + const status = await this.#runner.run( + target.project, + reportPath, + filters, + ); + + const statuses = this.#runner.fileStatuses(reportPath); + if (statuses === undefined) { + console.log( + `\n'${target.name}' produced no results file — the run was blocked ` + + '(dirty node / lock) or crashed before finishing.', + ); + return undefined; + } + const targetFailed = [...statuses.entries()] + .filter(([, s]) => s === 'failed') + .map(([name]) => name); + if (status !== 0 && targetFailed.length === 0) { + console.log( + `\n'${target.name}' exited non-zero without reporting failing ` + + 'files — aborting to be safe.', + ); + return undefined; + } + + failed.push(...targetFailed.map((file) => ({ file, target }))); + console.log( + `\n${target.name}: ${statuses.size} file(s), ${targetFailed.length} failed`, + ); + } + return failed; + } + + /** + * Re-run each failed file alone on a fresh node. + * + * The node is reset before *every* file, so state left by an earlier round-2 + * file can never fail a later one — that would misclassify a flake as REAL. + * + * @returns file → round-2 status, or `undefined` on an infrastructure abort + */ + async #round2( + failed: readonly FailedFile[], + ): Promise | undefined> { + banner('ROUND 2 — re-run each failed file alone on a fresh node'); + const statusByFile = new Map(); + + for (const [i, { file, target }] of failed.entries()) { + banner(`ROUND 2 · ${rel(file)} (${i + 1}/${failed.length})`); + if ((await this.#stack.up()) !== 0) { + console.log(`env-up failed before round 2 of '${rel(file)}'.`); + return undefined; + } + const reportPath = round2Report(file); + await this.#runner.run(target.project, reportPath, [file], { + MIDNIGHT_LIVE_WORKERS: '1', + }); + const statuses = this.#runner.fileStatuses(reportPath); + if (statuses === undefined) { + console.log( + `\nround 2 produced no results for '${rel(file)}' — cannot classify.`, + ); + return undefined; + } + // No entry means the file crashed without reporting; treat as not-passed. + statusByFile.set(file, statuses.get(file) ?? 'failed'); + } + return statusByFile; + } +} diff --git a/scripts/live/LiveStack.ts b/scripts/live/LiveStack.ts new file mode 100644 index 00000000..861c1605 --- /dev/null +++ b/scripts/live/LiveStack.ts @@ -0,0 +1,60 @@ +import { run, runSync } from './shell.ts'; + +/** Opt out of teardown to inspect a failed run's node state. */ +export const KEEP_ENV_VAR = 'MIDNIGHT_LIVE_KEEP_ENV'; +export const KEEP_ENV_HINT = `set ${KEEP_ENV_VAR}=1 to keep the stack up for inspection`; + +/** + * Owns the local stack's lifecycle for one orchestrator run: `make env-up` + * between phases, `make env-down` once at the end. + * + * `env-up` itself depends on `env-down` (see the Makefile), so each bring-up is + * also a reset — that is what gives every target and every round-2 file a fresh + * node, and why a stack leaked by a crashed run cannot corrupt the *next* one. + * + * Teardown therefore exists for the **current** run: the containers, and the + * `docker compose logs -f` streamers that each `env-up` backgrounds, otherwise + * outlive the process indefinitely. Container logs survive it — `env-down` kills + * the streamers, only `env-logs-clean` deletes files — so a failed run's + * `logs/*.log` stay readable. Only live node state is lost, hence + * {@link KEEP_ENV_VAR}. + */ +export class LiveStack { + /** Only tear down a stack this run actually started: an abort during compile + * (before the first `up()`) has nothing to stop, and `--list` never starts one. */ + #started = false; + #stopped = false; + + /** Reset and bring the stack up. Resolves to the `make` exit status. */ + up(): Promise { + // Marked before the call, not after: `env-up` can fail with containers + // already half-started (it stops on the `--wait`), and those still need + // stopping. + this.#started = true; + return run('make', ['env-up']); + } + + /** + * Stop the stack, at most once. A no-op if this run never started one, or if + * teardown already happened (the `finally` path and the signal path both call + * it, and a double Ctrl-C calls it twice). + * + * Deliberately synchronous: it runs inside the signal handler, which exits the + * process the moment it returns, so there is nowhere to await. + * + * @param reason - what triggered the teardown, for the log line + */ + stop(reason: string): void { + if (!this.#started || this.#stopped) return; + this.#stopped = true; + if (process.env[KEEP_ENV_VAR] === '1') { + console.log( + `\nleaving the live stack up (${KEEP_ENV_VAR}=1, ${reason}) — ` + + "run 'yarn env:down' when you are finished with it.", + ); + return; + } + console.log(`\nstopping the live stack (${reason})...`); + runSync('make', ['env-down']); + } +} diff --git a/scripts/live/Reporter.ts b/scripts/live/Reporter.ts new file mode 100644 index 00000000..9f49f572 --- /dev/null +++ b/scripts/live/Reporter.ts @@ -0,0 +1,87 @@ +import { appendFileSync } from 'node:fs'; +import { KEEP_ENV_HINT } from './LiveStack.ts'; +import { LOGS, rel } from './paths.ts'; +import { banner } from './shell.ts'; + +const FLAKE_NOTE = 'failed round 1, passed round 2 on a fresh node'; + +/** + * All run-level output: the verdict banner, GitHub Actions annotations, and the + * job summary. Everything here is a no-op outside CI except the console output, + * so local and CI runs go through the same path. + */ +export class Reporter { + /** Append markdown to the GitHub Actions job summary (no-op outside CI). */ + jobSummary(markdown: string): void { + const summaryPath = process.env.GITHUB_STEP_SUMMARY; + if (!summaryPath) return; + appendFileSync(summaryPath, `${markdown}\n`); + } + + /** Emit a GitHub Actions warning annotation (no-op outside CI). */ + ciWarn(file: string, message: string): void { + if (process.env.GITHUB_ACTIONS !== 'true') return; + console.log(`::warning file=${file}::${message}`); + } + + /** Nothing failed in round 1, so no classification was needed. */ + firstRunGreen(): number { + const headline = 'VERDICT: PASSED — all live specs green on the first run.'; + banner(headline); + this.jobSummary(`### ${headline}`); + return 0; + } + + /** + * Final verdict after round 2. + * + * @returns the process exit code — 0 for a flaky-only run, so an environment + * artifact never turns the build red, and 1 only for a real failure + */ + verdict(flaky: readonly string[], real: readonly string[]): number { + const headline = + real.length === 0 + ? `VERDICT: PASSED${flaky.length ? ` (with ${flaky.length} flaky file(s))` : ''}` + : `VERDICT: FAILED — ${real.length} real failure(s), ${flaky.length} flaky`; + banner(headline); + + if (flaky.length > 0) { + console.log(`\nFLAKY (${FLAKE_NOTE}):`); + for (const f of flaky) console.log(` ~ ${rel(f)}`); + } + if (real.length > 0) { + console.log('\nREAL (failed both rounds — investigate):'); + for (const f of real) console.log(` ✗ ${rel(f)}`); + // The stack is about to be stopped, so point at what survives it. + console.log( + `\ncontainer logs are kept in ${rel(LOGS)}/*.log after teardown; ` + + `on a re-run, ${KEEP_ENV_HINT}.`, + ); + } + // A flaky-only run exits 0, so without these a green CI run would swallow the + // flake report entirely. + for (const f of flaky) { + this.ciWarn(rel(f), `flaky live spec — ${FLAKE_NOTE}`); + } + this.jobSummary( + [ + `### ${headline}`, + ...(flaky.length > 0 + ? [ + '', + `Flaky (${FLAKE_NOTE}):`, + ...flaky.map((f) => `- ~ \`${rel(f)}\``), + ] + : []), + ...(real.length > 0 + ? [ + '', + 'Real failures (failed both rounds — investigate):', + ...real.map((f) => `- ✗ \`${rel(f)}\``), + ] + : []), + ].join('\n'), + ); + return real.length === 0 ? 0 : 1; + } +} diff --git a/scripts/live/RunLock.ts b/scripts/live/RunLock.ts new file mode 100644 index 00000000..6fdbeeb4 --- /dev/null +++ b/scripts/live/RunLock.ts @@ -0,0 +1,75 @@ +import { mkdirSync, readFileSync, unlinkSync, writeFileSync } from 'node:fs'; +import { LOGS, VERIFY_LOCK } from './paths.ts'; + +interface LockInfo { + readonly pid: number; + readonly startedAt: string; +} + +/** Whether a pid names a live process (EPERM means alive but not ours to signal). */ +function pidAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (e) { + return (e as NodeJS.ErrnoException).code === 'EPERM'; + } +} + +/** + * Pid-stamped lock making one orchestrator run exclusive. + * + * Two concurrent runs would interleave `env-up` resets and shielded-coin spends + * against the one shared node, so the second run fails fast instead. A lock left + * behind by a killed run is reclaimed, since its pid is no longer alive. + * + * This is the *orchestrator* lock. `live.globalSetup` holds a separate, + * deliberately reentrant one (`.live-run.lock`) scoped to a vitest process. + */ +export class RunLock { + readonly #path: string; + + constructor(lockPath: string = VERIFY_LOCK) { + this.#path = lockPath; + } + + #read(): LockInfo | undefined { + try { + return JSON.parse(readFileSync(this.#path, 'utf8')) as LockInfo; + } catch { + return undefined; + } + } + + /** Take the lock, or throw if a live process already holds it. */ + acquire(): void { + mkdirSync(LOGS, { recursive: true }); + const stamp = JSON.stringify({ + pid: process.pid, + startedAt: new Date().toISOString(), + }); + try { + writeFileSync(this.#path, stamp, { flag: 'wx' }); + } catch (e) { + if ((e as NodeJS.ErrnoException).code !== 'EEXIST') throw e; + const info = this.#read(); + if (info && pidAlive(info.pid)) { + throw new Error( + `another test:live run is already in progress (pid ${info.pid}, ` + + `started ${info.startedAt}). Wait for it, or remove ${this.#path}.`, + ); + } + writeFileSync(this.#path, stamp); // stale — reclaim + } + } + + /** Release only if we still own it, so a stale takeover is never clobbered. */ + release(): void { + if (this.#read()?.pid !== process.pid) return; + try { + unlinkSync(this.#path); + } catch { + // already gone + } + } +} diff --git a/scripts/live/VitestRunner.ts b/scripts/live/VitestRunner.ts new file mode 100644 index 00000000..fc18dc34 --- /dev/null +++ b/scripts/live/VitestRunner.ts @@ -0,0 +1,67 @@ +import { existsSync, readFileSync } from 'node:fs'; +import { CONTRACTS, PROGRESS_REPORTER, VITEST_BIN } from './paths.ts'; +import { run } from './shell.ts'; +import type { LiveTarget } from './targets.ts'; + +interface JsonTestResult { + readonly name: string; + readonly status: string; +} +interface JsonReport { + readonly testResults?: readonly JsonTestResult[]; +} + +/** Spawns vitest against one live project and reads back its JSON report. */ +export class VitestRunner { + /** + * Run one live project. + * + * @param project - the single project for this invocation (see `targets.ts` on + * why it is never more than one) + * @param reportPath - where the JSON reporter writes, for {@link fileStatuses} + * @param fileFilters - vitest positional filters; empty runs the include glob + * @param extraEnv - overrides layered over `MIDNIGHT_BACKEND=live` + * @returns vitest's exit status + */ + run( + project: LiveTarget['project'], + reportPath: string, + fileFilters: readonly string[], + extraEnv: Record = {}, + ): Promise { + return run( + VITEST_BIN, + [ + 'run', + '--project', + project, + // A target filtered down to zero matching files is a pass, not an error. + '--passWithNoTests', + // `default` prints one line per file (piped) plus failures/summary; the + // progress reporter adds the worker-tagged, counted per-test line. + '--reporter=default', + `--reporter=${PROGRESS_REPORTER}`, + '--reporter=json', + `--outputFile.json=${reportPath}`, + ...fileFilters, + ], + { ...process.env, MIDNIGHT_BACKEND: 'live', ...extraEnv }, + CONTRACTS, + ); + } + + /** + * File name → status for every file in the report. + * + * @returns `undefined` when no report exists at all — the run was blocked + * (dirty node / lock) or crashed before writing one, which callers must treat + * as an infrastructure abort rather than a test failure. + */ + fileStatuses(reportPath: string): Map | undefined { + if (!existsSync(reportPath)) return undefined; + const report = JSON.parse(readFileSync(reportPath, 'utf8')) as JsonReport; + return new Map( + (report.testResults ?? []).map((r) => [r.name, r.status] as const), + ); + } +} diff --git a/scripts/live/paths.ts b/scripts/live/paths.ts new file mode 100644 index 00000000..b8df2b96 --- /dev/null +++ b/scripts/live/paths.ts @@ -0,0 +1,53 @@ +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +/** + * Every filesystem location the live orchestrator touches, and the naming rules + * for its report files. Pure data — no side effects, nothing read at import. + */ + +export const REPO_ROOT = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + '../..', +); +export const CONTRACTS = path.join(REPO_ROOT, 'contracts'); +export const SRC = path.join(CONTRACTS, 'src'); +export const ARTIFACTS = path.join(CONTRACTS, 'artifacts'); +/** Integration mocks compile into the same `artifacts/` tree as `src`, so the + * truncated-key scan has to be told about them explicitly. */ +export const INTEGRATION_MOCKS = path.join( + CONTRACTS, + 'test/integration/_mocks', +); +export const LOGS = path.join(REPO_ROOT, 'logs'); +export const TURBO_CACHE = path.join(REPO_ROOT, '.turbo', 'cache'); +export const VITEST_BIN = path.join( + REPO_ROOT, + 'node_modules', + '.bin', + 'vitest', +); +export const PROGRESS_REPORTER = path.join( + CONTRACTS, + 'test-utils/harness/liveProgressReporter.ts', +); +export const VERIFY_LOCK = path.join(LOGS, '.live-verify.lock'); + +/** Repo-relative path, for readable console output. */ +export const rel = (abs: string): string => path.relative(REPO_ROOT, abs); + +/** Round-1 JSON report for a target (one per target, kept for the whole run). */ +export const round1Report = (target: string): string => + path.join(LOGS, `live-r1-${target}.json`); + +export const ROUND2_REPORT_PREFIX = 'live-r2-'; + +/** Round-2 JSON report for one re-run file. Unit specs are `*.test.ts`, + * integration specs `*.spec.ts` — both extensions are stripped. */ +export const round2Report = (file: string): string => + path.join( + LOGS, + `${ROUND2_REPORT_PREFIX}${path + .basename(file) + .replace(/\.(test|spec)\.ts$/, '')}.json`, + ); diff --git a/scripts/live/shell.ts b/scripts/live/shell.ts new file mode 100644 index 00000000..a1e71259 --- /dev/null +++ b/scripts/live/shell.ts @@ -0,0 +1,94 @@ +import { spawn, spawnSync } from 'node:child_process'; +import os from 'node:os'; +import { REPO_ROOT } from './paths.ts'; + +/** Process and console primitives shared by every live-orchestrator service. */ + +/** Exit status convention for a child killed by a signal (128 + signal number). */ +function signalStatus(signal: NodeJS.Signals): number { + const number = + os.constants.signals[signal as keyof typeof os.constants.signals]; + return 128 + (number ?? 0); +} + +/** + * Run a command with inherited stdio (so its output streams live) and resolve to + * its exit status. A spawn failure is reported and mapped to 1, and a child killed + * by a signal resolves to 128 + the signal number, so callers only branch on a + * number. + * + * **Asynchronous on purpose.** With `spawnSync` the event loop is blocked for the + * child's whole lifetime, and because a compile phase is one long synchronous + * chain the loop never turns between children either — so a queued SIGINT handler + * could not run until the entire phase finished. Ctrl-C during a compile was + * therefore ignored until it was too late, and the truncated keys the interruption + * itself had just created were then mistaken for a poisoned cache, draining it and + * kicking off a pointless serial recompile. Awaiting the child keeps the loop live, + * so {@link installSignalHandlers} fires immediately. + * + * Use {@link runSync} only where a result is needed without awaiting — teardown + * inside a signal handler. + */ +export function run( + cmd: string, + args: string[], + env: NodeJS.ProcessEnv = process.env, + cwd: string = REPO_ROOT, +): Promise { + return new Promise((resolve) => { + const child = spawn(cmd, args, { cwd, env, stdio: 'inherit' }); + child.on('error', (e) => { + console.log(`could not run ${cmd}: ${e.message}`); + resolve(1); + }); + child.on('close', (status, signal) => { + resolve(signal ? signalStatus(signal) : (status ?? 1)); + }); + }); +} + +/** + * Blocking variant, for the one caller that cannot await: a signal handler has to + * finish its cleanup before `process.exit`, and there is no way to await there. + * Everything else should use {@link run}. + */ +export function runSync( + cmd: string, + args: string[], + env: NodeJS.ProcessEnv = process.env, + cwd: string = REPO_ROOT, +): number { + const res = spawnSync(cmd, args, { cwd, env, stdio: 'inherit' }); + if (res.error) { + console.log(`could not run ${cmd}: ${res.error.message}`); + return 1; + } + if (res.signal) return signalStatus(res.signal); + return res.status ?? 1; +} + +/** A ruled section header, so phases stand out in a long streaming log. */ +export function banner(message: string): void { + const rule = '═'.repeat(64); + console.log(`\n${rule}\n${message}\n${rule}`); +} + +/** + * Run `onSignal` on Ctrl-C / SIGTERM, then exit with the conventional + * 128 + signal code. + * + * Node runs **no** `finally` block on a signal, so cleanup that lives only in a + * `try/finally` is skipped entirely when a run is interrupted. Anything that must + * happen on every exit path has to be registered here as well, and must be + * synchronous — the process exits as soon as `onSignal` returns. + */ +export function installSignalHandlers( + onSignal: (signal: 'SIGINT' | 'SIGTERM') => void, +): void { + for (const signal of ['SIGINT', 'SIGTERM'] as const) { + process.on(signal, () => { + onSignal(signal); + process.exit(signal === 'SIGINT' ? 130 : 143); + }); + } +} diff --git a/scripts/live/targets.ts b/scripts/live/targets.ts new file mode 100644 index 00000000..658dc451 --- /dev/null +++ b/scripts/live/targets.ts @@ -0,0 +1,128 @@ +import { readdirSync } from 'node:fs'; +import path from 'node:path'; +import { SRC } from './paths.ts'; + +/** + * What a live invocation resolves to. + * + * A target is either a unit category (`src/`, run under + * `--project unit-live`) or the composed-contract `integration` target (run under + * `--project integration-live` over `test/integration/specs`). + * + * Exactly ONE live project per vitest invocation: both live projects derive their + * wallets from `walletSeedsFor(VITEST_POOL_ID)`, so worker 1 of each resolves to + * the same genesis deployer. `live.globalSetup` rejects a second live project in + * one process; issuing one `--project` per invocation is how this side keeps that + * from happening in the first place. + */ + +/** `archive` is excluded from the unit/unit-live projects (see vitest.config). */ +const EXCLUDED_CATEGORIES = new Set(['archive']); + +/** + * Categories whose specs have been refactored for the live backend. The others + * still assume dry-only semantics (e.g. `.as()` identities derived from alias + * labels, which the live wallet pool cannot impersonate) and join this list as + * they are refactored, PR by PR. + */ +export const LIVE_READY = new Set(['multisig']); + +/** The composed-contract target. Not a `src/` category, so it is matched before + * the category branch — {@link liveCategories} will never contain it. */ +export const INTEGRATION = 'integration'; + +/** One unit of live work: a vitest project plus the files to run under it. */ +export interface LiveTarget { + /** Labels the target in banners and names its round-1 report. */ + readonly name: string; + readonly project: 'unit-live' | 'integration-live'; + /** vitest positional filters used when the dev gave no explicit file filter. + * Empty means "the project's whole include glob". */ + readonly defaultFilters: readonly string[]; +} + +export interface LivePlan { + readonly targets: readonly LiveTarget[]; + /** Live-ready-gated categories left out of an unscoped run, for reporting. */ + readonly skipped: readonly string[]; + readonly fileFilters: readonly string[]; + /** Whether the run needs full-key integration-mock artifacts. */ + readonly integration: boolean; +} + +export type PlanResolution = + | { readonly ok: true; readonly plan: LivePlan } + | { readonly ok: false; readonly message: string }; + +/** `src/` subdirectories that contain test files (future categories join + * automatically; no hardcoded list to maintain). */ +export function liveCategories(): string[] { + const hasTests = (dir: string): boolean => + readdirSync(dir, { withFileTypes: true }).some((entry) => + entry.isDirectory() + ? hasTests(path.join(dir, entry.name)) + : entry.name.endsWith('.test.ts'), + ); + return readdirSync(SRC, { withFileTypes: true }) + .filter((d) => d.isDirectory() && !EXCLUDED_CATEGORIES.has(d.name)) + .map((d) => d.name) + .filter((name) => hasTests(path.join(SRC, name))) + .sort(); +} + +/** Targets CI should spawn a job for. `LIVE_READY` plus the integration target + * stays the single source of truth for the matrix. */ +export function listTargets(allCategories: readonly string[]): string[] { + return [...allCategories.filter((c) => LIVE_READY.has(c)), INTEGRATION]; +} + +/** + * Resolve CLI args into a plan. Pure: no filesystem, no console, no exit codes — + * the caller decides what to do with a rejection. + * + * A first arg naming a target scopes the run (the `test:live:` scripts + * pass one); everything else is a vitest file filter. `integration` is matched + * first because it is not a `src/` category, so it would otherwise fall through + * to the unscoped path and silently run every live-ready unit category instead. + */ +export function resolvePlan( + args: readonly string[], + allCategories: readonly string[], +): PlanResolution { + const integration = args[0] === INTEGRATION; + const scoped = + !integration && args.length > 0 && allCategories.includes(args[0]); + + if (scoped && !LIVE_READY.has(args[0])) { + return { + ok: false, + message: + `'${args[0]}' is not live-ready yet — its specs still assume dry-only ` + + `semantics. Ready categories: ${[...LIVE_READY].join(', ')}, plus ` + + `'${INTEGRATION}'.`, + }; + } + + const targets: LiveTarget[] = integration + ? [{ name: INTEGRATION, project: 'integration-live', defaultFilters: [] }] + : (scoped ? [args[0]] : allCategories.filter((c) => LIVE_READY.has(c))).map( + (category) => ({ + name: category, + project: 'unit-live', + defaultFilters: [`src/${category}`], + }), + ); + + return { + ok: true, + plan: { + targets, + skipped: + integration || scoped + ? [] + : allCategories.filter((c) => !LIVE_READY.has(c)), + fileFilters: integration || scoped ? args.slice(1) : args, + integration, + }, + }; +} diff --git a/scripts/test-live.ts b/scripts/test-live.ts index 4267e6e4..a22a865d 100644 --- a/scripts/test-live.ts +++ b/scripts/test-live.ts @@ -1,472 +1,93 @@ -import { spawnSync } from 'node:child_process'; -import { - appendFileSync, - existsSync, - mkdirSync, - readdirSync, - readFileSync, - rmSync, - unlinkSync, - writeFileSync, -} from 'node:fs'; -import path from 'node:path'; -import { fileURLToPath } from 'node:url'; -import { emptyKeyArtifacts } from './keyIntegrity.ts'; +import { ArtifactCompiler } from './live/ArtifactCompiler.ts'; +import { INFRA_ABORT, LiveOrchestrator } from './live/LiveOrchestrator.ts'; +import { LiveStack } from './live/LiveStack.ts'; +import { Reporter } from './live/Reporter.ts'; +import { RunLock } from './live/RunLock.ts'; +import { installSignalHandlers } from './live/shell.ts'; +import { listTargets, liveCategories, resolvePlan } from './live/targets.ts'; +import { VitestRunner } from './live/VitestRunner.ts'; /** - * Live-test orchestrator: runs each category (`src/`) sequentially, - * each on a freshly reset node, then verifies any failures with a second round. + * Live-test orchestrator entry point: resolve args into a plan, wire the + * services, run it under the single-run lock, always stop the stack. * - * The live tests all run against one shared node, so state left by an earlier - * test can make a later one fail (a coin re-spent against stale node state is - * rejected with node "Custom error: 103"). A file that fails during a busy full - * run may therefore pass in isolation on a fresh node. Hence two rounds: + * Each concern lives in `scripts/live/` with the reasoning that belongs to it: + * - `targets.ts` — what an invocation resolves to (pure) + * - `RunLock.ts` — one orchestrator run at a time + * - `LiveStack.ts` — `make env-up` / `env-down` lifecycle + * - `ArtifactCompiler.ts` — build + truncated-ZK-key self-heal + * - `VitestRunner.ts` — spawn one live project, read its JSON report + * - `LiveOrchestrator.ts` — the two rounds and flake classification + * - `Reporter.ts` — verdict, CI annotations, job summary + * - `shell.ts` / `paths.ts` — process, console and filesystem primitives * - * Round 1: compile + harness smoke once, then per category: reset the stack - * and run that category's files (parallel workers). Collect the - * files that failed from the JSON reporter. - * Round 2: for each failed file, reset the stack and re-run just that file - * on its own (one worker), so no earlier round-2 file can dirty the - * node under a later one. - * - * A file that fails round 1 but passes round 2 is FLAKY (an environment - * artifact); one that fails both — or never reports in round 2 — is a REAL - * failure. Exit 0 unless there is a real failure, so an env flake never turns - * the run red — but it is reported loudly. - * - * Why a script and not turbo tasks: turbo models a DAG of stateless, - * cacheable tasks, and a live run needs stateful orchestration that a task - * graph cannot express: - * - the two-round flake classification above (re-run failures, classify, - * exit 0 on flaky-only); - * - docker lifecycle between categories and rounds (`make env-up`) against - * ONE shared node — parallel turbo tasks would race over it; + * Why a script and not turbo tasks: turbo models a DAG of stateless, cacheable + * tasks, and a live run needs stateful orchestration a task graph cannot express: + * - two-round flake classification (re-run failures, classify, exit 0 on + * flaky-only); + * - docker lifecycle between targets and rounds against ONE shared node — + * parallel turbo tasks would race over it; * - ZK-key integrity self-heal (turbo's own poisoned cache, #675); * - infra-vs-test exit codes (2 vs 1), the pid lock, CI verdict summaries. - * Turbo still runs where the DAG helps: the compile and harness-smoke steps - * below go through it (cached keygen, dependency ordering). + * Turbo still runs where the DAG helps: the compile and harness-smoke steps go + * through it (cached keygen, dependency ordering). * * Usage (via the root package.json scripts): * yarn test:live # every live-ready category * yarn test:live multisig # one category * yarn test:live multisig Forwarder # files within a category - * yarn test:live --list # live-ready categories (JSON) + * yarn test:live integration # the composed-contract integration specs + * yarn test:live --list # live targets, for the CI matrix (JSON) + * + * The stack's whole lifecycle belongs to this script: it starts it (`make env-up`, + * itself a reset) and stops it on every exit path, signals included. + * `MIDNIGHT_LIVE_KEEP_ENV=1` leaves it running for post-mortem inspection; + * container logs in `logs/` survive teardown either way. + * + * Exit codes: 0 pass (flaky-only included), 1 real test failure, 2 infrastructure + * abort, 130/143 interrupted. * * Node runs this .ts directly (type stripping); only `node:` builtins. */ - -const REPO_ROOT = path.resolve( - path.dirname(fileURLToPath(import.meta.url)), - '..', -); -const CONTRACTS = path.join(REPO_ROOT, 'contracts'); -const SRC = path.join(CONTRACTS, 'src'); -const LOGS = path.join(REPO_ROOT, 'logs'); -const VITEST = path.join(REPO_ROOT, 'node_modules', '.bin', 'vitest'); -const PROGRESS_REPORTER = path.join( - CONTRACTS, - 'test-utils/harness/liveProgressReporter.ts', -); -const VERIFY_LOCK = path.join(LOGS, '.live-verify.lock'); - -// `archive` is excluded from the unit/unit-live projects (see vitest.config). -const EXCLUDED_CATEGORIES = new Set(['archive']); - -// Categories whose specs have been refactored for the live backend. The others -// still assume dry-only semantics (e.g. `.as()` identities derived from alias -// labels, which the live wallet pool cannot impersonate) and join this list as -// they are refactored, PR by PR. -const LIVE_READY = new Set(['multisig']); - -interface JsonTestResult { - readonly name: string; - readonly status: string; -} -interface JsonReport { - readonly testResults?: readonly JsonTestResult[]; -} -interface LockInfo { - readonly pid: number; - readonly startedAt: string; -} - -const rel = (abs: string): string => path.relative(REPO_ROOT, abs); -const r1Json = (category: string): string => - path.join(LOGS, `live-r1-${category}.json`); -const r2Json = (file: string): string => - path.join( - LOGS, - `live-r2-${path.basename(file).replace(/\.test\.ts$/, '')}.json`, - ); - -function banner(message: string): void { - const rule = '═'.repeat(64); - console.log(`\n${rule}\n${message}\n${rule}`); -} - -/** Append markdown to the GitHub Actions job summary (no-op outside CI). */ -function appendJobSummary(markdown: string): void { - const summaryPath = process.env.GITHUB_STEP_SUMMARY; - if (!summaryPath) return; - appendFileSync(summaryPath, `${markdown}\n`); -} - -/** Emit a GitHub Actions warning annotation (no-op outside CI). */ -function ciWarn(file: string, message: string): void { - if (process.env.GITHUB_ACTIONS !== 'true') return; - console.log(`::warning file=${file}::${message}`); -} - -/** `src/` subdirectories that contain test files (future categories join - * automatically; no hardcoded list to maintain). */ -function liveCategories(): string[] { - const hasTests = (dir: string): boolean => - readdirSync(dir, { withFileTypes: true }).some((entry) => - entry.isDirectory() - ? hasTests(path.join(dir, entry.name)) - : entry.name.endsWith('.test.ts'), - ); - return readdirSync(SRC, { withFileTypes: true }) - .filter((d) => d.isDirectory() && !EXCLUDED_CATEGORIES.has(d.name)) - .map((d) => d.name) - .filter((name) => hasTests(path.join(SRC, name))) - .sort(); -} - -/** Run a command with inherited stdio (streams live). Returns its exit status. */ -function run( - cmd: string, - args: string[], - env: NodeJS.ProcessEnv = process.env, - cwd: string = REPO_ROOT, -): number { - const res = spawnSync(cmd, args, { cwd, env, stdio: 'inherit' }); - if (res.error) { - console.log(`could not run ${cmd}: ${res.error.message}`); - return 1; - } - return res.status ?? 1; -} - -function runLiveVitest( - jsonPath: string, - fileFilters: string[], - extraEnv: Record, -): number { - return run( - VITEST, - [ - 'run', - '--project', - 'unit-live', - // A category filtered down to zero matching files is a pass, not an error. - '--passWithNoTests', - // `default` prints one line per file (piped) plus failures/summary; the - // progress reporter adds the worker-tagged, counted per-test line. - '--reporter=default', - `--reporter=${PROGRESS_REPORTER}`, - '--reporter=json', - `--outputFile.json=${jsonPath}`, - ...fileFilters, - ], - { ...process.env, MIDNIGHT_BACKEND: 'live', ...extraEnv }, - CONTRACTS, - ); -} - -/** name → status for every file in the report, or undefined if none exists. */ -function fileStatuses(jsonPath: string): Map | undefined { - if (!existsSync(jsonPath)) return undefined; - const report = JSON.parse(readFileSync(jsonPath, 'utf8')) as JsonReport; - return new Map( - (report.testResults ?? []).map((r) => [r.name, r.status] as const), - ); -} - -function pidAlive(pid: number): boolean { - try { - process.kill(pid, 0); - return true; - } catch (e) { - return (e as NodeJS.ErrnoException).code === 'EPERM'; - } -} - -function readLock(): LockInfo | undefined { - try { - return JSON.parse(readFileSync(VERIFY_LOCK, 'utf8')) as LockInfo; - } catch { - return undefined; - } -} - -function acquireVerifyLock(): void { - mkdirSync(LOGS, { recursive: true }); - const stamp = JSON.stringify({ - pid: process.pid, - startedAt: new Date().toISOString(), - }); - try { - writeFileSync(VERIFY_LOCK, stamp, { flag: 'wx' }); - } catch (e) { - if ((e as NodeJS.ErrnoException).code !== 'EEXIST') throw e; - const info = readLock(); - if (info && pidAlive(info.pid)) { - throw new Error( - `another test:live run is already in progress (pid ${info.pid}, ` + - `started ${info.startedAt}). Wait for it, or remove ${VERIFY_LOCK}.`, - ); - } - writeFileSync(VERIFY_LOCK, stamp); // stale — reclaim - } -} - -function releaseVerifyLock(): void { - if (readLock()?.pid === process.pid) { - try { - unlinkSync(VERIFY_LOCK); - } catch { - // already gone - } - } -} - -const truncatedKeys = (): string[] => - emptyKeyArtifacts(path.join(CONTRACTS, 'artifacts'), SRC); - -/** - * Compile, then verify no truncated (0-byte) ZK key was left behind. A killed - * compile (or machine crash) can poison the turbo cache so that every later - * cache hit re-extracts a truncated key, and a concurrent compile racing this - * one over the shared `artifacts/` tree can truncate keys directly - * (OpenZeppelin/compact-contracts#675). Both repairs are mechanical, so - * self-heal once — drain the cache and recompile serially (a parallel - * recompile can re-poison it) — and only abort if keys are still truncated - * after the rebuild. - */ -function compileVerified(): boolean { - if (run('yarn', ['compile']) !== 0) { - console.log('compile failed — a compile error is real, not a flake.'); - return false; - } - const empty = truncatedKeys(); - if (empty.length === 0) return true; - - console.log( - '\ncompile reported success but left truncated (0-byte) ZK key(s):', - ); - for (const k of empty) console.log(` ✗ ${rel(k)}`); - console.log( - '\nPoisoned turbo cache or artifact tree ' + - '(OpenZeppelin/compact-contracts#675) — draining the cache and ' + - 'recompiling serially...', - ); - rmSync(path.join(REPO_ROOT, '.turbo', 'cache'), { - recursive: true, - force: true, - }); - if (run('yarn', ['compile', '--concurrency=1']) !== 0) { - console.log('serial recompile failed.'); - return false; - } - const stillEmpty = truncatedKeys(); - if (stillEmpty.length === 0) { - console.log('recovered — ZK keys intact after the serial recompile.'); - return true; - } - console.log( - '\nstill truncated after a serial recompile — needs investigation:', - ); - for (const k of stillEmpty) console.log(` ✗ ${rel(k)}`); - return false; -} - -function reportVerdict(flaky: string[], real: string[]): number { - const headline = - real.length === 0 - ? `VERDICT: PASSED${flaky.length ? ` (with ${flaky.length} flaky file(s))` : ''}` - : `VERDICT: FAILED — ${real.length} real failure(s), ${flaky.length} flaky`; - banner(headline); - if (flaky.length > 0) { - console.log('\nFLAKY (failed round 1, passed round 2 on a fresh node):'); - for (const f of flaky) console.log(` ~ ${rel(f)}`); - } - if (real.length > 0) { - console.log('\nREAL (failed both rounds — investigate):'); - for (const f of real) console.log(` ✗ ${rel(f)}`); - } - // A flaky-only run exits 0, so without these a green CI run would swallow - // the flake report entirely. - for (const f of flaky) { - ciWarn( - rel(f), - 'flaky live spec — failed round 1, passed round 2 on a fresh node', - ); - } - appendJobSummary( - [ - `### ${headline}`, - ...(flaky.length > 0 - ? [ - '', - 'Flaky (failed round 1, passed round 2 on a fresh node):', - ...flaky.map((f) => `- ~ \`${rel(f)}\``), - ] - : []), - ...(real.length > 0 - ? [ - '', - 'Real failures (failed both rounds — investigate):', - ...real.map((f) => `- ✗ \`${rel(f)}\``), - ] - : []), - ].join('\n'), - ); - return real.length === 0 ? 0 : 1; -} - async function main(): Promise { - // `--list` prints the live-ready categories as JSON and exits — CI derives - // its per-category matrix from this, so LIVE_READY stays the single source - // of truth. + // `--list` prints the CI matrix targets and exits without touching the stack. if (process.argv.includes('--list')) { - console.log( - JSON.stringify(liveCategories().filter((c) => LIVE_READY.has(c))), - ); + console.log(JSON.stringify(listTargets(liveCategories()))); return 0; } + const args = process.argv.slice(2).filter((a) => a !== '--'); - const allCategories = liveCategories(); - // First arg naming a category (the test:live: scripts pass one) - // scopes the run; everything else is a vitest file filter. - const scoped = args.length > 0 && allCategories.includes(args[0]); - if (scoped && !LIVE_READY.has(args[0])) { - console.log( - `'${args[0]}' is not live-ready yet — its specs still assume dry-only ` + - `semantics. Ready categories: ${[...LIVE_READY].join(', ')}.`, - ); - return 2; + const resolution = resolvePlan(args, liveCategories()); + if (!resolution.ok) { + console.log(resolution.message); + return INFRA_ABORT; } - const categories = scoped - ? [args[0]] - : allCategories.filter((c) => LIVE_READY.has(c)); - const skipped = scoped ? [] : allCategories.filter((c) => !LIVE_READY.has(c)); - const fileFilters = scoped ? args.slice(1) : args; - - acquireVerifyLock(); - try { - for (const c of categories) rmSync(r1Json(c), { force: true }); - if (existsSync(LOGS)) { - for (const f of readdirSync(LOGS)) { - if (f.startsWith('live-r2-') && f.endsWith('.json')) { - rmSync(path.join(LOGS, f), { force: true }); - } - } - } - - banner( - `ROUND 1 — categories: ${categories.join(', ')}` + - (fileFilters.length ? ` (filter: ${fileFilters.join(' ')})` : ''), - ); - if (skipped.length > 0) { - console.log(`skipped (not yet live-ready): ${skipped.join(', ')}`); - } - if (!compileVerified()) return 2; - if (run('make', ['env-up']) !== 0) { - console.log('env-up failed — cannot start the live stack.'); - return 2; - } - if (run('yarn', ['test:harness:live']) !== 0) { - console.log( - '\nlive harness smoke failed — this is an infrastructure problem, ' + - 'not a spec flake. Fix the stack and retry.', - ); - return 2; - } - - // Each category gets a freshly reset node: smaller coin tree, no - // cross-category state interactions. The smoke above already validated the - // stack, and its only on-chain footprint (NIGHT/dust) does not trip the - // freshness guard — so the first category reuses its node. - const failed: string[] = []; - for (const [i, category] of categories.entries()) { - banner(`ROUND 1 · ${category} (${i + 1}/${categories.length})`); - if (i > 0 && run('make', ['env-up']) !== 0) { - console.log(`env-up failed before '${category}'.`); - return 2; - } - // vitest ORs positional filters, so passing the category dir *and* a name - // filter would match the whole category (every file is under the dir). - // Use the name filters when given — they scope to the matching files; - // otherwise the category dir runs the whole set. - const round1Filters = - fileFilters.length > 0 ? fileFilters : [`src/${category}`]; - const status = runLiveVitest(r1Json(category), round1Filters, {}); - const statuses = fileStatuses(r1Json(category)); - if (statuses === undefined) { - console.log( - `\n'${category}' produced no results file — the run was blocked ` + - '(dirty node / lock) or crashed before finishing.', - ); - return 2; - } - const categoryFailed = [...statuses.entries()] - .filter(([, s]) => s === 'failed') - .map(([name]) => name); - if (status !== 0 && categoryFailed.length === 0) { - console.log( - `\n'${category}' exited non-zero without reporting failing files — ` + - 'aborting to be safe.', - ); - return 2; - } - failed.push(...categoryFailed); - console.log( - `\n${category}: ${statuses.size} file(s), ${categoryFailed.length} failed`, - ); - } - - if (failed.length === 0) { - banner('VERDICT: PASSED — all live specs green on the first run.'); - appendJobSummary( - '### VERDICT: PASSED — all live specs green on the first run.', - ); - return 0; - } - - banner(`ROUND 1 found ${failed.length} failing file(s)`); - for (const f of failed) console.log(` ✗ ${rel(f)}`); + const { plan } = resolution; + + const stack = new LiveStack(); + const lock = new RunLock(); + const orchestrator = new LiveOrchestrator({ + plan, + stack, + compiler: new ArtifactCompiler(plan.integration), + runner: new VitestRunner(), + reporter: new Reporter(), + }); - banner('ROUND 2 — re-run each failed file alone on a fresh node'); - // Reset the node before each file so state left by an earlier round-2 file - // can never fail a later one (which would misclassify a flake as REAL). - const round2 = new Map(); - for (const [i, file] of failed.entries()) { - banner(`ROUND 2 · ${rel(file)} (${i + 1}/${failed.length})`); - if (run('make', ['env-up']) !== 0) { - console.log(`env-up failed before round 2 of '${rel(file)}'.`); - return 2; - } - const jsonPath = r2Json(file); - runLiveVitest(jsonPath, [file], { MIDNIGHT_LIVE_WORKERS: '1' }); - const statuses = fileStatuses(jsonPath); - if (statuses === undefined) { - console.log( - `\nround 2 produced no results for '${rel(file)}' — cannot classify.`, - ); - return 2; - } - // No entry means the file crashed without reporting; treat as not-passed. - round2.set(file, statuses.get(file) ?? 'failed'); - } + // Teardown always precedes the lock release, so no other run can start against + // a half-stopped stack. Both cleanup paths are needed: `finally` covers normal + // and thrown exits, the signal handler covers Ctrl-C (where no `finally` runs). + const cleanup = (reason: string): void => { + stack.stop(reason); + lock.release(); + }; + installSignalHandlers(cleanup); - // Only an explicit round-2 pass demotes a failure to FLAKY; a file that - // failed again — or never reported (crashed) — stays REAL. - const flaky = failed.filter((f) => round2.get(f) === 'passed'); - const real = failed.filter((f) => round2.get(f) !== 'passed'); - return reportVerdict(flaky, real); + lock.acquire(); + try { + return await orchestrator.run(); } finally { - releaseVerifyLock(); + cleanup('run finished'); } } @@ -474,5 +95,5 @@ main() .then((code) => process.exit(code)) .catch((e) => { console.log(e instanceof Error ? e.message : String(e)); - process.exit(2); + process.exit(INFRA_ABORT); }); From 721aad1e39c69e987ecee5db6cac5a89ed18ddf7 Mon Sep 17 00:00:00 2001 From: 0xisk <0xisk@proton.me> Date: Mon, 27 Jul 2026 15:31:25 +0200 Subject: [PATCH 05/16] fix(harness): tag skipped live tests with a worker A skipped test never runs `beforeEach`, so no worker stamps its metadata and every skipped line printed `[w?]`. Fall back to the last worker seen for that test's module: the module was loaded and run by that worker even where an individual test was skipped. `?` now means only that nothing in the module ever reported a worker. --- .../test-utils/harness/liveProgressReporter.ts | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/contracts/test-utils/harness/liveProgressReporter.ts b/contracts/test-utils/harness/liveProgressReporter.ts index dfa4b9df..41ad36f3 100644 --- a/contracts/test-utils/harness/liveProgressReporter.ts +++ b/contracts/test-utils/harness/liveProgressReporter.ts @@ -10,6 +10,11 @@ import type { Reporter, TestCase, TestModule } from 'vitest/node'; * worker in `live.setup` (the only place that knows its `VITEST_POOL_ID`). The * total accrues as modules are collected, so the first few lines may show a * smaller denominator until collection finishes. + * + * A SKIPPED test never runs `beforeEach`, so no worker stamps it. Rather than + * print `[w?]`, fall back to the last worker seen for that test's module: the + * module was loaded and run by that worker even where an individual test was + * skipped. `?` remains only for a module where nothing ever reported a worker. */ const MARKS: Record = { passed: '✓', @@ -20,10 +25,13 @@ const MARKS: Record = { export default class LiveProgressReporter implements Reporter { private total = 0; private done = 0; + /** module id → the last worker that reported a test from it. */ + private workerByModule = new Map(); onTestRunStart(): void { this.total = 0; this.done = 0; + this.workerByModule.clear(); } onTestModuleCollected(module: TestModule): void { @@ -34,7 +42,10 @@ export default class LiveProgressReporter implements Reporter { const { state } = testCase.result(); if (state === 'pending') return; // not finished yet this.done += 1; - const worker = (testCase.meta() as { workerId?: number }).workerId ?? '?'; + const moduleId = testCase.module.moduleId; + const stamped = (testCase.meta() as { workerId?: number }).workerId; + if (stamped !== undefined) this.workerByModule.set(moduleId, stamped); + const worker = stamped ?? this.workerByModule.get(moduleId) ?? '?'; const mark = MARKS[state] ?? '·'; const ms = Math.round(testCase.diagnostic()?.duration ?? 0); console.log( From f453342d8ad0315367457e3ebf023813b2f0fa71 Mon Sep 17 00:00:00 2001 From: 0xisk <0xisk@proton.me> Date: Mon, 27 Jul 2026 15:31:26 +0200 Subject: [PATCH 06/16] refactor(test): rename the composed CFT mock The compiler names each artifact directory after the source basename, and `ConfidentialFungibleTokenPublicSupply.compact` existed both under `src/token/extensions` and under `test/integration/_mocks`. Two different contracts wrote one `artifacts//` directory, so whichever compiled last won. That is not theoretical. A background `yarn types` pulled in a src compile, replaced the directory, and every dry functional test then failed with "Contract state constructor: expected 1 argument, received 4". Nothing else catches it: the wrong artifact's keys are well-formed, just wrong, so the truncated-key scan cannot see it, and it survives lint and typecheck to surface at runtime as a confusing wrong-contract arity error. Prefixing the mock gives the two contracts separate directories and removes the whole class rather than ordering around it. The mock's own header records why the prefix exists so a future rename does not reintroduce it. --- ... ComposedConfidentialFungibleTokenPublicSupply.compact} | 4 ++++ .../fixtures/confidentialFungibleTokenPublicSupply.ts | 7 ++++--- 2 files changed, 8 insertions(+), 3 deletions(-) rename contracts/test/integration/_mocks/{ConfidentialFungibleTokenPublicSupply.compact => ComposedConfidentialFungibleTokenPublicSupply.compact} (95%) diff --git a/contracts/test/integration/_mocks/ConfidentialFungibleTokenPublicSupply.compact b/contracts/test/integration/_mocks/ComposedConfidentialFungibleTokenPublicSupply.compact similarity index 95% rename from contracts/test/integration/_mocks/ConfidentialFungibleTokenPublicSupply.compact rename to contracts/test/integration/_mocks/ComposedConfidentialFungibleTokenPublicSupply.compact index c1daec47..d0ae3c4d 100644 --- a/contracts/test/integration/_mocks/ConfidentialFungibleTokenPublicSupply.compact +++ b/contracts/test/integration/_mocks/ComposedConfidentialFungibleTokenPublicSupply.compact @@ -17,6 +17,10 @@ // "Public" supply means the total, and therefore each mint/burn delta, is // disclosed on chain. Balances stay confidential; only the aggregate is // visible. +// +// The `Composed` prefix is load-bearing: compactc keys each `artifacts//` +// directory on the source basename, so this file must not share a basename with +// the `src/token/extensions` extension it composes. pragma language_version >= 0.23.0; import CompactStandardLibrary; diff --git a/contracts/test/integration/fixtures/confidentialFungibleTokenPublicSupply.ts b/contracts/test/integration/fixtures/confidentialFungibleTokenPublicSupply.ts index 392d8803..ba163bef 100644 --- a/contracts/test/integration/fixtures/confidentialFungibleTokenPublicSupply.ts +++ b/contracts/test/integration/fixtures/confidentialFungibleTokenPublicSupply.ts @@ -7,7 +7,7 @@ import { ledger, Contract as MockConfidentialFungibleTokenPublicSupply, type Token_EscrowEntry, -} from '../../../artifacts/ConfidentialFungibleTokenPublicSupply/contract/index.js'; +} from '../../../artifacts/ComposedConfidentialFungibleTokenPublicSupply/contract/index.js'; import { ConfidentialFungibleTokenPrivateState, ConfidentialFungibleTokenWitnesses, @@ -16,7 +16,8 @@ import { /** * Integration fixture for the assembled ConfidentialFungibleToken + PublicSupply - * contract (`test/integration/_mocks/ConfidentialFungibleTokenPublicSupply`). + * contract + * (`test/integration/_mocks/ComposedConfidentialFungibleTokenPublicSupply`). * * It reuses the base token's confidential-token witnesses and private state (SK, * EK, plaintext cache, randomness seed) verbatim — the assembled contract's @@ -45,7 +46,7 @@ const Base = createSimulator< contractArgs: (name, symbol, decimals) => [name, symbol, decimals], ledgerExtractor: (state) => ledger(state), witnessesFactory: () => ConfidentialFungibleTokenWitnesses(), - artifactName: 'ConfidentialFungibleTokenPublicSupply', + artifactName: 'ComposedConfidentialFungibleTokenPublicSupply', }); export class ConfidentialFungibleTokenPublicSupplySimulator extends Base { From a8e872ac17c9bb29c9549609befeffdd24778620 Mon Sep 17 00:00:00 2001 From: 0xisk <0xisk@proton.me> Date: Mon, 27 Jul 2026 15:31:49 +0200 Subject: [PATCH 07/16] test(integration): add integration-live and a canary Let the existing live backend drive the `integration` project the same way it already drives `unit-live`. The backend is already backend-generic, so this is additive wiring rather than new harness logic: the new project reuses the same globalSetup (freshness, run lock) and setup (wallet pool, backend register), with its own scheduler group because `unit-live` holds group 1 with a different `maxWorkers`, which vitest rejects. One glob feeds both projects. Which blocks run under each backend is decided at runtime by `skipIf`/`runIf(isLiveBackend())` in the spec, so a spec's live/dry split lives next to the code it guards rather than in a naming convention or a central allowlist. The deliverable is the capability plus a block-limit canary, not green functional live coverage. The composed contract cannot deploy live: the base token already bundles four k=16 circuits' IR into one deploy tx and overruns the per-tx block byte budget, and this composition is strictly larger. Rather than skip and hide that, the canary asserts the rejection, so an unexpected success fails loudly the day a staged deploy or a smaller composition lets it through. It asserts the node's exact wording, since a loose match could be satisfied by an unrelated deploy failure and report a false green. The functional suite stays dry on two counts: the deploy above is rejected, and every flow mutates private state mid-test via `switchIdentity`/`cachePlaintext`, which the live backend throws on. `initStateIsolation` is dry-only by nature, constructing simulators directly and never deploying. `compile:integration` drops its hardcoded SKIP_ZK so one script serves both the dry and the full-key path, cached separately by turbo. The dry path is unchanged because the root `test:integration` still exports it. Each live project also publishes its own worker total, so the per-worker banner reports that project's `maxWorkers` instead of `unit-live`'s. --- contracts/package.json | 2 +- .../specs/confidentialFungibleToken.spec.ts | 301 +++++++++++------- .../specs/initStateIsolation.spec.ts | 7 +- contracts/vitest.config.ts | 56 +++- package.json | 1 + 5 files changed, 249 insertions(+), 118 deletions(-) diff --git a/contracts/package.json b/contracts/package.json index 0fa21b80..322e8164 100644 --- a/contracts/package.json +++ b/contracts/package.json @@ -32,7 +32,7 @@ "compile:security": "compact-compiler --dir security", "compile:token": "compact-compiler --dir token", "compile:utils": "compact-compiler --dir utils", - "compile:integration": "SKIP_ZK=true compact-compiler --src test/integration/_mocks", + "compile:integration": "compact-compiler --src test/integration/_mocks", "build": "compact-builder --hierarchical --out dist --clean-dist --exclude '*/archive/*' --exclude 'Mock*' --exclude '*.mock.compact' --copy package.json --copy ../README.md && find dist -type d -empty -delete", "test": "vitest run --project unit", "test:coverage": "vitest run --project unit --coverage", diff --git a/contracts/test/integration/specs/confidentialFungibleToken.spec.ts b/contracts/test/integration/specs/confidentialFungibleToken.spec.ts index c4096cf5..73672029 100644 --- a/contracts/test/integration/specs/confidentialFungibleToken.spec.ts +++ b/contracts/test/integration/specs/confidentialFungibleToken.spec.ts @@ -3,12 +3,13 @@ import { CompactTypeVector, persistentHash, } from '@midnight-ntwrk/compact-runtime'; +import { isLiveBackend } from '@openzeppelin/compact-simulator'; import { beforeEach, describe, expect, it } from 'vitest'; import { ConfidentialFungibleTokenPublicSupplySimulator } from '../fixtures/confidentialFungibleTokenPublicSupply.js'; /** * Integration spec for the assembled ConfidentialFungibleToken + PublicSupply - * contract (`_mocks/ConfidentialFungibleTokenPublicSupply`). + * contract (`_mocks/ComposedConfidentialFungibleTokenPublicSupply`). * * The token base and the supply extension are unit-tested in isolation. This * suite exercises the one property neither can: the COMPOSITION. Every @@ -21,6 +22,14 @@ import { ConfidentialFungibleTokenPublicSupplySimulator } from '../fixtures/conf * * so `totalSupply` must move in lockstep with the confidential balance change. * Balances stay hidden; `totalSupply` (the public aggregate) is what we assert. + * + * Backend split (`--project integration` vs `integration-live`), in file order: + * 1. The LIVE block is the block-limit canary: it asserts the composed deploy + * is rejected, which is what makes "no green functional live integration + * coverage" a verified claim rather than an assumption. + * 2. The functional block is DRY-ONLY, because of (1) — and because it drives + * confidential identities through `switchIdentity` / `cachePlaintext`, which + * the live backend throws on. */ // Mirrors the base suite's deterministic identity setup. @@ -51,131 +60,195 @@ const DECIMALS = 6n; let cft: ConfidentialFungibleTokenPublicSupplySimulator; -describe('ConfidentialFungibleToken + PublicSupply composition', () => { - beforeEach(async () => { - cft = await ConfidentialFungibleTokenPublicSupplySimulator.create( - NAME, - SYMBOL, - DECIMALS, - ); - }); - - const registerAll = async () => { - for (const u of [ALICE, BOB]) { - await cft.privateState.switchIdentity(u.secretKey, u.encryptionKey); - await cft.register(); - } - }; - - // Mints `amount` to Alice, sweeps it into spendable, and caches the swept - // balance so it can later be debited (burned). Leaves Alice active. - const fundAlice = async (amount: bigint) => { - await registerAll(); - await cft.privateState.switchIdentity(ALICE.secretKey, ALICE.encryptionKey); - await cft.mint(ALICE.accountId, amount); - await cft.sweep(); - await cft.privateState.cachePlaintext( - await cft.balanceOf(ALICE.accountId), - amount, - ); - }; - - describe('mint', () => { - it('increases totalSupply by exactly the minted value', async () => { - await registerAll(); - expect(await cft.totalSupply()).toBe(0n); - - await cft.mint(ALICE.accountId, 100n); - expect(await cft.totalSupply()).toBe(100n); - - await cft.mint(ALICE.accountId, 50n); - expect(await cft.totalSupply()).toBe(150n); - }); - - it('accumulates supply across recipients', async () => { - await registerAll(); - await cft.mint(ALICE.accountId, 100n); - await cft.mint(BOB.accountId, 50n); - expect(await cft.totalSupply()).toBe(150n); - }); - }); - - describe('mint + burn round-trip', () => { - it('mints a genuinely spendable balance and burns it back to zero supply', async () => { - await fundAlice(100n); - expect(await cft.totalSupply()).toBe(100n); - - // The mint credited Alice the full 100: burning 100 only proves out if her - // spendable balance truly encrypts >= 100. It also drops supply to 0. - await cft.burn(100n); - expect(await cft.totalSupply()).toBe(0n); - }); - }); - - describe('burn', () => { - it('moves totalSupply in lockstep with the caller debit', async () => { - await fundAlice(100n); - - await cft.burn(40n); - expect(await cft.totalSupply()).toBe(60n); - - // Alice's spendable is now 60; burning it drops supply to 0 in lockstep. - await cft.privateState.cachePlaintext( - await cft.balanceOf(ALICE.accountId), - 60n, - ); - await cft.burn(60n); - expect(await cft.totalSupply()).toBe(0n); - }); - - it('reverts a burn that exceeds the caller balance, leaving supply intact', async () => { - await fundAlice(100n); - - await expect(cft.burn(101n)).rejects.toThrow( - 'ConfidentialFungibleToken: insufficient balance', +const deploy = () => + ConfidentialFungibleTokenPublicSupplySimulator.create(NAME, SYMBOL, DECIMALS); + +// --------------------------------------------------------------------------- +// Live: block-limit canary. This block comes FIRST because it is the boundary +// condition that explains the rest of the file — it is the reason the functional +// suite below is dry-only. +// +// The base `ConfidentialFungibleToken` already bundles four k=16 circuits' IR +// into one deploy tx, which overruns the per-tx block byte budget — its own live +// block asserts that rejection (`src/token/test/ConfidentialFungibleToken.test.ts`). +// This composition adds the PublicSupply extension on top, so it is strictly +// larger and hits the same wall. +// +// Rather than skip and hide that, ASSERT the rejection. A green skip would let +// the branch claim live coverage it does not have; this canary instead fails the +// day a staged deploy, a smaller composition, or a looser ledger lets the +// composed contract through — which is exactly when green functional live +// integration coverage becomes possible, and the guards below can be inverted. +// --------------------------------------------------------------------------- + +describe.runIf(isLiveBackend())( + 'ConfidentialFungibleToken + PublicSupply composition: live deploy', + () => { + it('deploy is rejected for exceeding the ledger block byte budget', async () => { + // Fresh funded node, well-formed tx: the only reason the deploy can be + // rejected here is the block byte budget. Assert it, verbatim. + let error: unknown; + try { + await deploy(); + } catch (e) { + error = e; + } + expect( + error, + 'expected the node to reject the oversized deploy', + ).toBeDefined(); + const detail = [ + (error as Error)?.message, + (error as { cause?: unknown })?.cause, + String(error), + ] + .map((x) => String(x ?? '')) + .join(' | '); + // The node's exact words. Asserted in full rather than a loose + // `/block limits/` match, which an unrelated deploy failure could satisfy + // and report as a false green. + expect(detail).toContain( + '1010: Invalid Transaction: Transaction would exhaust the block limits', ); - // The value op reverted before the supply decrement, so supply is unchanged. - expect(await cft.totalSupply()).toBe(100n); }); - }); - - describe('burnFrom', () => { - it('drops totalSupply by spending the spender allowance', async () => { - // Alice mints 100 and approves Bob for 40. - await fundAlice(100n); - await cft.approve(BOB.accountId, 40n); - expect(await cft.totalSupply()).toBe(100n); - - // Bob decrypts his escrow copy (40), then burns 25 of the allowance. - await cft.privateState.switchIdentity(BOB.secretKey, BOB.encryptionKey); - const escrow = await cft.allowance(ALICE.accountId, BOB.accountId); - await cft.privateState.cachePlaintext(escrow.spenderCt, 40n); - - await cft.burnFrom(ALICE.accountId, 25n); - expect(await cft.totalSupply()).toBe(75n); + }, +); + +// --------------------------------------------------------------------------- +// Dry: the functional composition suite. Cannot run live on two counts — the +// deploy above is rejected, and every flow here mutates private state mid-test +// (`switchIdentity` / `cachePlaintext`), which the live backend throws on +// ('private-state mutation unsupported on live backend'). Going green live needs +// both a deploy that fits AND a deploy-seeded, memo-decrypt rewrite. +// --------------------------------------------------------------------------- + +describe.skipIf(isLiveBackend())( + 'ConfidentialFungibleToken + PublicSupply composition', + () => { + beforeEach(async () => { + cft = await deploy(); }); - }); - describe('supply invariant', () => { - it('totalSupply equals net minted across a mint/burn sequence', async () => { - await registerAll(); - await cft.mint(ALICE.accountId, 100n); - await cft.mint(BOB.accountId, 50n); - expect(await cft.totalSupply()).toBe(150n); + const registerAll = async () => { + for (const u of [ALICE, BOB]) { + await cft.privateState.switchIdentity(u.secretKey, u.encryptionKey); + await cft.register(); + } + }; - // Alice sweeps her 100 and burns 40: net minted is now 110. + // Mints `amount` to Alice, sweeps it into spendable, and caches the swept + // balance so it can later be debited (burned). Leaves Alice active. + const fundAlice = async (amount: bigint) => { + await registerAll(); await cft.privateState.switchIdentity( ALICE.secretKey, ALICE.encryptionKey, ); + await cft.mint(ALICE.accountId, amount); await cft.sweep(); await cft.privateState.cachePlaintext( await cft.balanceOf(ALICE.accountId), - 100n, + amount, ); - await cft.burn(40n); + }; + + describe('mint', () => { + it('increases totalSupply by exactly the minted value', async () => { + await registerAll(); + expect(await cft.totalSupply()).toBe(0n); + + await cft.mint(ALICE.accountId, 100n); + expect(await cft.totalSupply()).toBe(100n); + + await cft.mint(ALICE.accountId, 50n); + expect(await cft.totalSupply()).toBe(150n); + }); + + it('accumulates supply across recipients', async () => { + await registerAll(); + await cft.mint(ALICE.accountId, 100n); + await cft.mint(BOB.accountId, 50n); + expect(await cft.totalSupply()).toBe(150n); + }); + }); + + describe('mint + burn round-trip', () => { + it('mints a genuinely spendable balance and burns it back to zero supply', async () => { + await fundAlice(100n); + expect(await cft.totalSupply()).toBe(100n); + + // The mint credited Alice the full 100: burning 100 only proves out if her + // spendable balance truly encrypts >= 100. It also drops supply to 0. + await cft.burn(100n); + expect(await cft.totalSupply()).toBe(0n); + }); + }); + + describe('burn', () => { + it('moves totalSupply in lockstep with the caller debit', async () => { + await fundAlice(100n); + + await cft.burn(40n); + expect(await cft.totalSupply()).toBe(60n); + + // Alice's spendable is now 60; burning it drops supply to 0 in lockstep. + await cft.privateState.cachePlaintext( + await cft.balanceOf(ALICE.accountId), + 60n, + ); + await cft.burn(60n); + expect(await cft.totalSupply()).toBe(0n); + }); + + it('reverts a burn that exceeds the caller balance, leaving supply intact', async () => { + await fundAlice(100n); + + await expect(cft.burn(101n)).rejects.toThrow( + 'ConfidentialFungibleToken: insufficient balance', + ); + // The value op reverted before the supply decrement, so supply is unchanged. + expect(await cft.totalSupply()).toBe(100n); + }); + }); + + describe('burnFrom', () => { + it('drops totalSupply by spending the spender allowance', async () => { + // Alice mints 100 and approves Bob for 40. + await fundAlice(100n); + await cft.approve(BOB.accountId, 40n); + expect(await cft.totalSupply()).toBe(100n); + + // Bob decrypts his escrow copy (40), then burns 25 of the allowance. + await cft.privateState.switchIdentity(BOB.secretKey, BOB.encryptionKey); + const escrow = await cft.allowance(ALICE.accountId, BOB.accountId); + await cft.privateState.cachePlaintext(escrow.spenderCt, 40n); + + await cft.burnFrom(ALICE.accountId, 25n); + expect(await cft.totalSupply()).toBe(75n); + }); + }); - expect(await cft.totalSupply()).toBe(110n); + describe('supply invariant', () => { + it('totalSupply equals net minted across a mint/burn sequence', async () => { + await registerAll(); + await cft.mint(ALICE.accountId, 100n); + await cft.mint(BOB.accountId, 50n); + expect(await cft.totalSupply()).toBe(150n); + + // Alice sweeps her 100 and burns 40: net minted is now 110. + await cft.privateState.switchIdentity( + ALICE.secretKey, + ALICE.encryptionKey, + ); + await cft.sweep(); + await cft.privateState.cachePlaintext( + await cft.balanceOf(ALICE.accountId), + 100n, + ); + await cft.burn(40n); + + expect(await cft.totalSupply()).toBe(110n); + }); }); - }); -}); + }, +); diff --git a/contracts/test/integration/specs/initStateIsolation.spec.ts b/contracts/test/integration/specs/initStateIsolation.spec.ts index cff007c2..794c015f 100644 --- a/contracts/test/integration/specs/initStateIsolation.spec.ts +++ b/contracts/test/integration/specs/initStateIsolation.spec.ts @@ -1,3 +1,4 @@ +import { isLiveBackend } from '@openzeppelin/compact-simulator'; import { describe, expect, it } from 'vitest'; import { ComposedTokensSimulator } from '../fixtures/composedTokens.js'; import { SharedInitCollisionSimulator } from '../fixtures/sharedInitCollision.js'; @@ -20,9 +21,13 @@ import { SharedInitCollisionSimulator } from '../fixtures/sharedInitCollision.js * The first block documents the bug (and would have to be deleted/inverted if * the compiler ever isolates transitive ledger state); the second block guards * the fix against regression. + * + * Dry-only: this is a compiler-semantics test. It constructs the simulators + * directly, never `.create()`, so nothing here deploys and the live backend has + * no bearing on the outcome. */ -describe('Initializable state isolation (#556)', () => { +describe.skipIf(isLiveBackend())('Initializable state isolation (#556)', () => { describe('the bug — shared Initializable across same-directory modules', () => { it('should treat module B as initialized after only module A is initialized', () => { const c = new SharedInitCollisionSimulator(); diff --git a/contracts/vitest.config.ts b/contracts/vitest.config.ts index cc0bb332..64c9ad12 100644 --- a/contracts/vitest.config.ts +++ b/contracts/vitest.config.ts @@ -8,9 +8,17 @@ import { configDefaults, defineConfig } from 'vitest/config'; * via the live backend registered in `live.setup`. Driven * by `MIDNIGHT_BACKEND=live` (set by the `test:live` script). * - `integration` — composed-contract specs (`test/integration/specs`). + * - `integration-live` — the same specs against the local stack, through the + * same harness as `unit-live`. Only the blocks a spec marks + * `runIf(isLiveBackend())` run; see `test:live integration`. * - `harness` — dry unit tests for the live harness itself (`test-utils`). * - `harness-live` — live smoke that the real wallet pool funds + resolves on * the node, before the expensive contract live specs. + * - `scripts` — dry unit tests for the live orchestrator (`scripts/live`). + * + * Only ONE live project may run per vitest invocation: each derives its wallets + * from `walletSeedsFor(VITEST_POOL_ID)`, so worker 1 of two live projects would + * resolve to the same genesis deployer. `live.globalSetup` enforces this. * * Coverage is a root-level concern (applies to whichever project runs with * `--coverage`); the `unit` project is the one gated in CI. @@ -51,10 +59,17 @@ const liveWorkers = process.env.MIDNIGHT_WALLET_SEED ), ); -// Publish the resolved count so `live.setup` can print `w/` in its -// per-worker banner. Workers inherit this env at fork time. +// Publish the resolved count as the process-wide fallback. Each live project +// ALSO sets it per-project below (see `liveWorkerCount`), because the two differ: +// a global value would make `integration-live` (one worker) print `w1/3`. process.env.MIDNIGHT_LIVE_WORKERS = String(liveWorkers); +// Per-project worker total, handed to `live.setup` so its `w/` banner +// matches that project's own `maxWorkers` rather than `unit-live`'s. +const liveWorkerCount = (workers: number) => ({ + MIDNIGHT_LIVE_WORKERS: String(workers), +}); + export default defineConfig({ test: { reporters: 'verbose', @@ -115,6 +130,7 @@ export default defineConfig({ // scheduler group: a multi-project run otherwise rejects two projects // that share a group but differ in `maxWorkers`. maxWorkers: liveWorkers, + env: liveWorkerCount(liveWorkers), sequence: { concurrent: false, groupOrder: 1 }, }, }, @@ -125,6 +141,31 @@ export default defineConfig({ include: ['test/integration/specs/**/*.spec.ts'], }, }, + { + // Same files as `integration`; `MIDNIGHT_BACKEND=live` (set by + // `test:live integration`) flips the `isLiveBackend()`-gated blocks — + // today the composed-deploy block-limit canary — on, and skips the dry + // functional ones. Reuses `unit-live`'s globalSetup (freshness + run + // lock) and setup (wallet pool + backend register) verbatim. + test: { + ...NODE, + ...LIVE_TIMEOUTS, + name: 'integration-live', + include: ['test/integration/specs/**/*.spec.ts'], + globalSetup: ['./test-utils/harness/live.globalSetup.ts'], + setupFiles: ['./test-utils/harness/live.setup.ts'], + // Only the deployer wallet is in play (the canary is a single rejected + // deploy, no `.as(alias)` impersonation), so no wallet partition is + // needed. Widen to `unit-live`-style per-worker partitioning once + // green functional live integration specs exist. + maxWorkers: 1, + env: liveWorkerCount(1), + // Own scheduler group: `unit-live` holds group 1 with a different + // `maxWorkers`, and a multi-project run rejects two projects that + // share a group but differ in it. + sequence: { concurrent: false, groupOrder: 2 }, + }, + }, { test: { ...NODE, @@ -144,6 +185,17 @@ export default defineConfig({ globalSetup: ['./test-utils/harness/live.globalSetup.ts'], }, }, + { + // Dry unit tests for the live-orchestrator services (scripts/live). The + // scripts tree sits at the repo root, one level above this config's + // root, hence the `../` include. A per-project `root: '..'` override was + // tried first and discovered no files under vitest 4.1.10. + test: { + ...NODE, + name: 'scripts', + include: ['../scripts/**/*.test.ts'], + }, + }, ], }, }); diff --git a/package.json b/package.json index 24440e78..a44c2027 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,7 @@ "test:coverage": "SKIP_ZK=true turbo run test:coverage --filter=@openzeppelin/compact-contracts --env-mode=loose --log-prefix=none", "test:live": "node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON scripts/test-live.ts", "test:integration": "SKIP_ZK=true turbo run test:integration --filter=@openzeppelin/compact-contracts --env-mode=loose --log-prefix=none", + "test:integration:live": "node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON scripts/test-live.ts integration", "test:harness": "turbo run test:harness --filter=@openzeppelin/compact-contracts --env-mode=loose --log-prefix=none", "test:harness:live": "turbo run test:harness:live --filter=@openzeppelin/compact-contracts --env-mode=loose --log-prefix=none", "lint": "biome check .", From f50fd5e42fa30c3ffb4511803a26bad2ef7ecf87 Mon Sep 17 00:00:00 2001 From: 0xisk <0xisk@proton.me> Date: Mon, 27 Jul 2026 15:31:49 +0200 Subject: [PATCH 08/16] test(scripts): cover the live orchestrator dry The orchestrator had no test coverage at all. Add a `scripts` vitest project and cover the two paths where a regression would be silent rather than loud. Target resolution is the first: ask for `integration` and a broken guard would run every live-ready unit category instead, with nothing in the output to say so. The cases pin that guard, including the one where the target is deliberately absent from the category list. Flake classification is the second, and the higher-consequence one: a regression there flips a real failure to FLAKY and the build goes green over a genuine bug. `classify` is extracted as a pure function so it can be tested without a stack. Report naming is covered too, since round 2 cannot classify an integration file if its `.spec.ts` extension is not stripped. The spawn wrappers are deliberately not tested. Asserting the arguments we pass to spawnSync restates the implementation and locks in call shapes that should stay free to change. --- scripts/live/test/live.test.ts | 174 +++++++++++++++++++++++++++++++++ 1 file changed, 174 insertions(+) create mode 100644 scripts/live/test/live.test.ts diff --git a/scripts/live/test/live.test.ts b/scripts/live/test/live.test.ts new file mode 100644 index 00000000..322c6330 --- /dev/null +++ b/scripts/live/test/live.test.ts @@ -0,0 +1,174 @@ +import path from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { classify } from '../LiveOrchestrator.ts'; +import { round2Report } from '../paths.ts'; +import { resolvePlan } from '../targets.ts'; + +/** + * Dry unit tests for the pure pieces of the live orchestrator: plan resolution, + * flake classification, and report naming. Nothing here touches docker, the + * node, or the artifact tree. + */ + +/** `liveCategories()` reads `src/`, so every case passes this explicitly to keep + * the tests independent of the on-disk category set. */ +const CATEGORIES = ['multisig', 'token'] as const; + +describe('resolvePlan', () => { + it('scopes to the integration target', () => { + const resolution = resolvePlan(['integration'], CATEGORIES); + + expect(resolution.ok).toBe(true); + if (!resolution.ok) return; + // 'integration' is deliberately NOT in CATEGORIES: it is not a `src/` + // category, so the guard has to match it before the category branch or the + // run falls through to the unscoped path (the original INV-10 bug). + expect(resolution.plan.targets).toStrictEqual([ + { name: 'integration', project: 'integration-live', defaultFilters: [] }, + ]); + expect(resolution.plan.integration).toBe(true); + expect(resolution.plan.skipped).toStrictEqual([]); + expect(resolution.plan.fileFilters).toStrictEqual([]); + }); + + it('passes trailing args after the integration target as file filters', () => { + const resolution = resolvePlan( + ['integration', 'confidentialFungibleToken'], + CATEGORIES, + ); + + expect(resolution.ok).toBe(true); + if (!resolution.ok) return; + expect(resolution.plan.fileFilters).toStrictEqual([ + 'confidentialFungibleToken', + ]); + expect(resolution.plan.integration).toBe(true); + }); + + it('scopes to a live-ready unit category', () => { + const resolution = resolvePlan(['multisig'], CATEGORIES); + + expect(resolution.ok).toBe(true); + if (!resolution.ok) return; + expect(resolution.plan.targets).toStrictEqual([ + { + name: 'multisig', + project: 'unit-live', + defaultFilters: ['src/multisig'], + }, + ]); + expect(resolution.plan.integration).toBe(false); + expect(resolution.plan.skipped).toStrictEqual([]); + }); + + it('passes trailing args after a category as file filters', () => { + const resolution = resolvePlan(['multisig', 'Forwarder'], CATEGORIES); + + expect(resolution.ok).toBe(true); + if (!resolution.ok) return; + expect(resolution.plan.fileFilters).toStrictEqual(['Forwarder']); + }); + + it('rejects a category that is not live-ready yet', () => { + const resolution = resolvePlan(['token'], CATEGORIES); + + expect(resolution.ok).toBe(false); + if (resolution.ok) return; + expect(resolution.message).toContain("'token' is not live-ready yet"); + expect(resolution.message).toContain('Ready categories: multisig'); + expect(resolution.message).toContain("'integration'"); + }); + + it('runs only live-ready categories when unscoped, reporting the rest', () => { + const resolution = resolvePlan([], CATEGORIES); + + expect(resolution.ok).toBe(true); + if (!resolution.ok) return; + expect(resolution.plan.targets).toStrictEqual([ + { + name: 'multisig', + project: 'unit-live', + defaultFilters: ['src/multisig'], + }, + ]); + expect(resolution.plan.skipped).toStrictEqual(['token']); + expect(resolution.plan.fileFilters).toStrictEqual([]); + expect(resolution.plan.integration).toBe(false); + }); + + it('treats a non-category first arg as a file filter over every target', () => { + const resolution = resolvePlan(['someFileFilter'], CATEGORIES); + + expect(resolution.ok).toBe(true); + if (!resolution.ok) return; + expect(resolution.plan.targets).toStrictEqual([ + { + name: 'multisig', + project: 'unit-live', + defaultFilters: ['src/multisig'], + }, + ]); + expect(resolution.plan.fileFilters).toStrictEqual(['someFileFilter']); + expect(resolution.plan.skipped).toStrictEqual(['token']); + }); +}); + +describe('classify', () => { + it('demotes a round-2 pass to flaky', () => { + expect( + classify(['a.test.ts'], new Map([['a.test.ts', 'passed']])), + ).toStrictEqual({ flaky: ['a.test.ts'], real: [] }); + }); + + it('keeps a file that failed round 2 as a real failure', () => { + expect( + classify(['a.test.ts'], new Map([['a.test.ts', 'failed']])), + ).toStrictEqual({ flaky: [], real: ['a.test.ts'] }); + }); + + it('keeps a file missing from the round-2 map as a real failure', () => { + expect(classify(['a.test.ts'], new Map())).toStrictEqual({ + flaky: [], + real: ['a.test.ts'], + }); + }); + + it('splits a mixed round-2 result', () => { + const round2 = new Map([ + ['flake.test.ts', 'passed'], + ['broken.test.ts', 'failed'], + ['crashed.test.ts', 'skipped'], + ]); + + expect( + classify( + ['flake.test.ts', 'broken.test.ts', 'crashed.test.ts', 'gone.test.ts'], + round2, + ), + ).toStrictEqual({ + flaky: ['flake.test.ts'], + real: ['broken.test.ts', 'crashed.test.ts', 'gone.test.ts'], + }); + }); +}); + +describe('round2Report', () => { + it('strips the unit `.test.ts` extension', () => { + expect(path.basename(round2Report('/repo/src/multisig/Foo.test.ts'))).toBe( + 'live-r2-Foo.json', + ); + }); + + it('strips the integration `.spec.ts` extension', () => { + expect( + path.basename(round2Report('/repo/test/integration/specs/Bar.spec.ts')), + ).toBe('live-r2-Bar.json'); + }); + + it('writes the report under the repo logs directory', () => { + const report = round2Report('/repo/src/multisig/Foo.test.ts'); + + expect(path.basename(path.dirname(report))).toBe('logs'); + expect(path.isAbsolute(report)).toBe(true); + }); +}); From 2d7cb82883f80d437f42f7ecb0502e1d35daa7b0 Mon Sep 17 00:00:00 2001 From: 0xisk <0xisk@proton.me> Date: Tue, 28 Jul 2026 15:08:10 +0200 Subject: [PATCH 09/16] fix(scripts): make the stale run-lock reclaim atomic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reclaiming a lock left behind by a killed run overwrote it with a plain write, so two runs that both found the same dead pid could both "win" it. The last writer owned the file while the other ran on believing it held the lock — its `release()` sees a foreign pid and silently no-ops — and two orchestrators then drove the one shared node, which is exactly what the lock exists to prevent. Every contended step is now a single atomic filesystem call. A stale lock is claimed by renaming it aside (POSIX moves the inode once, so the second reclaimer gets ENOENT), then re-created under `wx`, so a third run that started in between still wins or loses cleanly. Both losing paths report the same "already in progress" rejection, unchanged in wording for the live-pid case. Adds the lock's first tests: fresh acquire, live-pid rejection, stale reclaim, no leftover stale copy, and release only unlinking a lock this run owns. Refs OpenZeppelin/compact-contracts#717 --- scripts/live/RunLock.ts | 67 +++++++++++++++++++++---- scripts/live/test/live.test.ts | 89 ++++++++++++++++++++++++++++++++-- 2 files changed, 142 insertions(+), 14 deletions(-) diff --git a/scripts/live/RunLock.ts b/scripts/live/RunLock.ts index 6fdbeeb4..6401298b 100644 --- a/scripts/live/RunLock.ts +++ b/scripts/live/RunLock.ts @@ -1,4 +1,10 @@ -import { mkdirSync, readFileSync, unlinkSync, writeFileSync } from 'node:fs'; +import { + mkdirSync, + readFileSync, + renameSync, + unlinkSync, + writeFileSync, +} from 'node:fs'; import { LOGS, VERIFY_LOCK } from './paths.ts'; interface LockInfo { @@ -41,25 +47,66 @@ export class RunLock { } } - /** Take the lock, or throw if a live process already holds it. */ + /** The "someone else holds it" rejection, shared by both losing paths. */ + #heldBy(info: LockInfo | undefined): Error { + const who = info ? ` (pid ${info.pid}, started ${info.startedAt})` : ''; + return new Error( + `another test:live run is already in progress${who}. ` + + `Wait for it, or remove ${this.#path}.`, + ); + } + + /** + * Take the lock, or throw if another run holds it. + * + * Every step that can be contended is a single atomic filesystem call, because + * two runs starting at the same moment reach each of them together: + * - `wx` create — only one process can create the path; + * - `rename` of a stale lock — POSIX moves the inode once, so the second + * reclaimer gets ENOENT and loses. + * A plain overwrite would let both reclaimers "win": the last writer owns the + * file while the other runs on believing it holds the lock (its `release()` + * finds a foreign pid and silently no-ops), so two orchestrators would drive + * the one node — exactly what the lock exists to prevent. + */ acquire(): void { mkdirSync(LOGS, { recursive: true }); const stamp = JSON.stringify({ pid: process.pid, startedAt: new Date().toISOString(), }); + try { + writeFileSync(this.#path, stamp, { flag: 'wx' }); + return; + } catch (e) { + if ((e as NodeJS.ErrnoException).code !== 'EEXIST') throw e; + } + + const info = this.#read(); + if (info && pidAlive(info.pid)) throw this.#heldBy(info); + + // Stale. Claim the right to reclaim it by moving it aside, then create the + // lock fresh under `wx` — so a third run that started in between still wins + // or loses cleanly rather than sharing. + const stolen = `${this.#path}.stale.${process.pid}`; + try { + renameSync(this.#path, stolen); + } catch (e) { + // ENOENT means another run reclaimed it first; anything else (a + // permissions problem on `logs/`) is its own fault and says so. + if ((e as NodeJS.ErrnoException).code !== 'ENOENT') throw e; + throw this.#heldBy(this.#read()); + } + try { + unlinkSync(stolen); + } catch { + // best effort — the lock itself is what matters + } try { writeFileSync(this.#path, stamp, { flag: 'wx' }); } catch (e) { if ((e as NodeJS.ErrnoException).code !== 'EEXIST') throw e; - const info = this.#read(); - if (info && pidAlive(info.pid)) { - throw new Error( - `another test:live run is already in progress (pid ${info.pid}, ` + - `started ${info.startedAt}). Wait for it, or remove ${this.#path}.`, - ); - } - writeFileSync(this.#path, stamp); // stale — reclaim + throw this.#heldBy(this.#read()); } } diff --git a/scripts/live/test/live.test.ts b/scripts/live/test/live.test.ts index 322c6330..d1796e5b 100644 --- a/scripts/live/test/live.test.ts +++ b/scripts/live/test/live.test.ts @@ -1,13 +1,17 @@ +import { mkdtempSync, readdirSync, readFileSync, writeFileSync } from 'node:fs'; +import os from 'node:os'; import path from 'node:path'; -import { describe, expect, it } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { classify } from '../LiveOrchestrator.ts'; import { round2Report } from '../paths.ts'; +import { RunLock } from '../RunLock.ts'; import { resolvePlan } from '../targets.ts'; /** - * Dry unit tests for the pure pieces of the live orchestrator: plan resolution, - * flake classification, and report naming. Nothing here touches docker, the - * node, or the artifact tree. + * Dry unit tests for the pure pieces of the live orchestrator (plan resolution, + * target listing, flake classification, report naming) plus the run lock, which + * is filesystem-only. Nothing here touches docker, the node, or the artifact + * tree. */ /** `liveCategories()` reads `src/`, so every case passes this explicitly to keep @@ -113,6 +117,83 @@ describe('resolvePlan', () => { }); }); +describe('RunLock', () => { + /** Out of every kernel's pid range, so `process.kill(pid, 0)` can only report + * "no such process" — a stale lock without having to kill a real one. */ + const DEAD_PID = 2 ** 31 - 1; + + let dir: string; + let lockPath: string; + + const stamp = (pid: number): void => { + writeFileSync(lockPath, JSON.stringify({ pid, startedAt: 'earlier' })); + }; + const holder = (): number => + (JSON.parse(readFileSync(lockPath, 'utf8')) as { pid: number }).pid; + + beforeEach(() => { + dir = mkdtempSync(path.join(os.tmpdir(), 'runlock-')); + lockPath = path.join(dir, '.live-verify.lock'); + }); + + afterEach(() => { + new RunLock(lockPath).release(); + }); + + it('stamps the lock with our pid when it is free', () => { + new RunLock(lockPath).acquire(); + + expect(holder()).toBe(process.pid); + }); + + it('refuses a lock held by a live process', () => { + // Our parent is alive by construction, and is not us. + stamp(process.ppid); + + expect(() => new RunLock(lockPath).acquire()).toThrow( + `another test:live run is already in progress (pid ${process.ppid}, started earlier)`, + ); + expect(holder()).toBe(process.ppid); + }); + + it('reclaims a lock left behind by a dead process', () => { + stamp(DEAD_PID); + + new RunLock(lockPath).acquire(); + + expect(holder()).toBe(process.pid); + }); + + it('leaves nothing behind when it reclaims', () => { + stamp(DEAD_PID); + + new RunLock(lockPath).acquire(); + + // The reclaim moves the stale file aside to win it atomically; that copy is + // a step, not an artifact. + expect(readdirSync(dir)).toStrictEqual([path.basename(lockPath)]); + }); + + it('releases a lock it owns', () => { + const lock = new RunLock(lockPath); + lock.acquire(); + + lock.release(); + + expect(readdirSync(dir)).toStrictEqual([]); + }); + + it('leaves a lock owned by another run alone', () => { + stamp(DEAD_PID); + + // A run that lost a stale-lock race must not delete the winner's lock on the + // way out, so `release` checks ownership rather than just unlinking. + new RunLock(lockPath).release(); + + expect(holder()).toBe(DEAD_PID); + }); +}); + describe('classify', () => { it('demotes a round-2 pass to flaky', () => { expect( From ce1183418425ec8b5c6ae87ad19a3a8c94e9c22c Mon Sep 17 00:00:00 2001 From: 0xisk <0xisk@proton.me> Date: Tue, 28 Jul 2026 15:09:02 +0200 Subject: [PATCH 10/16] fix(scripts): survive a truncated live JSON report MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `fileStatuses` documents `undefined` for a run that produced no result, and both rounds rely on it to abort gracefully with INFRA_ABORT. But a killed vitest can leave a partially-written report that still passes `existsSync`, and `JSON.parse` then threw straight through the callers, crashing the orchestrator instead — after the stack was already up, and with no verdict. Parsing now falls into the same no-result path and names the unreadable file, since the caller's "produced no results file" message would otherwise misdescribe it. Adds the first tests for the report reader: statuses mapped, the `--passWithNoTests` empty report, a missing report, and a truncated one. Refs OpenZeppelin/compact-contracts#717 --- scripts/live/VitestRunner.ts | 25 ++++++++++--- scripts/live/test/live.test.ts | 68 +++++++++++++++++++++++++++++++--- 2 files changed, 83 insertions(+), 10 deletions(-) diff --git a/scripts/live/VitestRunner.ts b/scripts/live/VitestRunner.ts index fc18dc34..d40acf24 100644 --- a/scripts/live/VitestRunner.ts +++ b/scripts/live/VitestRunner.ts @@ -1,5 +1,5 @@ import { existsSync, readFileSync } from 'node:fs'; -import { CONTRACTS, PROGRESS_REPORTER, VITEST_BIN } from './paths.ts'; +import { CONTRACTS, PROGRESS_REPORTER, rel, VITEST_BIN } from './paths.ts'; import { run } from './shell.ts'; import type { LiveTarget } from './targets.ts'; @@ -53,13 +53,28 @@ export class VitestRunner { /** * File name → status for every file in the report. * - * @returns `undefined` when no report exists at all — the run was blocked - * (dirty node / lock) or crashed before writing one, which callers must treat - * as an infrastructure abort rather than a test failure. + * @returns `undefined` when no *readable* report exists — the run was blocked + * (dirty node / lock), crashed before writing one, or was killed mid-write + * and left truncated JSON behind. Callers must treat that as an + * infrastructure abort rather than a test failure. */ fileStatuses(reportPath: string): Map | undefined { if (!existsSync(reportPath)) return undefined; - const report = JSON.parse(readFileSync(reportPath, 'utf8')) as JsonReport; + let report: JsonReport; + try { + report = JSON.parse(readFileSync(reportPath, 'utf8')) as JsonReport; + } catch (e) { + // A killed vitest can leave a partial report that still passes `existsSync`, + // so parsing is a second way to have no result — not an exception to throw + // through the callers, which are written to abort gracefully on `undefined`. + // Named here because the caller's message ("produced no results file") + // would otherwise misdescribe an unreadable one. + console.log( + `\ncould not read ${rel(reportPath)}: ` + + `${e instanceof Error ? e.message : String(e)}`, + ); + return undefined; + } return new Map( (report.testResults ?? []).map((r) => [r.name, r.status] as const), ); diff --git a/scripts/live/test/live.test.ts b/scripts/live/test/live.test.ts index d1796e5b..fb6c48a6 100644 --- a/scripts/live/test/live.test.ts +++ b/scripts/live/test/live.test.ts @@ -1,17 +1,18 @@ import { mkdtempSync, readdirSync, readFileSync, writeFileSync } from 'node:fs'; import os from 'node:os'; import path from 'node:path'; -import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { classify } from '../LiveOrchestrator.ts'; import { round2Report } from '../paths.ts'; import { RunLock } from '../RunLock.ts'; import { resolvePlan } from '../targets.ts'; +import { VitestRunner } from '../VitestRunner.ts'; /** - * Dry unit tests for the pure pieces of the live orchestrator (plan resolution, - * target listing, flake classification, report naming) plus the run lock, which - * is filesystem-only. Nothing here touches docker, the node, or the artifact - * tree. + * Dry unit tests for the live orchestrator's pure pieces (plan resolution, flake + * classification, report naming) plus the two services that only touch the + * filesystem: the run lock, and reading back a vitest JSON report. Nothing here + * touches docker, the node, or the artifact tree. */ /** `liveCategories()` reads `src/`, so every case passes this explicitly to keep @@ -194,6 +195,63 @@ describe('RunLock', () => { }); }); +describe('VitestRunner.fileStatuses', () => { + let dir: string; + const report = (name: string, body: string): string => { + const p = path.join(dir, name); + writeFileSync(p, body); + return p; + }; + + beforeEach(() => { + dir = mkdtempSync(path.join(os.tmpdir(), 'live-report-')); + }); + + it('maps each file in the report to its status', () => { + const p = report( + 'ok.json', + JSON.stringify({ + testResults: [ + { name: 'a.test.ts', status: 'passed' }, + { name: 'b.test.ts', status: 'failed' }, + ], + }), + ); + + expect(new VitestRunner().fileStatuses(p)).toStrictEqual( + new Map([ + ['a.test.ts', 'passed'], + ['b.test.ts', 'failed'], + ]), + ); + }); + + it('returns an empty map when the run matched no files', () => { + // vitest still writes a report under `--passWithNoTests`, with no results. + const p = report('empty.json', JSON.stringify({ testResults: [] })); + + expect(new VitestRunner().fileStatuses(p)).toStrictEqual(new Map()); + }); + + it('reports no result when the report is missing', () => { + expect( + new VitestRunner().fileStatuses(path.join(dir, 'absent.json')), + ).toBeUndefined(); + }); + + it('reports no result when the report is truncated', () => { + // A killed vitest leaves a partial file that still exists, so parsing has to + // fail into the same graceful abort rather than throwing through the caller. + const p = report('partial.json', '{"testResults":[{"name":"a.test.ts"'); + const logged = vi.spyOn(console, 'log').mockImplementation(() => {}); + + expect(new VitestRunner().fileStatuses(p)).toBeUndefined(); + expect(logged.mock.calls.flat().join('\n')).toContain('partial.json'); + + logged.mockRestore(); + }); +}); + describe('classify', () => { it('demotes a round-2 pass to flaky', () => { expect( From d0885c8f25d4f4b008c7374c9959caf5a94aea2b Mon Sep 17 00:00:00 2001 From: 0xisk <0xisk@proton.me> Date: Tue, 28 Jul 2026 15:09:40 +0200 Subject: [PATCH 11/16] fix(scripts): let the live verdict drain before exiting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The orchestrator ended on `process.exit(code)`. Under CI its stdout is a pipe (`tee`, a log collector), where writes are asynchronous and `process.exit` drops whatever is still queued — and what a run prints last is the verdict block, so that is exactly what a piped run could lose. Setting `process.exitCode` instead lets the queued writes flush. Nothing holds the loop open once `main` resolves: every child is awaited to `close`, and Node unrefs signal listeners. The handler in `shell.ts` keeps `process.exit`, where leaving immediately after synchronous cleanup is the point. Verified piped: `--list` still exits 0 and a non-live-ready category still exits 2, both with their output intact. Refs OpenZeppelin/compact-contracts#717 --- scripts/test-live.ts | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/scripts/test-live.ts b/scripts/test-live.ts index a22a865d..c30b3c71 100644 --- a/scripts/test-live.ts +++ b/scripts/test-live.ts @@ -91,9 +91,19 @@ async function main(): Promise { } } +// `process.exitCode`, not `process.exit()`: a run under CI has its stdout piped +// into `tee`/a log collector, where writes are asynchronous, and `process.exit` +// discards whatever is still queued. What a run prints last is the verdict block, +// so that is precisely what would be lost. Nothing holds the loop open once +// `main` resolves — every child is awaited to `close`, and Node unrefs signal +// listeners — so the process still exits immediately. The signal handler in +// `shell.ts` keeps `process.exit` on purpose: it has to leave the moment its +// synchronous cleanup returns. main() - .then((code) => process.exit(code)) + .then((code) => { + process.exitCode = code; + }) .catch((e) => { console.log(e instanceof Error ? e.message : String(e)); - process.exit(INFRA_ABORT); + process.exitCode = INFRA_ABORT; }); From a5edfbd63cde45d63de74223c52d82c3ca28cd41 Mon Sep 17 00:00:00 2001 From: 0xisk <0xisk@proton.me> Date: Tue, 28 Jul 2026 15:12:19 +0200 Subject: [PATCH 12/16] fix(scripts): abort a live run that matched no test file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `resolvePlan` cannot tell a mistyped target from a file filter, so `yarn test:live multsig` becomes a positional filter across every live-ready target. With `--passWithNoTests` vitest then exits 0 and writes a report with no results, and the run finished as PASSED having executed nothing — the one verdict a live run must never produce by accident. Round 1 now counts the files each target reported and aborts with INFRA_ABORT when the whole run matched none, naming the filter and pointing at `--list`. A single target contributing zero files is still a pass: in an unscoped run a name filter may only exist under some targets, which is why `--passWithNoTests` is there. Prefers this over near-miss detection in `resolvePlan`, which would have to guess which unrecognised arguments are typos, and would still miss a mistyped file filter. Also adds the orchestrator's first round tests, over stand-in collaborators: this abort, the no-report abort, and the green path. Refs OpenZeppelin/compact-contracts#717 --- scripts/live/LiveOrchestrator.ts | 18 ++++++- scripts/live/VitestRunner.ts | 5 +- scripts/live/test/live.test.ts | 90 ++++++++++++++++++++++++++++++-- 3 files changed, 106 insertions(+), 7 deletions(-) diff --git a/scripts/live/LiveOrchestrator.ts b/scripts/live/LiveOrchestrator.ts index 86a0f7d6..46492816 100644 --- a/scripts/live/LiveOrchestrator.ts +++ b/scripts/live/LiveOrchestrator.ts @@ -61,8 +61,8 @@ export function classify( * turns the build red, but it is reported loudly. * * Anything that prevents classification (no report written, a non-zero exit with - * no failing files, a stack that will not come up) aborts with - * {@link INFRA_ABORT} rather than being guessed at. + * no failing files, a run that matched no test file at all, a stack that will not + * come up) aborts with {@link INFRA_ABORT} rather than being guessed at. */ export class LiveOrchestrator { readonly #plan: LivePlan; @@ -155,6 +155,7 @@ export class LiveOrchestrator { async #round1(): Promise { const { targets, fileFilters } = this.#plan; const failed: FailedFile[] = []; + let filesRun = 0; for (const [i, target] of targets.entries()) { banner(`ROUND 1 · ${target.name} (${i + 1}/${targets.length})`); @@ -196,11 +197,24 @@ export class LiveOrchestrator { return undefined; } + filesRun += statuses.size; failed.push(...targetFailed.map((file) => ({ file, target }))); console.log( `\n${target.name}: ${statuses.size} file(s), ${targetFailed.length} failed`, ); } + + if (targets.length > 0 && filesRun === 0) { + console.log( + `\nno test file matched across ${targets.map((t) => t.name).join(', ')}` + + (fileFilters.length ? ` (filter: ${fileFilters.join(' ')})` : '') + + ' — nothing ran, so there is no result to report.\n' + + 'A mistyped target is the usual cause: an unrecognised first argument ' + + 'is a file filter, not an error, so it matches nothing across every ' + + "live-ready target. Run 'yarn test:live --list' for the target names.", + ); + return undefined; + } return failed; } diff --git a/scripts/live/VitestRunner.ts b/scripts/live/VitestRunner.ts index d40acf24..6c856c55 100644 --- a/scripts/live/VitestRunner.ts +++ b/scripts/live/VitestRunner.ts @@ -35,7 +35,10 @@ export class VitestRunner { 'run', '--project', project, - // A target filtered down to zero matching files is a pass, not an error. + // One target filtered down to zero matching files is a pass, not an + // error: a name filter may only exist under some of the targets in an + // unscoped run. Zero across the WHOLE run is a different thing, and + // `LiveOrchestrator.#round1` rejects it. '--passWithNoTests', // `default` prints one line per file (piped) plus failures/summary; the // progress reporter adds the worker-tagged, counted per-test line. diff --git a/scripts/live/test/live.test.ts b/scripts/live/test/live.test.ts index fb6c48a6..3e2aa9d0 100644 --- a/scripts/live/test/live.test.ts +++ b/scripts/live/test/live.test.ts @@ -2,19 +2,35 @@ import { mkdtempSync, readdirSync, readFileSync, writeFileSync } from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { classify } from '../LiveOrchestrator.ts'; +import type { ArtifactCompiler } from '../ArtifactCompiler.ts'; +import { + classify, + INFRA_ABORT, + LiveOrchestrator, +} from '../LiveOrchestrator.ts'; +import type { LiveStack } from '../LiveStack.ts'; import { round2Report } from '../paths.ts'; +import type { Reporter } from '../Reporter.ts'; import { RunLock } from '../RunLock.ts'; import { resolvePlan } from '../targets.ts'; import { VitestRunner } from '../VitestRunner.ts'; /** * Dry unit tests for the live orchestrator's pure pieces (plan resolution, flake - * classification, report naming) plus the two services that only touch the - * filesystem: the run lock, and reading back a vitest JSON report. Nothing here - * touches docker, the node, or the artifact tree. + * classification, report naming), the two services that only touch the filesystem + * (the run lock, and reading back a vitest JSON report), and a round driven + * through stand-in collaborators. Nothing here touches docker, the node, or the + * artifact tree. */ +// The one collaborator the orchestrator does not take by injection is the +// harness-smoke spawn, so `run` is stubbed to succeed. Everything else in +// `shell.ts` stays real (`banner` prints through the console spies below). +vi.mock('../shell.ts', async (importOriginal) => ({ + ...(await importOriginal()), + run: async () => 0, +})); + /** `liveCategories()` reads `src/`, so every case passes this explicitly to keep * the tests independent of the on-disk category set. */ const CATEGORIES = ['multisig', 'token'] as const; @@ -252,6 +268,72 @@ describe('VitestRunner.fileStatuses', () => { }); }); +describe('LiveOrchestrator', () => { + // Deliberately not a real category, so clearing stale reports finds nothing. + const TARGET = { + name: 'faketarget', + project: 'unit-live', + defaultFilters: ['src/faketarget'], + } as const; + + /** A round wired to stand-ins: every collaborator but the harness-smoke spawn + * is constructor-injected, so a whole round runs without docker or vitest. */ + const roundOver = ( + fileStatuses: () => Map | undefined, + fileFilters: readonly string[] = [], + ): LiveOrchestrator => + new LiveOrchestrator({ + plan: { targets: [TARGET], skipped: [], fileFilters, integration: false }, + stack: { up: async () => 0, stop: () => {} } as unknown as LiveStack, + compiler: { + compileVerified: async () => true, + } as unknown as ArtifactCompiler, + runner: { run: async () => 0, fileStatuses } as unknown as VitestRunner, + reporter: { + firstRunGreen: () => 0, + verdict: () => 0, + } as unknown as Reporter, + }); + + let logged: ReturnType; + + beforeEach(() => { + logged = vi.spyOn(console, 'log').mockImplementation(() => {}); + }); + + afterEach(() => { + logged.mockRestore(); + }); + + const output = (): string => logged.mock.calls.flat().join('\n'); + + it('aborts when the run matched no test file', async () => { + // A mistyped target is indistinguishable from a file filter, and + // `--passWithNoTests` makes vitest exit 0 with an empty report — so without + // this guard the run reports PASSED having executed nothing. + const code = await roundOver(() => new Map(), ['multsig']).run(); + + expect(code).toBe(INFRA_ABORT); + expect(output()).toContain('no test file matched'); + expect(output()).toContain('filter: multsig'); + }); + + it('aborts when a target wrote no report at all', async () => { + const code = await roundOver(() => undefined).run(); + + expect(code).toBe(INFRA_ABORT); + expect(output()).toContain('produced no results file'); + }); + + it('reports the first run green when every file passed', async () => { + const code = await roundOver( + () => new Map([['a.test.ts', 'passed']]), + ).run(); + + expect(code).toBe(0); + }); +}); + describe('classify', () => { it('demotes a round-2 pass to flaky', () => { expect( From 7ff77fccc86b644664ed316a735808ce65c56126 Mon Sep 17 00:00:00 2001 From: 0xisk <0xisk@proton.me> Date: Tue, 28 Jul 2026 15:12:43 +0200 Subject: [PATCH 13/16] test(scripts): cover the live target list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `listTargets` is the single source of truth for the CI matrix, so dropping an entry would surface only as a live target nobody runs — a silently missing job rather than a failure. It was the one pure function in `targets.ts` with no test. Covers the live-ready categories plus `integration`, and that `integration` survives an empty category list, since it is not a `src/` category and so does not depend on LIVE_READY. Refs OpenZeppelin/compact-contracts#717 --- scripts/live/test/live.test.ts | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/scripts/live/test/live.test.ts b/scripts/live/test/live.test.ts index 3e2aa9d0..01e6fd33 100644 --- a/scripts/live/test/live.test.ts +++ b/scripts/live/test/live.test.ts @@ -12,7 +12,7 @@ import type { LiveStack } from '../LiveStack.ts'; import { round2Report } from '../paths.ts'; import type { Reporter } from '../Reporter.ts'; import { RunLock } from '../RunLock.ts'; -import { resolvePlan } from '../targets.ts'; +import { listTargets, resolvePlan } from '../targets.ts'; import { VitestRunner } from '../VitestRunner.ts'; /** @@ -35,6 +35,20 @@ vi.mock('../shell.ts', async (importOriginal) => ({ * the tests independent of the on-disk category set. */ const CATEGORIES = ['multisig', 'token'] as const; +describe('listTargets', () => { + it('lists the live-ready categories plus the integration target', () => { + // CI builds its matrix from this (`test:live --list`), so a dropped entry + // would surface only as a silently missing job — a live target nobody runs. + expect(listTargets(CATEGORIES)).toStrictEqual(['multisig', 'integration']); + }); + + it('still offers the integration target when no category is live-ready', () => { + // `integration` is not a `src/` category, so it does not depend on + // LIVE_READY the way the unit categories do. + expect(listTargets([])).toStrictEqual(['integration']); + }); +}); + describe('resolvePlan', () => { it('scopes to the integration target', () => { const resolution = resolvePlan(['integration'], CATEGORIES); From 44009e0da6b17f58e06f83841fd8e151775ff21d Mon Sep 17 00:00:00 2001 From: 0xisk <0xisk@proton.me> Date: Thu, 30 Jul 2026 15:07:01 +0200 Subject: [PATCH 14/16] test(integration): scope the canary to ledger v8 The block-limit rejection the canary asserts is a property of the pinned ledger, not of the contract. The note now names the version it was verified against (ledger-v8 8.1.0) instead of reading as permanent, and is shorter for it. Both assertions carry messages, so the day the budget moves the failure explains itself. A successful deploy names the guards to invert. A rejection for some other reason separates an unrelated cause (funding, proving, a submission bounce) from a reworded node string. --- .../specs/confidentialFungibleToken.spec.ts | 39 ++++++++++++------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/contracts/test/integration/specs/confidentialFungibleToken.spec.ts b/contracts/test/integration/specs/confidentialFungibleToken.spec.ts index 73672029..67f99089 100644 --- a/contracts/test/integration/specs/confidentialFungibleToken.spec.ts +++ b/contracts/test/integration/specs/confidentialFungibleToken.spec.ts @@ -24,12 +24,13 @@ import { ConfidentialFungibleTokenPublicSupplySimulator } from '../fixtures/conf * Balances stay hidden; `totalSupply` (the public aggregate) is what we assert. * * Backend split (`--project integration` vs `integration-live`), in file order: - * 1. The LIVE block is the block-limit canary: it asserts the composed deploy - * is rejected, which is what makes "no green functional live integration - * coverage" a verified claim rather than an assumption. - * 2. The functional block is DRY-ONLY, because of (1) — and because it drives - * confidential identities through `switchIdentity` / `cachePlaintext`, which - * the live backend throws on. + * 1. The LIVE block is the block-limit canary. It asserts the composed deploy + * is rejected, which turns "no green functional live integration coverage" + * into a verified claim rather than an assumption. True as of ledger v8; see + * the note above it. + * 2. The functional block is DRY-ONLY, for two reasons: the deploy in (1) is + * rejected, and it drives confidential identities through `switchIdentity` / + * `cachePlaintext`, which the live backend throws on. */ // Mirrors the base suite's deterministic identity setup. @@ -75,10 +76,11 @@ const deploy = () => // larger and hits the same wall. // // Rather than skip and hide that, ASSERT the rejection. A green skip would let -// the branch claim live coverage it does not have; this canary instead fails the -// day a staged deploy, a smaller composition, or a looser ledger lets the -// composed contract through — which is exactly when green functional live -// integration coverage becomes possible, and the guards below can be inverted. +// the branch claim live coverage it does not have. +// +// Scoped to ledger v8 (`@midnight-ntwrk/ledger-v8` 8.1.0). The budget is a ledger +// property, not a contract property, so a ledger bump can move it. If this block +// goes red, the composed deploy fits now: drop it and invert the guards below. // --------------------------------------------------------------------------- describe.runIf(isLiveBackend())( @@ -95,7 +97,9 @@ describe.runIf(isLiveBackend())( } expect( error, - 'expected the node to reject the oversized deploy', + 'the composed deploy SUCCEEDED, so it no longer exceeds the block budget: ' + + 'delete this block and invert the guards in this file, to run the ' + + 'functional suite live instead.', ).toBeDefined(); const detail = [ (error as Error)?.message, @@ -104,10 +108,15 @@ describe.runIf(isLiveBackend())( ] .map((x) => String(x ?? '')) .join(' | '); - // The node's exact words. Asserted in full rather than a loose - // `/block limits/` match, which an unrelated deploy failure could satisfy - // and report as a false green. - expect(detail).toContain( + // The node's exact words, as of ledger v8. Asserted in full rather than a + // loose `/block limits/` match, which an unrelated deploy failure could + // satisfy and report as a false green. + expect( + detail, + 'the deploy was rejected, but not for the block budget. Either the cause ' + + 'is unrelated (funding, proving, a submission bounce), or a ledger bump ' + + 'reworded the message. Re-verify against the node before relaxing this.', + ).toContain( '1010: Invalid Transaction: Transaction would exhaust the block limits', ); }); From d14268df57c4b13e30b9fd4a42f855e87ec9069a Mon Sep 17 00:00:00 2001 From: 0xisk <0xisk@proton.me> Date: Thu, 30 Jul 2026 15:07:07 +0200 Subject: [PATCH 15/16] docs(vitest): rephrase the integration-live note An em-dash pair split "flips ... on" across the aside it inserted, which left a grammatical but unreadable sentence. Three sentences now, with the aside moved out. --- contracts/vitest.config.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/contracts/vitest.config.ts b/contracts/vitest.config.ts index 64c9ad12..c24bd1d5 100644 --- a/contracts/vitest.config.ts +++ b/contracts/vitest.config.ts @@ -142,11 +142,13 @@ export default defineConfig({ }, }, { - // Same files as `integration`; `MIDNIGHT_BACKEND=live` (set by - // `test:live integration`) flips the `isLiveBackend()`-gated blocks — - // today the composed-deploy block-limit canary — on, and skips the dry - // functional ones. Reuses `unit-live`'s globalSetup (freshness + run - // lock) and setup (wallet pool + backend register) verbatim. + // Same files as `integration`, run with `MIDNIGHT_BACKEND=live` (set by + // `test:live integration`). That turns on the `isLiveBackend()`-gated + // blocks and skips the dry functional ones. Today the only live-gated + // block is the composed-deploy block-limit canary. + // + // Reuses `unit-live`'s globalSetup (freshness + run lock) and setup + // (wallet pool + backend register) verbatim. test: { ...NODE, ...LIVE_TIMEOUTS, From 35ae397be34837ba812ef85efd9174df041b04f0 Mon Sep 17 00:00:00 2001 From: 0xisk <0xisk@proton.me> Date: Thu, 30 Jul 2026 15:07:16 +0200 Subject: [PATCH 16/16] docs(contributing): document the integration target The live section predated the integration target, so it claimed multisig was the only thing test:live could run, and told the reader to bring the stack up and down by hand. * integration is a target rather than a category, so an unscoped run skips it. --list is what the CI matrix reads * the canary is a boundary check as of ledger v8, not functional coverage * the runner owns the stack on every exit path now, with MIDNIGHT_LIVE_KEEP_ENV to opt out * integration-live runs one worker Also adds an Integration Tests section, since the dry test:integration was undocumented. --- CONTRIBUTING.md | 38 ++++++++++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c93db8ec..66487fd8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -26,6 +26,7 @@ We really appreciate and value contributions to OpenZeppelin Contracts for Compa [Running Tests](#running-tests) * [Unit Tests](#unit-tests) +* [Integration Tests](#integration-tests) * [Live Tests](#live-tests) [Styleguides](#styleguides) @@ -169,6 +170,14 @@ Unit tests run against an in-process mock backend (no network, ZK proving skippe yarn test ``` +### Integration Tests + +Composed-contract specs (`contracts/test/integration/specs`): several modules assembled into one contract under `test/integration/_mocks`, then deployed and driven as a unit. Same mock backend as the unit tests. + +```bash +yarn test:integration +``` + ### Live Tests Live tests run against a local Midnight network (node, indexer, and proof server) defined in [`local-env.yml`](./local-env.yml). They require [Docker](https://docs.docker.com/get-docker/) and a completed `yarn install`. @@ -179,29 +188,37 @@ One command runs everything — it compiles, resets the stack, runs a quick harn yarn test:live ``` -Currently `multisig` is the only live-ready category; the others still assume dry-only semantics and are skipped (listed in the run banner). Each category joins the run — with its own `test:live:` script — as its specs are refactored for the live backend. +Currently `multisig` is the only live-ready category. The others still assume dry-only semantics and are skipped, and the run banner lists them. Each joins the run as its specs are refactored for the live backend, with its own `test:live:` script. + +`integration` is a target of its own, not a category, so an unscoped run skips it. Ask for it by name. Only one live target runs per invocation, since both live projects draw wallets from the same genesis-funded pool. + +```bash +yarn test:live integration # or: yarn test:integration:live +yarn test:live --list # the live targets, as the CI matrix reads them +``` + +> **Note:** the `integration` live target is the harness capability plus a boundary check, not functional coverage. As of ledger v8 the composed contract does not deploy: its circuits' IR overruns the per-tx block byte budget, and the spec asserts that rejection rather than skipping. A ledger bump can move the budget, so a red spec there means the deploy now fits and the functional specs are worth porting to live. If any files fail, a second round re-runs just those files on a fresh node with one worker, to separate a real failure from an environment flake: * Fails round 1, passes round 2 → **FLAKY** (exit 0, reported loudly). * Fails both rounds → **REAL** (exit non-zero). -Scope the same mechanism to one category, or to a subset within it. The first -argument names the category; any further argument is a filename substring -(vitest matches it), so pass a spec name to run every file whose name matches it -on the live backend — the fast loop while iterating on one feature, instead of -waiting for the whole category. The match is a substring, not an exact file, so -a name that prefixes others runs all of them: +Scope the same mechanism to one target, or a subset within it. The first argument +names the target (a category, or `integration`). Any further argument is a +filename substring vitest matches, which is the fast loop while iterating on one +feature. Being a substring, a name that prefixes others runs all of them: ```bash yarn test:live multisig # the whole category yarn test:live multisig ShieldedTreasury # any file matching "ShieldedTreasury" +yarn test:live integration ConfidentialFungibleToken # one integration spec ``` The two-round flake check still applies to a scoped run, so a green result means the same thing it does for the full suite. -Stop the network when done: `yarn env:down`. (No manual `env:up` is needed — the runner resets the stack itself.) +The runner owns the stack: it starts it (`make env-up`, itself a reset) and stops it on every exit path, Ctrl-C included. No manual `env:up` or `env:down` needed. To inspect a run afterwards, set `MIDNIGHT_LIVE_KEEP_ENV=1` and stop it yourself. Container logs land in `logs/` either way. > **Note:** The live tests all run against one shared node, so state left by an earlier run can make a later one fail. Two rules keep them reliable, both enforced by a guard that fails fast, before any wallet build: > @@ -212,12 +229,13 @@ Environment knobs: | Variable | Default | Effect | | --- | --- | --- | -| `MIDNIGHT_LIVE_WORKERS` | 3 | Parallel spec files (max 3 — one genesis-funded deployer each). | +| `MIDNIGHT_LIVE_WORKERS` | 3 | Parallel spec files under `unit-live` (max 3, one genesis-funded deployer each). | | `MIDNIGHT_LIVE_ALLOW_DIRTY` | unset | `1` skips the freshness check (run against a dirty node). | | `MIDNIGHT_LIVE_MAX_COIN_EVENTS` | 0 | Coin events beyond genesis tolerated before "not fresh". | | `MIDNIGHT_LIVE_MAX_SCAN_BLOCKS` | 3600 | Above this indexer head, the guard asks you to `env:up` rather than scan. | +| `MIDNIGHT_LIVE_KEEP_ENV` | unset | `1` leaves the stack running after the run instead of tearing it down. | -`unit-live` runs up to 3 workers in parallel, so their output interleaves. It is tagged per worker: a `▶ live worker N/3 ready` banner when a worker's wallets are funded, a `[wN] ❯ ` line as each spec file starts, and a `[wN] ✓ () [done/total]` line per test — showing the worker, the result, and overall progress through the run. Each worker also writes a detailed log to `logs/live-harness-wN.log`. +`integration-live` runs one worker (only the deployer wallet is in play). `unit-live` runs up to 3 workers in parallel, so their output interleaves. It is tagged per worker: a `▶ live worker N/3 ready` banner when a worker's wallets are funded, a `[wN] ❯ ` line as each spec file starts, and a `[wN] ✓ () [done/total]` line per test — showing the worker, the result, and overall progress through the run. Each worker also writes a detailed log to `logs/live-harness-wN.log`. > **Tip:** to save the run to a colored, readable log, force color and pipe to `tee`. Piping (stdout is no longer a TTY) makes vitest print one clean line per result instead of an animated spinner, and `FORCE_COLOR=1` keeps the color. Write it to a `.ansi` file: >