From ad3bc28cf8302f736aa549e525de8176522f748c Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Sun, 16 Aug 2026 07:37:11 +0000 Subject: [PATCH] node:test: report a verdict for suites registered with bun:test under run() Under run(), a describe()/suite() block registered with bun:test emitted no test:pass/test:fail of its own, so `suites` undercounted and a failure of the suite itself (a failing before()/after() hook or describe callback) was only visible as a synthesized file-level failure. In a run() child, the describe callback wrapper now brackets the block with a beforeAll/afterAll pair: the afterAll runs the suite's after() hooks and reports the suite once its children have reported. Top-level tests and nested suites roll their verdicts up into the enclosing suite the way node's Test.postRun() does; before() hook failures and a throwing or rejecting describe callback are recorded against the suite, and a todo suite's own failures are reported on it without failing the file, as in node. The inline suite path shares the verdict and event code. On the parent side, run() tracks success the way node's harness does, so a file whose only failure is a suite reports success:false with `failed 0`, and a nonzero child exit that a reported suite failure already explains no longer produces a file-level test:fail. --- src/js/node/test.ts | 313 +++++++++++++----- test/js/node/test_runner/node-test.test.ts | 364 ++++++++++++++++++++- 2 files changed, 593 insertions(+), 84 deletions(-) diff --git a/src/js/node/test.ts b/src/js/node/test.ts index 75abf5fb79e3..c527dc0d026c 100644 --- a/src/js/node/test.ts +++ b/src/js/node/test.ts @@ -348,8 +348,18 @@ function makeRunCounts() { } as unknown as Record; } -function addRunCounts(into: Record, from: Record) { - for (const key of Object.keys(from)) into[key] += from[key]; +// node's harness: the counters reported in test:summary plus `success`, which +// the counters alone cannot express because a failed suite is only counted in +// `suites` (countCompletedTest in node's test_runner/utils.js). +function makeRunTally() { + return { counts: makeRunCounts(), success: true }; +} + +type RunTally = ReturnType; + +function mergeRunTally(into: RunTally, from: RunTally) { + for (const key of Object.keys(from.counts)) into.counts[key] += from.counts[key]; + if (!from.success) into.success = false; } function emitRunDiagnostics(reporter: TestsStream, counts: Record, durationMs: number) { @@ -367,7 +377,7 @@ function emitRunDiagnostics(reporter: TestsStream, counts: Record, reporter: TestsStream) { const started = Date.now(); - const counts = makeRunCounts(); + const run = makeRunTally(); // run() returns the stream before any file starts, and callers attach their // listeners synchronously on the returned stream. Yield first so the earliest @@ -381,20 +391,21 @@ async function runFiles(opts: ReturnType, reporter: T let i = 0; for (; i < files.length; i++) { if (opts.signal?.aborted) break; - await runOneFile(files[i], opts, reporter, counts); + await runOneFile(files[i], opts, reporter, run); } // Node cancels each not-yet-started FileTest with cancelledByParent rather // than silently dropping it; an aborted run must not report success:true. for (; i < files.length; i++) { - reportCancelledFile(files[i], opts, reporter, counts); + reportCancelledFile(files[i], opts, reporter, run); } + const { counts } = run; reporter.plan({ __proto__: null, nesting: 0, count: counts.topLevel }); const durationMs = Date.now() - started; emitRunDiagnostics(reporter, counts, durationMs); reporter.summary({ __proto__: null, - success: counts.failed === 0 && counts.cancelled === 0, + success: run.success, counts, duration_ms: durationMs, file: undefined, @@ -410,7 +421,7 @@ function reportCancelledFile( file: string, opts: ReturnType, reporter: TestsStream, - counts: Record, + run: RunTally, ) { const path = require("node:path"); const absolute = path.resolve(opts.cwd as string, file); @@ -437,16 +448,18 @@ function reportCancelledFile( details: { ...details, passed: false }, }); reporter.fail({ __proto__: null, ...fileNode, type: undefined, testNumber: 1, details }); + const { counts } = run; counts.tests++; counts.cancelled++; counts.topLevel++; + run.success = false; } async function runOneFile( file: string, opts: ReturnType, reporter: TestsStream, - counts: Record, + run: RunTally, ) { const path = require("node:path"); const absolute = path.resolve(opts.cwd as string, file); @@ -455,7 +468,8 @@ async function runOneFile( // before the `test` keyword and user args after the path. const args = [process.execPath, ...(opts.execArgv as string[]), "test", absolute, ...(opts.argv as string[])]; const fileStarted = Date.now(); - const fileCounts = makeRunCounts(); + const tally = makeRunTally(); + const fileCounts = tally.counts; // Under process isolation node models the file itself as a top-level test, // named by the path as it was passed in and located at 1:1. @@ -527,7 +541,7 @@ async function runOneFile( return; } if (event == null || typeof event.data !== "object" || event.data === null) return; - republishChildEvent(event, absolute, reporter, fileCounts); + republishChildEvent(event, absolute, reporter, tally); }; const decoder = new TextDecoder(); @@ -546,10 +560,10 @@ async function runOneFile( await drainStderr; const exitCode = await proc.exited; - // A nonzero exit with no child-reported failures means the file itself died - // (top-level throw); child-reported failures are already covered by the - // republished events and need no file-level verdict. - const fileFailed = exitCode !== 0 && fileCounts.failed === 0; + // A nonzero exit that no republished failure (test or suite) accounts for + // means the file itself died (top-level throw); child-reported failures are + // already covered by the republished events and need no file-level verdict. + const fileFailed = exitCode !== 0 && tally.success; const fileDuration = Date.now() - fileStarted; // Node's FileTest.#skipReporting(): no file-level complete/pass/fail when // the child reported at least one test and the only error is subtestsFailed @@ -563,8 +577,12 @@ async function runOneFile( const reportFileNode = reportedChildren === 0 || fileFailed; if (reportFileNode) { fileCounts.tests++; - if (fileFailed) fileCounts.failed++; - else fileCounts.passed++; + if (fileFailed) { + fileCounts.failed++; + tally.success = false; + } else { + fileCounts.passed++; + } } if (fileFailed) { @@ -572,7 +590,7 @@ async function runOneFile( } else { reporter.summary({ __proto__: null, - success: fileCounts.failed === 0, + success: tally.success, counts: { __proto__: null, ...fileCounts }, duration_ms: fileDuration, file: absolute, @@ -600,7 +618,7 @@ async function runOneFile( reporter.pass({ __proto__: null, ...fileNode, type: undefined, testNumber: 1, details }); } } - addRunCounts(counts, fileCounts); + mergeRunTally(run, tally); } finally { proc.kill(); if (drainStderr !== undefined) await drainStderr.catch(() => {}); @@ -618,27 +636,29 @@ function rebuildError(serialized: any, depth = 0): Error { return error; } -function republishChildEvent( - event: { type: string; data: any }, - file: string, - reporter: TestsStream, - counts: Record, -) { +function republishChildEvent(event: { type: string; data: any }, file: string, reporter: TestsStream, tally: RunTally) { const { type, data } = event; Object.setPrototypeOf(data, null); data.file = file; data.nesting = (data.nesting ?? 0) + 1; if (type === "test:pass" || type === "test:fail") { + const { counts } = tally; const isSuite = data.type === "suite"; // node counts a suite in `suites` and stops there: a skipped or todo suite - // never lands in skipped/todo/passed/tests (countCompletedTest, test.js). - if (isSuite) counts.suites++; - else { + // never lands in skipped/todo/passed/tests (countCompletedTest, utils.js), + // but a failed suite still fails the run unless it is todo. + if (isSuite) { + counts.suites++; + if (type === "test:fail" && !data.todo) tally.success = false; + } else { counts.tests++; if (data.skip) counts.skipped++; else if (data.todo) counts.todo++; else if (type === "test:pass") counts.passed++; - else counts.failed++; + else { + counts.failed++; + tally.success = false; + } } // node carries the node kind on `details`, not on the event itself. const detailType = isSuite ? "suite" : "test"; @@ -697,27 +717,43 @@ function serializeRunError(error: unknown, depth = 0) { return { __proto__: null, message: String(error), stack: undefined, code: undefined, name: "Error" }; } -// A test or suite that bun:test will never invoke (the `skip` and `todo` -// options). Node still reports it as a pass carrying the directive. -function reportDirectiveOnlyNode(node: TestNode, mode: "skip" | "todo") { +// A skipped test or suite, which bun:test never invokes. Node still reports it +// as a pass carrying the directive; `{ skip: true, todo: true }` reports as a +// skip too, since node checks `skipped` before `isTodo` (test.js +// getReportDetails). +function reportSkippedNode(node: TestNode) { if (!runChildReporterEnabled) return; - // `{ skip: true, todo: true }` reports as a skip: node checks `skipped` - // first and only then `isTodo` (test.js getReportDetails). - const skipped = node.skipped || mode === "skip"; emitRunChildEvent("test:pass", { __proto__: null, name: node.name, nesting: nestingOf(node), testNumber: 0, duration_ms: 0, - skip: skipped ? (node.message ?? true) : undefined, - todo: !skipped ? (node.message ?? true) : undefined, + skip: node.message ?? true, + todo: undefined, type: node.isSuite ? "suite" : "test", tags: node.tags, error: undefined, }); } +// Called once a suite's verdict is final (concludeSuite), whether it was +// registered with bun:test or ran inline. No-op outside a run() child. +function reportSuiteToRunParent(suite: TestNode, startedAt: number) { + if (!runChildReporterEnabled) return; + emitRunChildEvent(suite.passed ? "test:pass" : "test:fail", { + __proto__: null, + name: suite.name, + nesting: nestingOf(suite), + testNumber: 0, + duration_ms: performance.now() - startedAt, + type: "suite", + tags: suite.tags, + todo: suite.todoFlag ? (suite.message ?? true) : undefined, + error: suite.passed ? undefined : serializeRunError(suite.error), + }); +} + // Called for every test node as its result is finalized, so subtests report // with the same shape as top-level tests. No-op outside a run() child. function reportNodeToRunParent(node: TestNode, startedAt: number) { @@ -1386,6 +1422,16 @@ function makeTestFailure(message: string, failureType?: string) { return error; } +// The failure a test or suite gets when children of it failed (node's +// Test.postRun()). The first child failure rides along as the cause so that +// bun:test's own report shows what actually failed. +function makeSubtestsFailedError(node: TestNode) { + const { failedSubtests, firstSubtestError } = node; + const error = makeTestFailure(`${failedSubtests} subtest${failedSubtests > 1 ? "s" : ""} failed`, "subtestsFailed"); + if (firstSubtestError !== undefined) (error as { cause?: unknown }).cause = firstSubtestError; + return error; +} + class TestPlan { expected: number; actual = 0; @@ -2329,16 +2375,8 @@ async function executeTestNode(node: TestNode, fn: TestFn): Promise { node.plan?.cancel(); } - const { failedSubtests, firstSubtestError } = node; - if (failure === undefined && failedSubtests > 0) { - const error = makeTestFailure( - `${failedSubtests} subtest${failedSubtests > 1 ? "s" : ""} failed`, - "subtestsFailed", - ); - if (firstSubtestError !== undefined) { - (error as { cause?: unknown }).cause = firstSubtestError; - } - failure = error; + if (failure === undefined && node.failedSubtests > 0) { + failure = makeSubtestsFailedError(node); } } @@ -2412,11 +2450,26 @@ function scheduleSubtest(parent: TestNode, child: TestNode, fn: TestFn, ownTodo: return result.then(() => undefined); } +// Inline suites fold a failure of their own (build or hook) into the failed +// subtest count; suites registered with bun:test record it in `error` instead +// (failSuiteBuild, failSuiteFromHook). concludeSuite() accepts either. function recordSuiteFailure(suite: TestNode, err: unknown) { suite.failedSubtests++; suite.firstSubtestError ??= err ?? makeTestFailure("suite failed"); } +// node's Suite.run() + Test.postRun(): a failure recorded against the suite +// itself stands; otherwise failed children fail it with subtestsFailed. The +// verdict then goes to the run() parent, if any. +function concludeSuite(suite: TestNode, startedAt: number) { + if (suite.error === null && suite.failedSubtests > 0) { + suite.error = makeSubtestsFailedError(suite); + } + suite.passed = suite.error === null; + suite.finished = true; + reportSuiteToRunParent(suite, startedAt); +} + // Awaits a node's subtest chain, including links appended while waiting. async function drainSubtestChain(node: TestNode) { let chain; @@ -2444,6 +2497,7 @@ function scheduleSuiteSubtest(parent: TestNode, suite: TestNode, build: unknown, recordSuiteFailure(suite, err); } } + const startedAt = runChildReporterEnabled ? performance.now() : 0; try { await runOwnBeforeHooks(suite); } catch (err) { @@ -2459,30 +2513,9 @@ function scheduleSuiteSubtest(parent: TestNode, suite: TestNode, build: unknown, recordSuiteFailure(suite, err); } } - suite.finished = true; - suite.passed = suite.failedSubtests === 0; - if (runChildReporterEnabled) { - emitRunChildEvent(suite.passed ? "test:pass" : "test:fail", { - __proto__: null, - name: suite.name, - nesting: nestingOf(suite), - testNumber: 0, - duration_ms: 0, - type: "suite", - tags: suite.tags, - todo: suite.todoFlag ? (suite.message ?? true) : undefined, - error: suite.passed - ? undefined - : serializeRunError( - makeTestFailure( - `${suite.failedSubtests} subtest${suite.failedSubtests > 1 ? "s" : ""} failed`, - "subtestsFailed", - ), - ), - }); - } + concludeSuite(suite, startedAt); // A todo suite's failures do not fail the owning test (Node). - if (suite.failedSubtests > 0 && !ownTodo) { + if (!suite.passed && !ownTodo) { parent.failedSubtests++; parent.firstSubtestError ??= suite.firstSubtestError; } @@ -2525,6 +2558,103 @@ function currentCollectionParent(): TestNode { return getRootNode(); } +// node's Test.postRun(): a child that finished without passing is a failed +// subtest of its parent unless it is todo. Counterpart of the roll-up in +// scheduleSubtest()/scheduleSuiteSubtest() for tests and suites registered with +// bun:test, whose parent is the enclosing suite (or the root, which reports +// nothing). Only a run() child reports suite verdicts, so only it keeps count. +function rollUpIntoEnclosingSuite(node: TestNode) { + if (!runChildReporterEnabled || node.passed || node.todoFlag) return; + const parent = node.parent!; + parent.failedSubtests++; + parent.firstSubtestError ??= node.error; +} + +// run() child: a suite registered with bun:test reports a verdict of its own, +// like node's Suite. bun:test runs a scope's beforeAll hooks before and its +// afterAll hooks after everything declared inside the scope, nested describes +// included, so one hook of each kind brackets the suite: the first starts its +// clock and the second runs the suite's after() hooks and reports. +function buildReportedSuite(suite: TestNode, fn: TestFn) { + const { beforeAll, afterAll } = bunTest(); + let startedAt = 0; + // Both hooks take `done`, like every callback this module hands to bun:test: + // bun:test decides whether to wait for done() from the callback's arity, which + // it misreads as nonzero once the callback is registered under an async + // context, and a nested describe's callback runs under its parent's + // runWithNode(). A zero-arity hook there would wait 5s for a done() nobody calls. + beforeAll((done: (error?: unknown) => void) => { + startedAt = performance.now(); + done(); + }); + afterAll((done: (error?: unknown) => void) => { + settleReportedSuite(suite, startedAt).then(done, done); + }); + + let build: unknown; + try { + build = runWithNode(suite, () => fn(suite.getSuiteCtx())); + } catch (err) { + return failSuiteBuild(suite, err); + } + if (build != null && typeof (build as PromiseLike).then === "function") { + return (build as Promise).then(undefined, err => failSuiteBuild(suite, err)); + } + return build; +} + +// A describe() callback that throws or rejects fails the suite with that error +// (node: Suite.createBuild). The error is rethrown so bun:test reports it as +// before; bun:test then drops the scope, the two hooks above included, so the +// verdict has to be reported right here, at collection time. In a todo suite +// the failure is advisory (node), so it is only recorded: the scope stays, its +// children still run and the closing hook reports the verdict after them. +function failSuiteBuild(suite: TestNode, err: unknown): undefined { + suite.error = err ?? makeTestFailure("suite failed"); + if (suite.todoFlag) return; + finishReportedSuite(suite, performance.now()); + throw err; +} + +async function settleReportedSuite(suite: TestNode, startedAt: number) { + let hookError: unknown; + for (const hook of suite.hooks.after) { + try { + await runHook(hook, suite, suite.getSuiteCtx()); + } catch (err) { + // node's Test.runHook() stops at the first failing hook. + failSuiteFromHook(suite, "after", err); + hookError = err; + break; + } + } + finishReportedSuite(suite, startedAt); + // The failure still goes to bun:test, which reports it and fails the file as + // it did when after() registered the hook directly, unless the suite is todo: + // node treats a todo suite's hook failure as advisory. + return suite.todoFlag ? undefined : hookError; +} + +function finishReportedSuite(suite: TestNode, startedAt: number) { + concludeSuite(suite, startedAt); + rollUpIntoEnclosingSuite(suite); +} + +// A failing suite-level hook fails the suite (node's Test.runHook()); the first +// failure a suite records is the one it reports. +function failSuiteFromHook(suite: TestNode, kind: "before" | "after", err: unknown) { + const failure = makeTestFailure(`failed running ${kind} hook`, "hookFailed"); + (failure as { cause?: unknown }).cause = err; + suite.error ??= failure; +} + +// Whether before()/after() hooks declared on `owner` feed a verdict reported by +// buildReportedSuite: a suite registered with bun:test, in a run() child. The +// root reports no verdict and inline suites run their hooks in scheduleSuiteSubtest. +function isReportedSuite(owner: TestNode) { + return runChildReporterEnabled && owner.isSuite && owner.parent !== undefined && !owner.isExecutionPhase; +} + function createTopLevelTestRunner(node: TestNode, fn: TestFn, declaredTodo = false) { // bun:test invokes this with a `done` callback because the function declares // one parameter. @@ -2536,6 +2666,7 @@ function createTopLevelTestRunner(node: TestNode, fn: TestFn, declaredTodo = fal const todoBefore = node.todoFlag; executeTestNode(node, fn).then( failure => { + rollUpIntoEnclosingSuite(node); // A runtime t.skip()/t.todo() overrides bun:test's pass/fail accounting // (Node counts these as skip/todo even when the body threw); a declared // todo body's failure must reach bun:test's own todo accounting instead. @@ -2577,9 +2708,7 @@ function addTest( child.ownTags = ownTags; if (mode === "skip" || options.skip) { // Chain onto subtestChain so the directive lands after earlier siblings. - const chained = (runningNode.subtestChain = runningNode.subtestChain.then(() => - reportDirectiveOnlyNode(child, "skip"), - )); + const chained = (runningNode.subtestChain = runningNode.subtestChain.then(() => reportSkippedNode(child))); return chained.then(() => undefined); } const ownTodo = mode === "todo" || !!options.todo; @@ -2605,7 +2734,7 @@ function addTest( // event fires in execution order (not at collection time). if (runChildReporterEnabled && effectiveMode === "skip") { const runner = function (done: (err?: unknown) => void) { - reportDirectiveOnlyNode(node, "skip"); + reportSkippedNode(node); markCurrentResult(false, done); done(undefined); }; @@ -2673,9 +2802,7 @@ function addSuite( suite.ownTags = ownTags; if (mode === "skip" || options.skip) { // Chain onto subtestChain so the directive lands after earlier siblings. - const chained = (runningNode.subtestChain = runningNode.subtestChain.then(() => - reportDirectiveOnlyNode(suite, "skip"), - )); + const chained = (runningNode.subtestChain = runningNode.subtestChain.then(() => reportSkippedNode(suite))); return chained.then(() => undefined); } const ownTodo = mode === "todo" || !!options.todo; @@ -2723,19 +2850,23 @@ function addSuite( const wrapped = effectiveMode === "skip" ? kDefaultFunction - : () => { - return runWithNode(suiteNode, () => fn(suiteNode.getSuiteCtx())); - }; + : runChildReporterEnabled + ? () => buildReportedSuite(suiteNode, fn) + : () => runWithNode(suiteNode, () => fn(suiteNode.getSuiteCtx())); const passOptions = bunTestOptions(options); let register: Function = describe; - if (effectiveMode === "skip") register = describe.skip; - else if (effectiveMode === "todo") { + if (effectiveMode === "skip") { + register = describe.skip; + reportSkippedNode(suiteNode); + } else if (effectiveMode === "todo") { suiteNode.todoFlag = true; + // A run() child registers a plain describe: bun:test's todo scope would only + // run the children under --todo, and the suite's own verdict (todo, failed + // if a hook or the callback fails, like node's) comes from buildReportedSuite. register = runChildReporterEnabled ? describe : describe.todo; } - if (effectiveMode !== undefined) reportDirectiveOnlyNode(suiteNode, effectiveMode); if (passOptions !== undefined) { register(name, wrapped, passOptions); @@ -2807,7 +2938,17 @@ function before(arg0: unknown, arg1: unknown) { beforeAll((done: (error?: unknown) => void) => { Promise.resolve(runHook(hook, owner, hookArgFor(owner))).then( () => done(), - err => done(err ?? new Error("before hook failed")), + err => { + if (isReportedSuite(owner)) { + failSuiteFromHook(owner, "before", err); + // Advisory in a todo suite (node), so bun:test is not told about it. + if (owner.todoFlag) { + done(); + return; + } + } + done(err ?? new Error("before hook failed")); + }, ); }); } @@ -2819,6 +2960,12 @@ function after(arg0: unknown, arg1: unknown) { owner.hooks.after.push(hook); return; } + if (isReportedSuite(owner)) { + // Run by the hook that closes the suite (settleReportedSuite), so that the + // suite's verdict comes after its after() hooks and reflects their failures. + owner.hooks.after.push(hook); + return; + } const { afterAll } = bunTest(); afterAll((done: (error?: unknown) => void) => { Promise.resolve(runHook(hook, owner, hookArgFor(owner))).then( diff --git a/test/js/node/test_runner/node-test.test.ts b/test/js/node/test_runner/node-test.test.ts index 2a27963203c0..b5653781ad5c 100644 --- a/test/js/node/test_runner/node-test.test.ts +++ b/test/js/node/test_runner/node-test.test.ts @@ -1,6 +1,6 @@ import { spawn } from "bun"; import { describe, expect, test } from "bun:test"; -import { bunEnv, bunExe } from "harness"; +import { bunEnv, bunExe, tempDir } from "harness"; import { join } from "node:path"; describe("node:test", () => { @@ -344,6 +344,368 @@ async function runTests(filenames: string[], env: Record = {}, a return { exitCode, stdout, stderr }; } +// Drives node:test's run() over the given files and digests its stream: every +// test:pass/test:fail verdict in order (name, details.type, directive, error), +// the `suites N` diagnostic, the per-file and run-level test:summary (the fields +// suite reporting affects) and the children's stdout. +const kRunDriver = ` + import { basename } from "node:path"; + import { run } from "node:test"; + + const out = { verdicts: [], suitesDiagnostic: undefined, summaries: {}, stdout: "" }; + for await (const { type, data } of run({ files: process.argv.slice(2) })) { + if (type === "test:pass" || type === "test:fail") { + const verdict = { type, name: data.name, kind: data.details.type }; + if (data.skip !== undefined) verdict.skip = data.skip; + if (data.todo !== undefined) verdict.todo = data.todo; + const { error } = data.details; + if (error !== undefined) { + verdict.error = error.message; + if (error.failureType !== undefined) verdict.failureType = error.failureType; + if (error.cause !== undefined) verdict.cause = error.cause.message; + } + out.verdicts.push(verdict); + } else if (type === "test:diagnostic" && data.message.startsWith("suites ")) { + out.suitesDiagnostic = data.message; + } else if (type === "test:summary") { + const { tests, suites, passed, failed, skipped, todo } = data.counts; + out.summaries[data.file === undefined ? "" : basename(data.file)] = { + success: data.success, + tests, + suites, + passed, + failed, + skipped, + todo, + }; + } else if (type === "test:stdout") { + out.stdout += data.message; + } + } + console.log(JSON.stringify(out)); +`; + +type RunVerdict = { + type: "test:pass" | "test:fail"; + name: string; + kind: "test" | "suite"; + skip?: boolean | string; + todo?: boolean | string; + error?: string; + failureType?: string; + cause?: string; +}; + +type RunDigest = { + verdicts: RunVerdict[]; + suitesDiagnostic: string | undefined; + summaries: Record>; + stdout: string; +}; + +async function digestRun(files: Record): Promise { + using dir = tempDir("node-test-run", { ...files, "driver.mjs": kRunDriver }); + await using proc = spawn({ + cmd: [bunExe(), "driver.mjs", ...Object.keys(files)], + cwd: String(dir), + env: bunEnv, + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + expect(stderr).toBe(""); + expect(exitCode).toBe(0); + return JSON.parse(stdout); +} + +describe("node:test run()", () => { + // Unless a comment says otherwise, every expectation below is what node + // v26.3.0 reports for the same file. Two shape differences hold throughout: + // bun's child does not label test code failures with failureType + // "testCodeFailure", and it attaches the first failure as the cause of a + // "N subtests failed" error where node repeats the message. + // + // Each case spawns a driver process that in turn spawns one debug+ASAN + // `bun test` child (about 4.5s locally), hence the generous timeouts. + test.concurrent( + "reports a verdict for every suite registered with bun:test, after its children, and counts it in suites", + async () => { + const { verdicts, suitesDiagnostic, summaries } = await digestRun({ + "suites.js": ` + const { describe, it, test, suite } = require("node:test"); + + describe.skip("skipped suite", () => { + it("never declared", () => {}); + }); + + describe("outer", () => { + it("ok", () => {}); + describe("inner", () => { + it("bad", () => { + throw new Error("bad child"); + }); + }); + it("ok2", () => {}); + }); + + suite("empty suite", () => {}); + + describe.todo("todo suite", () => { + it("fails inside a todo suite", () => { + throw new Error("x"); + }); + }); + + describe("suite with directives", () => { + it.todo("todo child", () => { + throw new Error("y"); + }); + it("skipped child", { skip: "why" }, () => {}); + }); + + test("parent test", () => { + describe("inline suite", () => { + it("inline child", () => {}); + }); + }); + `, + }); + + expect(verdicts).toEqual([ + { type: "test:pass", name: "skipped suite", kind: "suite", skip: true }, + { type: "test:pass", name: "ok", kind: "test" }, + { type: "test:fail", name: "bad", kind: "test", error: "bad child" }, + // A failing child fails its suite, and that failure the enclosing suite. + { + type: "test:fail", + name: "inner", + kind: "suite", + error: "1 subtest failed", + failureType: "subtestsFailed", + cause: "bad child", + }, + { type: "test:pass", name: "ok2", kind: "test" }, + { + type: "test:fail", + name: "outer", + kind: "suite", + error: "1 subtest failed", + failureType: "subtestsFailed", + cause: "1 subtest failed", + }, + { type: "test:pass", name: "empty suite", kind: "suite" }, + // Children of a todo suite are todo themselves, so their failures do + // not fail it; the suite reports after them, carrying the directive. + { type: "test:fail", name: "fails inside a todo suite", kind: "test", todo: true, error: "x" }, + { type: "test:pass", name: "todo suite", kind: "suite", todo: true }, + { type: "test:fail", name: "todo child", kind: "test", todo: true, error: "y" }, + { type: "test:pass", name: "skipped child", kind: "test", skip: "why" }, + { type: "test:pass", name: "suite with directives", kind: "suite" }, + { type: "test:pass", name: "inline child", kind: "test" }, + { type: "test:pass", name: "inline suite", kind: "suite" }, + { type: "test:pass", name: "parent test", kind: "test" }, + ]); + expect(suitesDiagnostic).toBe("suites 7"); + const counts = { success: false, tests: 8, suites: 7, passed: 4, failed: 1, skipped: 1, todo: 2 }; + expect(summaries).toEqual({ "suites.js": counts, "": counts }); + }, + 60_000, + ); + + test.concurrent( + "a failing after() hook fails its suite and the run, and is not mistaken for a file-level failure", + async () => { + const { verdicts, summaries, stdout } = await digestRun({ + "after-hook.js": ` + const { describe, it, after } = require("node:test"); + + describe("suite", () => { + after(() => { + throw new Error("after boom"); + }); + after(() => { + console.log("SECOND_AFTER_HOOK_RAN"); + }); + it("ok", () => {}); + }); + `, + }); + + // The child exits nonzero over the hook, but the suite's own verdict + // accounts for that: no synthesized verdict for after-hook.js itself. + expect(verdicts).toEqual([ + { type: "test:pass", name: "ok", kind: "test" }, + { + type: "test:fail", + name: "suite", + kind: "suite", + error: "failed running after hook", + failureType: "hookFailed", + cause: "after boom", + }, + ]); + // `failed` counts tests only; the failed suite still makes the run fail. + const counts = { success: false, tests: 1, suites: 1, passed: 1, failed: 0, skipped: 0, todo: 0 }; + expect(summaries).toEqual({ "after-hook.js": counts, "": counts }); + // node stops at the first failing hook. + expect(stdout).not.toContain("SECOND_AFTER_HOOK_RAN"); + }, + 60_000, + ); + + test.concurrent( + "a todo suite's own failures are reported on the suite but do not fail the run", + async () => { + const { verdicts, summaries } = await digestRun({ + "todo-suites.js": ` + const { describe, it, before, after } = require("node:test"); + + describe.todo("before fails", () => { + before(() => { + throw new Error("before boom"); + }); + it("child of before-fails", () => {}); + }); + + describe("after fails", { todo: "later" }, () => { + after(() => { + throw new Error("after boom"); + }); + it("child of after-fails", () => {}); + }); + + describe.todo("callback throws", () => { + it("child of callback-throws", () => {}); + throw new Error("todo build boom"); + }); + `, + }); + + expect(verdicts).toEqual([ + // node cancels the children of the first and third suite instead + // (test:fail, still todo); bun:test has no way to cancel a test yet, so + // they run. Either way they are counted as todo. + { type: "test:pass", name: "child of before-fails", kind: "test", todo: true }, + { + type: "test:fail", + name: "before fails", + kind: "suite", + todo: true, + error: "failed running before hook", + failureType: "hookFailed", + cause: "before boom", + }, + { type: "test:pass", name: "child of after-fails", kind: "test", todo: true }, + { + type: "test:fail", + name: "after fails", + kind: "suite", + todo: "later", + error: "failed running after hook", + failureType: "hookFailed", + cause: "after boom", + }, + { type: "test:pass", name: "child of callback-throws", kind: "test", todo: true }, + { type: "test:fail", name: "callback throws", kind: "suite", todo: true, error: "todo build boom" }, + ]); + // Nothing here makes the child process exit nonzero, so the file gets no + // synthesized verdict and the run succeeds, as in node. + const counts = { success: true, tests: 3, suites: 3, passed: 0, failed: 0, skipped: 0, todo: 3 }; + expect(summaries).toEqual({ "todo-suites.js": counts, "": counts }); + }, + 60_000, + ); + + test.concurrent( + "a failing before() hook fails its suite", + async () => { + const { verdicts, summaries, stdout } = await digestRun({ + "before-hook.js": ` + const { describe, it, before, after } = require("node:test"); + + describe("suite", () => { + before(() => { + throw new Error("before boom"); + }); + after(() => { + console.log("AFTER_HOOK_RAN"); + }); + it("child", () => {}); + }); + `, + }); + + // What becomes of the child is bun:test's call (node cancels it), so only + // the suite's own verdict is pinned here. + expect(verdicts.filter(verdict => verdict.kind === "suite")).toEqual([ + { + type: "test:fail", + name: "suite", + kind: "suite", + error: "failed running before hook", + failureType: "hookFailed", + cause: "before boom", + }, + ]); + expect(verdicts.map(verdict => verdict.name)).not.toContain("before-hook.js"); + expect(summaries["before-hook.js"]).toMatchObject({ success: false, suites: 1, failed: 0 }); + expect(summaries[""]).toMatchObject({ success: false, suites: 1, failed: 0 }); + // node runs the after hooks of a suite whose before hook failed. + expect(stdout).toContain("AFTER_HOOK_RAN"); + }, + 60_000, + ); + + test.concurrent( + "a describe() callback that throws or rejects fails its suite, which fails the enclosing suite", + async () => { + const { verdicts, suitesDiagnostic, summaries } = await digestRun({ + "build-failures.js": ` + const { describe, it, test } = require("node:test"); + + describe("outer", () => { + it("ok", () => {}); + describe("inner", () => { + throw new Error("build boom"); + }); + }); + + describe("async", async () => { + await Promise.resolve(); + throw new Error("async build boom"); + }); + + test("sibling", () => {}); + `, + }); + + // bun:test drops a scope whose callback threw, so such a suite is reported + // as soon as it is declared rather than in node's execution order; compare + // by name. + const expected: RunVerdict[] = [ + { type: "test:fail", name: "async", kind: "suite", error: "async build boom" }, + { type: "test:fail", name: "inner", kind: "suite", error: "build boom" }, + { type: "test:pass", name: "ok", kind: "test" }, + { + type: "test:fail", + name: "outer", + kind: "suite", + error: "1 subtest failed", + failureType: "subtestsFailed", + cause: "build boom", + }, + { type: "test:pass", name: "sibling", kind: "test" }, + ]; + const byName = (a: RunVerdict, b: RunVerdict) => a.name.localeCompare(b.name); + expect(verdicts.toSorted(byName)).toEqual(expected.toSorted(byName)); + expect(suitesDiagnostic).toBe("suites 3"); + const counts = { success: false, tests: 2, suites: 3, passed: 2, failed: 0, skipped: 0, todo: 0 }; + expect(summaries).toEqual({ "build-failures.js": counts, "": counts }); + }, + 60_000, + ); +}); + describe("node:test mock", () => { const { mock } = require("node:test");