diff --git a/src/js/node/test.ts b/src/js/node/test.ts index 75abf5fb79e3..d833c28b22de 100644 --- a/src/js/node/test.ts +++ b/src/js/node/test.ts @@ -425,7 +425,7 @@ function reportCancelledFile( column: 1, file: absolute, }; - const error = makeTestFailure("test did not finish before its parent and was cancelled", "cancelledByParent"); + const error = makeCancelledByParentError(); const details = { __proto__: null, duration_ms: 0, type: "test", error }; reporter.enqueue({ __proto__: null, ...fileNode }); reporter.dequeue({ __proto__: null, ...fileNode }); @@ -1386,6 +1386,15 @@ function makeTestFailure(message: string, failureType?: string) { return error; } +function makeCancelledByParentError() { + return makeTestFailure("test did not finish before its parent and was cancelled", "cancelledByParent"); +} + +// Node's abortHandler (test.js:1059) substitutes its own error for a falsy reason. +function abortFailure(signal: AbortSignal): unknown { + return signal.reason || $makeAbortError("The test was aborted"); +} + class TestPlan { expected: number; actual = 0; @@ -1544,13 +1553,26 @@ class TestNode { // Inline subtests are serialized through this chain. `concurrency` is // validated for Node-compat error codes but subtests always run serially. subtestChain: Promise = Promise.resolve(); + // Node's unfinishedSubtests: every child, top-level and inline, until it finishes. + unfinishedSubtests = new Set(); failedSubtests = 0; firstSubtestError: unknown = undefined; // First failure from a before hook created while this test was running. hookFailure: unknown = undefined; + // The verdict fail()/cancel() set from outside the node's own run. + cancelError: unknown = undefined; + // An enclosing suite has a timeout or signal option and may cancel this node. + inStoppableSuite: boolean; + // Suites only: passed the abort check (Node's startTime), and the armed stop. + suiteStarted = false; + suiteStop: StopController | undefined = undefined; #ctx: TestContext | undefined; #suiteCtx: SuiteContext | undefined; #tags: string[] | undefined; + // t.signal, created on first use; #aborted records an abort() made before that. + #abortController: AbortController | undefined; + #aborted = false; + #cancellation: PromiseWithResolvers | undefined; constructor( name: string, @@ -1574,6 +1596,9 @@ class TestNode { if (typeof skip === "string") this.message = skip; else if (typeof todo === "string") this.message = todo; this.expectFailure = parseExpectFailure(options.expectFailure) || parent?.expectFailure || false; + this.inStoppableSuite = + parent !== undefined && (parent.inStoppableSuite || (parent.isSuite && hasStopOptions(parent.options))); + parent?.unfinishedSubtests.add(this); } get tags(): string[] { @@ -1614,6 +1639,58 @@ class TestNode { return this.#suiteCtx; } + get signal(): AbortSignal { + if (this.#abortController === undefined) { + this.#abortController = new AbortController(); + if (this.#aborted) this.#abortController.abort(); + } + return this.#abortController.signal; + } + + abort() { + this.#aborted = true; + this.#abortController?.abort(); + } + + // Node's Test#fail: the first verdict wins, and a finished node keeps its own. + fail(error: unknown): boolean { + if (this.finished || this.cancelError !== undefined) return false; + this.cancelError = error; + this.#cancellation?.reject(error); + return true; + } + + // Node's Test#cancel (test.js:1064): fail() plus aborting t.signal. + cancel(error: unknown) { + if (this.fail(error ?? makeCancelledByParentError())) this.abort(); + } + + // Raced by executeTestNode; rejects once fail() reaches the node. + cancellation(): Promise { + let pending = this.#cancellation; + if (pending === undefined) { + pending = this.#cancellation = Promise.withResolvers(); + // Swallow the rejection when nothing is racing it anymore. + pending.promise.catch(() => {}); + const { cancelError } = this; + if (cancelError !== undefined) pending.reject(cancelError); + } + return pending.promise; + } + + // Node's postRun() sweep (test.js:1476). + cancelSubtests() { + for (const subtest of this.unfinishedSubtests) { + subtest.cancel(undefined); + subtest.cancelSubtests(); + } + } + + finish() { + this.finished = true; + this.parent?.unfinishedSubtests.delete(this); + } + // True while user code reached from this node should treat new tests as // inline subtests instead of bun:test registrations. isRunning(): boolean { @@ -1666,7 +1743,6 @@ function getRootNode(): TestNode { */ class TestContext { #node: TestNode; - #abortController?: AbortController; #assert: Record | undefined; constructor(node: TestNode) { @@ -1674,10 +1750,7 @@ class TestContext { } get signal(): AbortSignal { - if (this.#abortController === undefined) { - this.#abortController = new AbortController(); - } - return this.#abortController.signal; + return this.#node.signal; } get name(): string { @@ -1828,17 +1901,13 @@ class TestContext { */ class SuiteContext { #node: TestNode; - #abortController?: AbortController; constructor(node: TestNode) { this.#node = node; } get signal(): AbortSignal { - if (this.#abortController === undefined) { - this.#abortController = new AbortController(); - } - return this.#abortController.signal; + return this.#node.signal; } get name(): string { @@ -1948,6 +2017,11 @@ function validateTimeoutAndSignal(options: TestOptions | HookOptions) { } } +// Suites have no default or inherited timeout (test.js:1763). +function hasStopOptions({ timeout, signal }: TestOptions): boolean { + return (typeof timeout === "number" && Number.isFinite(timeout)) || signal !== undefined; +} + // Port of Node's parseExpectFailure (test.js:528). A string is a label, a // function or RegExp validates the error, an object may carry both, and any // other object is itself the validation. @@ -2018,8 +2092,8 @@ function applyExpectFailure(node: TestNode, failure: unknown): unknown { function validateTestOptions(options: TestOptions): { ownTags: string[] | undefined } { const { concurrency, tags, plan } = options; - // signal and concurrency are validated for Node's error contract but not yet - // enforced (t.signal never aborts; subtests always run serially). + // A test's signal, and concurrency, are validated for Node's error contract + // but not yet enforced (subtests always run serially). validateTimeoutAndSignal(options); if (concurrency != null && typeof concurrency !== "boolean") { if (typeof concurrency === "number") { @@ -2133,22 +2207,39 @@ function invokeTestFn(fn: Function, arg: unknown) { return fn(arg); } -// A single timeout armed once per test and raced against both the body and -// plan.check(), matching Node's stopTest()/stopPromise. `promise` never -// resolves; it only rejects with the timeout error. Callers must dispose(). -function createStopController(timeout: number | undefined) { - if (typeof timeout !== "number" || !Number.isFinite(timeout)) { +let addAbortListener; + +type StopController = { promise: Promise; dispose(): void }; + +// Node's stopTest() (test.js:145): `promise` never resolves, it rejects on the +// timeout or (suites) the signal option. Callers must dispose(). +function createStopController(timeout: number | undefined, signal?: AbortSignal): StopController | undefined { + const hasTimeout = typeof timeout === "number" && Number.isFinite(timeout); + if (!hasTimeout && signal === undefined) { return undefined; } - let timer: ReturnType; + let timer: ReturnType | undefined; + let abortListener: Disposable | undefined; const promise = new Promise((_, reject) => { - // Not unref'd: dispose() always clears it, and on Windows an unref'd timer - // alone under bun:test leaves the uws loop inactive so auto_tick busy-spins. - timer = realSetTimeout(() => reject(makeTestFailure(`test timed out after ${timeout}ms`)), timeout); + if (hasTimeout) { + // Not unref'd: dispose() always clears it, and on Windows an unref'd timer + // alone under bun:test leaves the uws loop inactive so auto_tick busy-spins. + timer = realSetTimeout(() => reject(makeTestFailure(`test timed out after ${timeout}ms`)), timeout); + } + if (signal !== undefined) { + addAbortListener ??= require("internal/abort_listener").addAbortListener; + abortListener = addAbortListener(signal, () => reject(abortFailure(signal))); + } }); // Swallow the rejection when nothing is racing it anymore. promise.catch(() => {}); - return { promise, dispose: () => realClearTimeout(timer) }; + return { + promise, + dispose() { + if (timer !== undefined) realClearTimeout(timer); + abortListener?.[Symbol.dispose](); + }, + }; } // Runs `run` racing Node's test timeout; the timer starts before the body so a @@ -2160,8 +2251,6 @@ function awaitWithTimeout(run: () => unknown, timeout: number | undefined) { return raceWithTimeoutAndSignal(run, timeout, undefined); } -let addAbortListener; - async function raceWithTimeoutAndSignal( run: () => unknown, timeout: number | undefined, @@ -2259,9 +2348,22 @@ async function executeTestNode(node: TestNode, fn: TestFn): Promise { // test's own after hooks. Returns the failure (if any) instead of throwing. node.started = true; const started = runChildReporterEnabled ? performance.now() : 0; + let failure: unknown; + + const cancelledBeforeStart = node.cancelError; + if (cancelledBeforeStart !== undefined) { + // Swept while queued: no hooks and no body ([kShouldAbort], test.js:1306). + failure = applyExpectFailure(node, cancelledBeforeStart); + node.passed = failure === undefined; + node.error = failure ?? cancelledBeforeStart; + node.finish(); + reportNodeToRunParent(node, started); + return failure; + } + const ctx = node.getCtx(); const ancestors = ancestorChain(node); - let failure: unknown; + const cancellation = node.inStoppableSuite ? node.cancellation() : undefined; // Node applies the plan option before the beforeEach hooks run, and only for a // truthy count, so `{ plan: 0 }` installs no plan at all (test.js:1313-1315). @@ -2281,11 +2383,17 @@ async function executeTestNode(node: TestNode, fn: TestFn): Promise { failure = err; } - if (failure === undefined) { + // Swept during the beforeEach hooks: the body never starts. + if (failure === undefined && node.cancelError === undefined) { // Node arms one stopPromise (timeout + signal) and races both the body // AND the plan wait against it. Arm timeout once here so plan({wait:true}) // is bounded by the same test timeout, not left unbounded. const stop = createStopController(node.options.timeout); + const stops: Promise[] = []; + if (stop !== undefined) stops.push(stop.promise); + if (cancellation !== undefined) stops.push(cancellation); + const untilStopped = (pending: Promise) => + stops.length === 0 ? pending : Promise.race([...stops, pending]); try { const runBody = async () => { await runWithNode(node, () => invokeTestFn(fn, ctx)); @@ -2295,7 +2403,7 @@ async function executeTestNode(node: TestNode, fn: TestFn): Promise { }; try { - await (stop === undefined ? runBody() : Promise.race([stop.promise, runBody()])); + await untilStopped(runBody()); } catch (err) { // A body that throws or rejects with a nullish value must still fail. failure = err ?? makeTestFailure("test failed"); @@ -2310,15 +2418,14 @@ async function executeTestNode(node: TestNode, fn: TestFn): Promise { try { const pending = plan.check(); if (pending !== undefined) { - // Defuse: if stop wins the race, plan's own wait-timeout may still + // Defuse: if a stop wins the race, plan's own wait-timeout may still // reject `pending` afterward with no one listening. pending.catch(() => {}); - await (stop === undefined ? pending : Promise.race([stop.promise, pending])); + await untilStopped(pending); // A t.test() that fulfilled the plan from an async callback was // scheduled onto subtestChain during the wait; drain again so its // failure reaches failedSubtests below (Node fails the parent). - const drain = drainSubtestChain(node); - await (stop === undefined ? drain : Promise.race([stop.promise, drain])); + await untilStopped(drainSubtestChain(node)); } } catch (err) { failure = err; @@ -2342,6 +2449,9 @@ async function executeTestNode(node: TestNode, fn: TestFn): Promise { } } + // A cancellation that arrived while running is the verdict (Node's fail() keeps the first error). + failure = node.cancelError ?? failure; + const bodyFailure = failure; failure = applyExpectFailure(node, failure); const acceptedXfail = bodyFailure !== undefined && failure === undefined; @@ -2353,7 +2463,7 @@ async function executeTestNode(node: TestNode, fn: TestFn): Promise { node.error = failure ?? (acceptedXfail ? bodyFailure : null); // Mark finished before hooks so a late t.test() from an after/afterEach // hook hits addTest()'s parentAlreadyFinished path (Node cancels these). - node.finished = true; + node.finish(); for (let i = ancestors.length - 1; i >= 0; i--) { const ancestor = ancestors[i]; @@ -2389,13 +2499,14 @@ async function executeTestNode(node: TestNode, fn: TestFn): Promise { function scheduleSubtest(parent: TestNode, child: TestNode, fn: TestFn, ownTodo: boolean): Promise { const run = async () => { if (child.options.skip) { - child.finished = true; + child.finish(); child.passed = true; return; } let failure: unknown; try { - await runOwnBeforeHooks(parent); + // A swept child must not trigger the before hooks of a suite that never started. + if (child.cancelError === undefined) await runOwnBeforeHooks(parent); failure = await executeTestNode(child, fn); } catch (err) { failure = err; @@ -2430,10 +2541,56 @@ async function drainSubtestChain(node: TestNode) { } while (chain !== node.subtestChain); } +// Node's [kShouldAbort] (test.js:1245 and, after the before hooks, 1866). Node +// drops the children; cancelled ones report without running when reached. +function suiteShouldAbort(suite: TestNode): boolean { + if (suite.cancelError === undefined) { + const { signal } = suite.options; + if (!signal?.aborted) return false; + suite.cancel(abortFailure(signal)); + } + suite.cancelSubtests(); + return true; +} + +// stopTest() of Suite.run() (test.js:1865). Node sweeps the children after the +// after hooks; here the sweep is what lets the children drain before them. +function armSuiteStop(suite: TestNode) { + if (suiteShouldAbort(suite)) return; + const { timeout, signal } = suite.options; + const stop = createStopController(timeout, signal); + if (stop === undefined) return; + suite.suiteStop = stop; + stop.promise.catch(error => { + stop.dispose(); + // A timeout only fails the suite (test.js:1876); an aborted signal cancels + // it, aborting the suite's own signal too (test.js:724). + if (signal?.aborted) suite.cancel(error); + else suite.fail(error); + suite.cancelSubtests(); + }); +} + +// First half of Suite.run(); scheduleSuiteSubtest's run() is the second. Must +// not reject: the children are chained behind it. +async function startInlineSuite(suite: TestNode) { + if (suiteShouldAbort(suite)) return; + suite.suiteStarted = true; + try { + await runOwnBeforeHooks(suite); + } catch (err) { + // Fails the suite like Node; the children fail through the memoized rejection. + recordSuiteFailure(suite, err); + return; + } + armSuiteStop(suite); +} + function scheduleSuiteSubtest(parent: TestNode, suite: TestNode, build: unknown, ownTodo: boolean): Promise { // A describe()/suite() created while a test is running becomes a suite // subtest: its children were collected eagerly when the callback ran and are - // already chained on the suite's own subtestChain; failures roll up here. + // already chained on the suite's own subtestChain behind startInlineSuite; + // failures roll up here. const run = async () => { if (build !== undefined) { try { @@ -2444,23 +2601,24 @@ function scheduleSuiteSubtest(parent: TestNode, suite: TestNode, build: unknown, recordSuiteFailure(suite, err); } } - try { - await runOwnBeforeHooks(suite); - } catch (err) { - // A failing suite-level before hook fails the suite, like Node. - recordSuiteFailure(suite, err); - } - // Wait for children created during the callback and any they schedule. + // Covers startInlineSuite too. A stopped suite drains as its children + // report, once any hook they were in the middle of returns (Node moves on + // without waiting for such a hook). await drainSubtestChain(suite); - for (const hook of suite.hooks.after) { - try { - await runHook(hook, suite, suite.getSuiteCtx()); - } catch (err) { - recordSuiteFailure(suite, err); + suite.suiteStop?.dispose(); + // A suite that never started skips its after hooks too (Node). + if (suite.suiteStarted) { + for (const hook of suite.hooks.after) { + try { + await runHook(hook, suite, suite.getSuiteCtx()); + } catch (err) { + recordSuiteFailure(suite, err); + } } } - suite.finished = true; - suite.passed = suite.failedSubtests === 0; + suite.finish(); + const { cancelError } = suite; + suite.passed = cancelError === undefined && suite.failedSubtests === 0; if (runChildReporterEnabled) { emitRunChildEvent(suite.passed ? "test:pass" : "test:fail", { __proto__: null, @@ -2474,17 +2632,18 @@ function scheduleSuiteSubtest(parent: TestNode, suite: TestNode, build: unknown, error: suite.passed ? undefined : serializeRunError( - makeTestFailure( - `${suite.failedSubtests} subtest${suite.failedSubtests > 1 ? "s" : ""} failed`, - "subtestsFailed", - ), + cancelError ?? + makeTestFailure( + `${suite.failedSubtests} subtest${suite.failedSubtests > 1 ? "s" : ""} failed`, + "subtestsFailed", + ), ), }); } // A todo suite's failures do not fail the owning test (Node). - if (suite.failedSubtests > 0 && !ownTodo) { + if (!suite.passed && !ownTodo) { parent.failedSubtests++; - parent.firstSubtestError ??= suite.firstSubtestError; + parent.firstSubtestError ??= cancelError ?? suite.firstSubtestError; } }; const result = (parent.subtestChain = parent.subtestChain.then(run)); @@ -2517,6 +2676,47 @@ function bunTestOptions(options: TestOptions) { return undefined; } +// Suite.run() for a top-level suite, whose children bun:test runs itself: hooks +// of one kind run in registration order, so this pair (registered before the +// describe callback) runs ahead of the suite's own hooks and the trailing pair +// behind them. They take `done` like every callback handed to bun:test, and +// fail by throwing, which bun:test reports as the suite's own failure (from a +// beforeAll it also skips the block); a todo suite's failure is none in Node. +function registerTopLevelSuiteLeadingHooks(suite: TestNode) { + const { beforeAll, afterAll } = bunTest(); + beforeAll((done: (error?: unknown) => void) => { + if (!suiteShouldAbort(suite)) suite.suiteStarted = true; + else if (!suite.todoFlag) throw suite.cancelError; + done(); + }); + // Before the suite's after hooks: the stop is over once the children are. + afterAll((done: (error?: unknown) => void) => { + suite.suiteStop?.dispose(); + done(); + }); +} + +function registerTopLevelSuiteTrailingHooks(suite: TestNode) { + const { beforeAll, afterAll } = bunTest(); + // After the suite's before hooks, which do not count against the timeout. + beforeAll((done: (error?: unknown) => void) => { + armSuiteStop(suite); + done(); + }); + afterAll((done: (error?: unknown) => void) => { + const stopped = suite.suiteStarted && suite.cancelError !== undefined; + suite.finish(); + if (stopped && !suite.todoFlag) throw suite.cancelError; + done(); + }); +} + +// Checked by before()/after(): bun:test still runs a block's afterAll hooks +// after its beforeAll failed, and a todo suite's leading beforeAll never fails. +function suiteNeverStarted(suite: TestNode): boolean { + return suite.cancelError !== undefined && !suite.suiteStarted; +} + function currentCollectionParent(): TestNode { const node = currentNode(); if (node !== undefined && !node.isExecutionPhase && node.isSuite) { @@ -2682,11 +2882,12 @@ function addSuite( if (ownTodo) suite.todoFlag = true; // The suite's children must run after the parent's previously scheduled // subtests AND after the describe callback's own returned promise settles - // (Node's Suite.run awaits buildPromise before iterating subtests). The - // callback has not returned yet so its promise does not exist; seed the - // chain through a gate the callback's settlement opens. + // (Node's Suite.run awaits buildPromise before iterating subtests), then + // behind startInlineSuite. The callback has not returned yet so its promise + // does not exist; seed the chain through a gate the callback's settlement + // opens. const gate = Promise.withResolvers(); - suite.subtestChain = runningNode.subtestChain.then(() => gate.promise); + suite.subtestChain = runningNode.subtestChain.then(() => gate.promise).then(() => startInlineSuite(suite)); // Build the suite eagerly (Node also runs describe callbacks immediately), // collecting children onto the suite's own subtest chain. let build: unknown; @@ -2718,13 +2919,26 @@ function addSuite( // describe.todo(name, { skip: true }, fn) is a skip. const effectiveMode = mode === "skip" || options.skip ? "skip" : mode === "todo" || options.todo ? "todo" : undefined; + const stoppable = hasStopOptions(options) || suiteNode.inStoppableSuite; // Node never invokes a skipped suite's callback (it does run a todo one), so // the children are never declared and side effects in the body never happen. const wrapped = effectiveMode === "skip" ? kDefaultFunction : () => { - return runWithNode(suiteNode, () => fn(suiteNode.getSuiteCtx())); + if (stoppable) registerTopLevelSuiteLeadingHooks(suiteNode); + const built = runWithNode(suiteNode, () => fn(suiteNode.getSuiteCtx())); + if (!stoppable) return built; + if (built != null && typeof (built as PromiseLike).then === "function") { + // The scope stays open until the promise settles, so these land + // behind hooks the callback registered after an await. + return (built as PromiseLike).then(value => { + registerTopLevelSuiteTrailingHooks(suiteNode); + return value; + }); + } + registerTopLevelSuiteTrailingHooks(suiteNode); + return built; }; const passOptions = bunTestOptions(options); @@ -2805,6 +3019,10 @@ function before(arg0: unknown, arg1: unknown) { } const { beforeAll } = bunTest(); beforeAll((done: (error?: unknown) => void) => { + if (suiteNeverStarted(owner)) { + done(); + return; + } Promise.resolve(runHook(hook, owner, hookArgFor(owner))).then( () => done(), err => done(err ?? new Error("before hook failed")), @@ -2821,6 +3039,10 @@ function after(arg0: unknown, arg1: unknown) { } const { afterAll } = bunTest(); afterAll((done: (error?: unknown) => void) => { + if (suiteNeverStarted(owner)) { + done(); + return; + } Promise.resolve(runHook(hook, owner, hookArgFor(owner))).then( () => done(), err => done(err ?? new Error("after hook failed")), diff --git a/test/js/node/test_runner/fixtures/32-suite-timeout.js b/test/js/node/test_runner/fixtures/32-suite-timeout.js new file mode 100644 index 000000000000..384ff0d3f5c6 --- /dev/null +++ b/test/js/node/test_runner/fixtures/32-suite-timeout.js @@ -0,0 +1,68 @@ +// The `timeout` option of a top-level suite, as in Node v26.3.0's Suite.run(): +// when it fires, the child that is running is cancelled, the children and +// nested suites queued behind it never run, the suite's own after hooks still +// run (with the suite's own signal left alone: a timeout fails a suite, it does +// not abort it), and the suite itself fails with the timeout. The first suite +// here fails on purpose; node-test.test.ts asserts the exact counts and the +// OBS markers. Differences from `node --test` with this file: a suite's own +// failure shows up as a failed hook of its describe block, the children of a +// suite that never started are skipped rather than listed one by one, and the +// running child is cancelled before the after hooks run rather than after. +const { describe, it, before, after } = require("node:test"); + +const sleep = ms => new Promise(resolve => setTimeout(resolve, ms)); +// A cancelled test's t.signal aborts; the sleep only bounds a runner that never +// cancels, so that such a runner reports the wrong outcome instead of hanging. +const untilCancelled = t => + Promise.race([new Promise(resolve => t.signal.addEventListener("abort", resolve)), sleep(200)]); + +describe("suite that times out", { timeout: 20 }, () => { + before(() => console.log("OBS before ran")); + after(suite => console.log("OBS after ran, suite signal aborted: " + suite.signal.aborted)); + + it("running when the suite times out", async t => { + await untilCancelled(t); + console.log("OBS running child settled, signal aborted: " + t.signal.aborted); + }); + + it("queued when the suite times out", () => { + console.log("OBS queued child ran"); + }); + + describe("nested suite queued when the suite times out", () => { + before(() => console.log("OBS nested before ran")); + after(() => console.log("OBS nested after ran")); + it("nested child", () => console.log("OBS nested child ran")); + }); +}); + +// Also pins that the stop is released once the suite is done: a leaked 30s +// timer would keep the run() child of node-test.test.ts alive. +describe("suite within its timeout", { timeout: 30_000 }, () => { + it("passes", () => {}); + + // A test's own timeout still applies inside a suite that has one (this test + // fails on purpose), and it does not stop the suite around it. + it("has a shorter timeout of its own", { timeout: 20 }, async () => { + await sleep(200); + }); + + it("still runs after a sibling timed out on its own", () => {}); + + describe("nested suite without a timeout of its own", () => { + it("passes too", () => {}); + }); +}); + +// Node arms the suite's stop after its before hooks, so they do not count +// against the timeout. Had the stop been armed before the hook, its shorter +// timer would have fired during the hook and the child would have been +// cancelled without ever starting. The async callback also covers registering +// the suite's own bookkeeping after the callback's promise settles. +describe("suite whose before hook outlasts its timeout", { timeout: 1000 }, async () => { + await null; + before(() => sleep(1100)); + it("runs after the slow before hook", () => { + console.log("OBS child ran after the slow before hook"); + }); +}); diff --git a/test/js/node/test_runner/fixtures/33-suite-signal.js b/test/js/node/test_runner/fixtures/33-suite-signal.js new file mode 100644 index 000000000000..6ff38b7fcfea --- /dev/null +++ b/test/js/node/test_runner/fixtures/33-suite-signal.js @@ -0,0 +1,89 @@ +// The `signal` option of a top-level suite, as in Node v26.3.0. A suite whose +// signal has aborted by the time its turn comes (before it was declared, or by +// an earlier test) fails with the reason and runs nothing: no hooks, no +// children, although its describe callback does run. A signal that aborts +// while the suite runs cancels the running child and the queued one, still +// runs the after hooks, and fails the suite with the reason; one aborted by +// the suite's own before hook still runs the remaining before hooks and the +// after hooks, and cancels every child without starting it; a falsy reason is +// reported as Node's own "The test was aborted". Five suites here fail on +// purpose; node-test.test.ts asserts the exact counts and the OBS markers. +// Differences from `node --test` with this file: a suite's own failure shows up +// as a failed hook of its describe block, the children of a suite that never +// started are skipped rather than listed one by one, and the running child is +// cancelled before the after hooks run rather than after. +const { describe, it, before, after, test } = require("node:test"); + +const sleep = ms => new Promise(resolve => setTimeout(resolve, ms)); +// A cancelled test's t.signal aborts; the sleep only bounds a runner that never +// cancels, so that such a runner reports the wrong outcome instead of hanging. +const untilCancelled = t => + Promise.race([new Promise(resolve => t.signal.addEventListener("abort", resolve)), sleep(200)]); + +describe( + "suite aborted before it was declared", + { signal: AbortSignal.abort(new Error("aborted before declaration")) }, + () => { + console.log("OBS pre-aborted suite callback ran"); + before(() => console.log("OBS pre-aborted suite before ran")); + after(() => console.log("OBS pre-aborted suite after ran")); + it("child of the pre-aborted suite", () => console.log("OBS pre-aborted suite child ran")); + }, +); + +const abortedByEarlierTest = new AbortController(); + +test("aborts the next suite's signal before that suite runs", () => { + abortedByEarlierTest.abort(new Error("aborted by an earlier test")); +}); + +describe("suite aborted before it ran", { signal: abortedByEarlierTest.signal }, () => { + it("child of the suite aborted before it ran", () => console.log("OBS aborted-before-run suite child ran")); +}); + +const abortedWhileRunning = new AbortController(); + +describe("suite aborted while running", { signal: abortedWhileRunning.signal }, () => { + after(suite => console.log("OBS after ran, suite signal aborted: " + suite.signal.aborted)); + + it("finished before the abort", () => console.log("OBS first child ran")); + + it("running when the signal aborts", async t => { + abortedWhileRunning.abort(new Error("aborted while running")); + await untilCancelled(t); + console.log("OBS running child settled, signal aborted: " + t.signal.aborted); + }); + + it("queued when the signal aborts", () => console.log("OBS queued child ran")); +}); + +const abortedByBeforeHook = new AbortController(); + +describe("suite aborted by its own before hook", { signal: abortedByBeforeHook.signal }, () => { + before(() => { + abortedByBeforeHook.abort(new Error("aborted by a before hook")); + console.log("OBS before hook aborted the suite"); + }); + before(() => console.log("OBS second before hook still ran")); + after(suite => + console.log("OBS after of the suite aborted by its hook ran, suite signal aborted: " + suite.signal.aborted), + ); + + it("child of the suite aborted by its hook", () => console.log("OBS child of the suite aborted by its hook ran")); +}); + +describe("suite aborted with a falsy reason", { signal: AbortSignal.abort(0) }, () => { + it("child of the suite aborted with a falsy reason", () => console.log("OBS falsy-reason suite child ran")); +}); + +// The listener is removed once the suite is done; aborting afterwards changes +// nothing (node-test.test.ts only checks that this suite passes). +const neverAbortedInTime = new AbortController(); + +describe("suite whose signal does not abort while it runs", { signal: neverAbortedInTime.signal }, () => { + it("passes", () => {}); +}); + +test("aborting a finished suite's signal is a no-op", () => { + neverAbortedInTime.abort(new Error("too late to matter")); +}); diff --git a/test/js/node/test_runner/fixtures/34-inline-suite-stop.js b/test/js/node/test_runner/fixtures/34-inline-suite-stop.js new file mode 100644 index 000000000000..c09273512e63 --- /dev/null +++ b/test/js/node/test_runner/fixtures/34-inline-suite-stop.js @@ -0,0 +1,114 @@ +// The `timeout` and `signal` options of inline suites (describe() inside a +// running test), as in Node v26.3.0: a suite that times out is failed by the +// timeout, its running child is cancelled, the children and nested suites +// queued behind it never run, and its after hooks still run (with the suite's +// own signal left alone: a timeout fails a suite, it does not abort it); a +// suite whose signal has already aborted is failed by the reason and runs +// nothing; one whose own before hook aborts the signal still runs its other +// hooks but starts no child. All three fail the test that owns them, so the +// first test fails on purpose ("3 subtests failed"); the last test checks what +// the suites left behind, and passes verbatim under `node --test` too. +// node-test.test.ts also runs this file through run(), which reports each +// suite's own error. One difference from Node there: Node drops the children +// of an aborted suite without reporting them, while here they report as +// cancelled. +const assert = require("node:assert"); +const { test, describe, it, before, after } = require("node:test"); + +const sleep = ms => new Promise(resolve => setTimeout(resolve, ms)); +// A cancelled test's t.signal aborts; the sleep only bounds a runner that never +// cancels, so that such a runner reports the wrong outcome instead of hanging. +const untilCancelled = t => + Promise.race([new Promise(resolve => t.signal.addEventListener("abort", resolve)), sleep(200)]); + +const seen = { + hooks: [], + runningChildSignalAborted: undefined, + queuedChildRan: false, + nestedChildRan: false, + abortedSuiteChildRan: false, + childOfSuiteAbortedByHookRan: false, + childWithinTimeoutRan: false, + childAfterSlowBeforeHookRan: false, +}; + +test("inline suites are stopped by their timeout or an aborted signal", () => { + describe("times out", { timeout: 20 }, () => { + before(() => seen.hooks.push("before")); + after(suite => seen.hooks.push("after, suite signal aborted: " + suite.signal.aborted)); + it("running when the suite times out", async t => { + await untilCancelled(t); + seen.runningChildSignalAborted = t.signal.aborted; + }); + it("queued when the suite times out", () => { + seen.queuedChildRan = true; + }); + describe("nested suite queued when the suite times out", () => { + before(() => seen.hooks.push("nested before")); + after(() => seen.hooks.push("nested after")); + it("nested child", () => { + seen.nestedChildRan = true; + }); + }); + }); + + describe("already aborted", { signal: AbortSignal.abort(new Error("inline abort reason")) }, () => { + before(() => seen.hooks.push("aborted suite before")); + after(() => seen.hooks.push("aborted suite after")); + it("child of the aborted suite", () => { + seen.abortedSuiteChildRan = true; + }); + }); + + const abortedByBeforeHook = new AbortController(); + describe("aborted by its own before hook", { signal: abortedByBeforeHook.signal }, () => { + before(() => { + abortedByBeforeHook.abort(new Error("inline suite aborted by a before hook")); + seen.hooks.push("aborting before"); + }); + before(() => seen.hooks.push("second before after the abort")); + after(suite => seen.hooks.push("after the abort, suite signal aborted: " + suite.signal.aborted)); + it("child of the suite aborted by its hook", () => { + seen.childOfSuiteAbortedByHookRan = true; + }); + }); + + describe("within its timeout", { timeout: 30_000 }, () => { + it("passes", () => { + seen.childWithinTimeoutRan = true; + }); + }); +}); + +// Node arms the suite's stop after its before hooks, so they do not count +// against the timeout. Had the stop been armed before the hook, its shorter +// timer would have fired during the hook and the child would have been +// cancelled without ever starting, failing this test as well. +test("an inline suite's before hooks do not count against its timeout", () => { + describe("before hook outlasts the timeout", { timeout: 1000 }, () => { + before(() => sleep(1100)); + it("runs after the slow before hook", () => { + seen.childAfterSlowBeforeHookRan = true; + }); + }); +}); + +test("what the stopped inline suites left behind", () => { + // Each test above waited for its suites before it finished. + assert.deepStrictEqual(seen, { + hooks: [ + "before", + "after, suite signal aborted: false", + "aborting before", + "second before after the abort", + "after the abort, suite signal aborted: true", + ], + runningChildSignalAborted: true, + queuedChildRan: false, + nestedChildRan: false, + abortedSuiteChildRan: false, + childOfSuiteAbortedByHookRan: false, + childWithinTimeoutRan: true, + childAfterSlowBeforeHookRan: true, + }); +}); diff --git a/test/js/node/test_runner/node-test.test.ts b/test/js/node/test_runner/node-test.test.ts index 2a27963203c0..698c4f849287 100644 --- a/test/js/node/test_runner/node-test.test.ts +++ b/test/js/node/test_runner/node-test.test.ts @@ -2,6 +2,7 @@ import { spawn } from "bun"; import { describe, expect, test } from "bun:test"; import { bunEnv, bunExe } from "harness"; import { join } from "node:path"; +import { run } from "node:test"; describe("node:test", () => { // These three drive the largest fixtures (01-harness has 32 node:test cases); @@ -323,6 +324,113 @@ describe("node:test", () => { stderr: expect.stringContaining("0 fail"), }); }); + + test("should enforce a top-level suite's signal option", async () => { + const { exitCode, stdout, stderr } = await runTests(["33-suite-signal.js"]); + const markers = stdout.split("\n").filter(line => line.startsWith("OBS ")); + expect(markers).toEqual([ + "OBS pre-aborted suite callback ran", + "OBS first child ran", + "OBS running child settled, signal aborted: true", + "OBS after ran, suite signal aborted: true", + "OBS before hook aborted the suite", + "OBS second before hook still ran", + "OBS after of the suite aborted by its hook ran, suite signal aborted: true", + ]); + // Each aborted suite fails with its signal's reason. + expect(stderr).toContain("aborted before declaration"); + expect(stderr).toContain("aborted by an earlier test"); + expect(stderr).toContain("aborted while running"); + expect(stderr).toContain("aborted by a before hook"); + expect(stderr).toContain("The test was aborted"); + expect(stderr).toContain("test did not finish before its parent and was cancelled"); + expect(stderr).not.toContain("too late to matter"); + // The deliberate failures: five suites, the two children cancelled by the + // abort while running, and the child of the suite its before hook aborted. + expect(stderr).toContain("4 pass"); + expect({ exitCode, stderr }).toMatchObject({ + exitCode: 1, + stderr: expect.stringContaining("8 fail"), + }); + }); + + // Fixtures 32 and 34 each hold a before hook that deliberately outlasts a 1s + // suite timeout, on top of the debug+ASAN child's startup, so like the + // fixtures at the top they get headroom and overlap with each other. + test.concurrent( + "should enforce a top-level suite's timeout option", + async () => { + const { exitCode, stdout, stderr } = await runTests(["32-suite-timeout.js"]); + const markers = stdout.split("\n").filter(line => line.startsWith("OBS ")); + expect(markers).toEqual([ + "OBS before ran", + "OBS running child settled, signal aborted: true", + "OBS after ran, suite signal aborted: false", + "OBS child ran after the slow before hook", + ]); + // The suite's own verdict, and the verdict of the children it stopped. + expect(stderr).toContain("test timed out after 20ms"); + expect(stderr).toContain("test did not finish before its parent and was cancelled"); + // The deliberate failures: the running child, the queued child, the + // queued nested suite, the suite itself, and the test with a timeout of + // its own. + expect(stderr).toContain("4 pass"); + expect({ exitCode, stderr }).toMatchObject({ + exitCode: 1, + stderr: expect.stringContaining("5 fail"), + }); + }, + 30_000, + ); + + test.concurrent( + "should enforce an inline suite's timeout and signal options", + async () => { + const { exitCode, stderr } = await runTests(["34-inline-suite-stop.js"]); + // The owning test fails for the three stopped suites, with the timeout as + // the reported cause; the before-hook test and the one inspecting what + // the suites left behind pass. + expect(stderr).toContain("error: 3 subtests failed"); + expect(stderr).toContain("test timed out after 20ms"); + expect(stderr).toContain("2 pass"); + expect({ exitCode, stderr }).toMatchObject({ + exitCode: 1, + stderr: expect.stringContaining("1 fail"), + }); + }, + 30_000, + ); + + test.concurrent( + "should report a stopped inline suite's own error and its cancelled children through run()", + async () => { + const events = await run({ files: [join(import.meta.dirname, "fixtures", "34-inline-suite-stop.js")] }).toArray(); + const outcomes = events + .filter(({ type }) => type === "test:pass" || type === "test:fail") + .map(({ type, data }) => [data.name, type === "test:pass" ? "pass" : data.details.error.message]) + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)); + const cancelled = "test did not finish before its parent and was cancelled"; + expect(outcomes).toEqual([ + ["aborted by its own before hook", "inline suite aborted by a before hook"], + ["already aborted", "inline abort reason"], + ["an inline suite's before hooks do not count against its timeout", "pass"], + ["before hook outlasts the timeout", "pass"], + ["child of the aborted suite", cancelled], + ["child of the suite aborted by its hook", cancelled], + ["inline suites are stopped by their timeout or an aborted signal", "3 subtests failed"], + ["nested child", cancelled], + ["nested suite queued when the suite times out", cancelled], + ["passes", "pass"], + ["queued when the suite times out", cancelled], + ["running when the suite times out", cancelled], + ["runs after the slow before hook", "pass"], + ["times out", "test timed out after 20ms"], + ["what the stopped inline suites left behind", "pass"], + ["within its timeout", "pass"], + ]); + }, + 30_000, + ); }); async function runTests(filenames: string[], env: Record = {}, args: string[] = []) {