diff --git a/scripts/runner.node.mjs b/scripts/runner.node.mjs index 67d86bd6fe50..f9afde13b7e0 100755 --- a/scripts/runner.node.mjs +++ b/scripts/runner.node.mjs @@ -759,11 +759,36 @@ async function runTests() { runWithBunTest ||= title === "test/js/node/test/parallel/test-fs-append-file-flush.js"; runWithBunTest ||= title === "test/js/node/test/parallel/test-fs-write-file-flush.js"; runWithBunTest ||= title === "test/js/node/test/parallel/test-fs-write-stream-flush.js"; + // A file that only drives node:test's run() is the parent of the run, + // not a test file: Node executes it as a plain script, and under + // `bun test` a file registering no tests of its own exits before its + // run() finishes. Files that also register tests at the top level (as + // opposed to inside a NODE_TEST_CONTEXT child branch) still need + // `bun test`. run() spawns its own children with `bun test`. + const importsRun = + /\brun\b[^\n]*=\s*require\(['"]node:test['"]\)/.test(testContent) || + /import\s*{[^}]*\brun\b[^}]*}\s*from\s*['"]node:test['"]/.test(testContent); + // Registrations behind a NODE_TEST_CONTEXT guard belong to the child + // run() spawns, not to this process; an unindented one is this file's + // own and must keep `bun test`, or it silently never runs and the file + // "passes" having tested nothing. Requiring the guard as well keeps a + // file whose only registrations are indented for some other reason + // (inside an `if`, an IIFE) on `bun test`, where the worst case is a + // real run rather than a vacuous pass. + const registersAtColumnZero = /^(?:test|it|describe|suite)\s*[.(]/m.test(testContent); + const registersTests = /(?:^|[^.\w])(?:test|it|describe|suite)\s*[.(]/m.test(testContent); + const guardsOnTestContext = testContent.includes("NODE_TEST_CONTEXT"); + const isRunDriver = importsRun && !registersAtColumnZero && (!registersTests || guardsOnTestContext); + // The needs-test filename opt-in wins over this heuristic. + if (isRunDriver && !title.includes("needs-test")) runWithBunTest = false; const subcommand = runWithBunTest ? "test" : "run"; const env = { FORCE_COLOR: "0", NO_COLOR: "1", BUN_DEBUG_QUIET_LOGS: "1", + // Node parity: a node test process exits only when its event loop + // drains, and common.mustCall() verifies counts in 'exit' handlers. + BUN_TEST_DRAIN_EVENT_LOOP: "1", }; if (!isWindows && title.includes("/sequential/")) { // Sequential node tests share common.PORT (12346); a cluster worker diff --git a/src/bun_core/env_var.rs b/src/bun_core/env_var.rs index 25cc3222e26f..c1d94863c740 100644 --- a/src/bun_core/env_var.rs +++ b/src/bun_core/env_var.rs @@ -112,6 +112,10 @@ platform_specific_new!(pub C_INCLUDE_PATH: string, posix = "C_INCLUDE_PATH", win // Standard C compiler environment variable for library paths (colon-separated). // Used by bun:ffi's TinyCC integration for systems like NixOS. platform_specific_new!(pub LIBRARY_PATH: string, posix = "LIBRARY_PATH", windows = None, {}); +// Drain the event loop after a file's tests finish so node-style +// `process.on('exit')` checks (e.g. common.mustCall) see completed async work. +// Opt-in for the vendored node:test suite and run() children. +new!(pub BUN_TEST_DRAIN_EVENT_LOOP: boolean, "BUN_TEST_DRAIN_EVENT_LOOP", { default: false }); new!(pub BUN_TMPDIR: string, "BUN_TMPDIR", {}); new!(pub BUN_TRACY_PATH: string, "BUN_TRACY_PATH", {}); new!(pub BUN_WATCHER_TRACE: string, "BUN_WATCHER_TRACE", {}); diff --git a/src/js/node/net.ts b/src/js/node/net.ts index ebed05641f93..5429f63e9ae3 100644 --- a/src/js/node/net.ts +++ b/src/js/node/net.ts @@ -3551,6 +3551,10 @@ Server.prototype.close = function close(callback) { }; Server.prototype[Symbol.asyncDispose] = function () { + // Node resolves immediately when the server is not listening (lib/net.js + // SymbolAsyncDispose); without the guard a second dispose rejects with + // ERR_SERVER_NOT_RUNNING and re-emits 'close'. + if (!this._handle) return Promise.$resolve(); const { resolve, reject, promise } = Promise.withResolvers(); this.close(function (err, ...args) { if (err) reject(err); diff --git a/src/js/node/test.ts b/src/js/node/test.ts index 79c0eba91e9b..75abf5fb79e3 100644 --- a/src/js/node/test.ts +++ b/src/js/node/test.ts @@ -18,6 +18,7 @@ const { validateArray, validateAbortSignal, validateUint32, + validateOneOf, } = require("internal/validators"); const kDefaultName = ""; @@ -34,8 +35,710 @@ const kTimeoutMax = 2 ** 31 - 1; const kBunTestDefaultTimeoutMs = 5_000; const kJoinSeparator = " > "; -function run() { - throwNotImplemented("run()", 5090, "Use `bun:test` in the interim."); +// ----------------------------------------------------------------------------- +// run() +// +// Port of Node.js lib/internal/test_runner/{runner,tests_stream}.js (v26.3.0). +// Files run in child processes (node's isolation:'process'); the child is spawned +// with kRunChildEnv set, which makes this module stream one JSON event per line +// on stdout. Unmarked stdout/stderr lines become test:stdout/test:stderr, the +// same split node makes around its V8-serializer framing. +// ----------------------------------------------------------------------------- + +// node's own tests branch on NODE_TEST_CONTEXT to tell the parent from the +// spawned child, so use node's variable and value rather than a bun-specific one. +const kRunChildEnv = "NODE_TEST_CONTEXT"; +const kRunChildEnvValue = "child-v8"; +const kRunEventPrefix = "\0bun:test:run\0"; + +// Created lazily on the first run() call so the common test()/describe() +// path never loads node:stream. +type TestsStream = InstanceType>; +let TestsStreamClass: ReturnType | undefined; + +function getTestsStreamClass() { + const { Readable } = require("node:stream"); + // Mirrors node lib/internal/test_runner/tests_stream.js: a plain-array + // buffer plus one public method per event type delegating to #emit. + return class TestsStream extends (Readable as typeof import("node:stream").Readable) { + #buffer: unknown[] = []; + #canPush = true; + + constructor() { + super({ __proto__: null, objectMode: true, highWaterMark: Number.MAX_SAFE_INTEGER }); + } + + _read() { + this.#canPush = true; + while (this.#buffer.length > 0) { + const obj = this.#buffer.shift(); + if (!this.#tryPush(obj)) return; + } + } + + #tryPush(message: unknown) { + if (this.#canPush) { + this.#canPush = this.push(message); + } else { + $arrayPush(this.#buffer, message); + } + return this.#canPush; + } + + #emit(type: string, data?: unknown) { + this.emit(type, data); + this.#tryPush({ __proto__: null, type, data }); + } + + enqueue(data: unknown) { + this.#emit("test:enqueue", data); + } + dequeue(data: unknown) { + this.#emit("test:dequeue", data); + } + complete(data: unknown) { + this.#emit("test:complete", data); + } + pass(data: unknown) { + this.#emit("test:pass", data); + } + fail(data: unknown) { + this.#emit("test:fail", data); + } + plan(data: unknown) { + this.#emit("test:plan", data); + } + diagnostic(data: unknown) { + this.#emit("test:diagnostic", data); + } + stderr(data: unknown) { + this.#emit("test:stderr", data); + } + stdout(data: unknown) { + this.#emit("test:stdout", data); + } + summary(data: unknown) { + this.#emit("test:summary", data); + } + republish(type: string, data: unknown) { + this.#emit(type, data); + } + + endStream() { + this.#tryPush(null); + } + }; +} + +function createTestsStream(): TestsStream { + TestsStreamClass ??= getTestsStreamClass(); + return new TestsStreamClass(); +} + +function validateStringArray(value: unknown, name: string) { + validateArray(value, name); + for (let i = 0; i < (value as unknown[]).length; i++) { + validateString((value as unknown[])[i], `${name}[${i}]`); + } +} + +// node canonicalizes tag filters to lower case and rejects empty strings +// (lib/internal/test_runner/tag_filter.js). +function validateAndCanonicalizeTagFilter(value: unknown, name: string) { + validateString(value, name); + if ((value as string).length === 0) { + throw $ERR_INVALID_ARG_VALUE(name, value, "must not be empty"); + } + return (value as string).toLowerCase(); +} + +function toRegExpPatterns(value: unknown, name: string) { + const patterns = $isArray(value) ? value : [value]; + return patterns.map((entry: unknown, i: number) => { + if ($isRegExpObject(entry)) return entry; + if (typeof entry === "string") return convertStringToRegExp(entry, `${name}[${i}]`); + throw $ERR_INVALID_ARG_TYPE(`${name}[${i}]`, ["string", "RegExp"], entry); + }); +} + +// node's utils.js convertStringToRegExp: a "/pattern/flags" string becomes that +// RegExp, anything else is matched literally. +function convertStringToRegExp(str: string, name: string) { + const match = str.match(/^\/(.*)\/([a-z]*)$/); + const pattern = match?.[1] ?? str; + const flags = match?.[2] ?? ""; + try { + return new RegExp(pattern, flags); + } catch (err) { + throw $ERR_INVALID_ARG_VALUE(name, str, `is an invalid regular expression: ${(err as Error).message}`); + } +} + +// node's emitExperimentalWarning is one-shot per feature process-wide, so +// test({ tags }) and run({ testTagFilters }) share this flag. +let tagsExperimentalWarningEmitted = false; +function emitTagsExperimentalWarning() { + if (tagsExperimentalWarningEmitted) return; + tagsExperimentalWarningEmitted = true; + process.emitWarning("Test tags is an experimental feature and might change at any time", "ExperimentalWarning"); +} + +function validateRunOptions(options: Record) { + validateObject(options, "options"); + + let { testNamePatterns, testSkipPatterns, testTagFilters, shard } = options as Record; + const { + files, + forceExit, + isolation = "process", + watch, + setup, + globalSetupPath, + only, + globPatterns, + coverage = false, + lineCoverage = 0, + branchCoverage = 0, + functionCoverage = 0, + execArgv = [], + argv = [], + cwd = process.cwd(), + env, + } = options as Record; + + // Order mirrors node's runner.js:731-909 — the errors are observable. + if (files != null) validateArray(files, "options.files"); + if (watch != null) validateBoolean(watch, "options.watch"); + if (forceExit != null) { + validateBoolean(forceExit, "options.forceExit"); + if (forceExit && watch) { + throw $ERR_INVALID_ARG_VALUE("options.forceExit", watch, "is not supported with watch mode"); + } + } + if (only != null) validateBoolean(only, "options.only"); + if (globPatterns != null) validateArray(globPatterns, "options.globPatterns"); + validateString(cwd, "options.cwd"); + if (globPatterns?.length > 0 && files?.length > 0) { + throw $ERR_INVALID_ARG_VALUE( + "options.globPatterns", + globPatterns, + "is not supported when specifying 'options.files'", + ); + } + if (shard != null) { + validateObject(shard, "options.shard"); + shard = { __proto__: null, index: shard.index, total: shard.total }; + validateInteger(shard.total, "options.shard.total", 1); + validateInteger(shard.index, "options.shard.index", 1, shard.total); + if (watch) { + throw $ERR_INVALID_ARG_VALUE("options.shard", watch, "shards not supported with watch mode"); + } + } + if (setup != null) validateFunction(setup, "options.setup"); + if (testNamePatterns != null) testNamePatterns = toRegExpPatterns(testNamePatterns, "options.testNamePatterns"); + if (testSkipPatterns != null) testSkipPatterns = toRegExpPatterns(testSkipPatterns, "options.testSkipPatterns"); + + let testTagFilterExpressions = null; + if (testTagFilters != null) { + if (!$isArray(testTagFilters)) testTagFilters = [testTagFilters]; + if (testTagFilters.length === 0) { + testTagFilters = null; + } else { + emitTagsExperimentalWarning(); + testTagFilters = testTagFilters.map((value: unknown, i: number) => + validateAndCanonicalizeTagFilter(value, `options.testTagFilters[${i}]`), + ); + testTagFilterExpressions = testTagFilters; + } + } + + validateOneOf(isolation, "options.isolation", ["process", "none"]); + validateBoolean(coverage, "options.coverage"); + validateInteger(lineCoverage, "options.lineCoverage", 0, 100); + validateInteger(branchCoverage, "options.branchCoverage", 0, 100); + validateInteger(functionCoverage, "options.functionCoverage", 0, 100); + validateStringArray(argv, "options.argv"); + validateStringArray(execArgv, "options.execArgv"); + if (globalSetupPath != null) validateString(globalSetupPath, "options.globalSetupPath"); + if (env != null) { + validateObject(env, "options.env"); + if (isolation === "none") { + throw $ERR_INVALID_ARG_VALUE("options.env", env, "is not supported with isolation='none'"); + } + } + // Node validates these via the root Test constructor, not inline here; + // the observable error is the same synchronous throw. + const { concurrency, timeout, signal } = options as Record; + if (signal !== undefined) validateAbortSignal(signal, "options.signal"); + if (timeout != null && timeout !== Infinity) validateNumber(timeout, "options.timeout", 0, kTimeoutMax); + if (concurrency != null && typeof concurrency !== "boolean") { + if (typeof concurrency === "number") validateUint32(concurrency, "options.concurrency", true); + else throw $ERR_INVALID_ARG_TYPE("options.concurrency", ["boolean", "number"], concurrency); + } + + return { + files, + forceExit, + setup, + cwd, + env, + argv, + execArgv, + isolation, + watch, + coverage, + shard, + globPatterns, + globalSetupPath, + only, + testNamePatterns, + testSkipPatterns, + testTagFilterExpressions, + concurrency, + timeout, + signal, + }; +} + +function run(options: Record = kEmptyObject) { + const opts = validateRunOptions(options); + const reporter = createTestsStream(); + + // A test file that calls run() on itself would otherwise fork forever; node + // skips the files and returns an empty stream instead. + if (runChildReporterEnabled) { + process.emitWarning("node:test run() is being called recursively within a test file. skipping running files."); + reporter.endStream(); + return reporter; + } + + // Options whose semantics we cannot honor yet must fail loudly rather than be + // silently ignored. testTagFilters and timeout are the deliberate exceptions: + // validated for node's error contract but not yet forwarded (node's own + // test-runner-filetest-location.js passes timeout). + if (opts.watch) throwNotImplemented("run({ watch: true })", 5090, "Use `bun:test --watch` in the interim."); + if (opts.coverage) throwNotImplemented("run({ coverage: true })", 5090, "Use `bun:test --coverage` in the interim."); + if (opts.shard) throwNotImplemented("run({ shard })", 5090); + if (opts.isolation === "none") throwNotImplemented("run({ isolation: 'none' })", 5090); + if (opts.globPatterns?.length > 0) throwNotImplemented("run({ globPatterns })", 5090); + if (opts.globalSetupPath != null) throwNotImplemented("run({ globalSetupPath })", 5090); + if (opts.only) throwNotImplemented("run({ only: true })", 5090); + if (opts.testNamePatterns != null) throwNotImplemented("run({ testNamePatterns })", 5090); + if (opts.testSkipPatterns != null) throwNotImplemented("run({ testSkipPatterns })", 5090); + if (opts.forceExit) throwNotImplemented("run({ forceExit: true })", 5090); + if (opts.concurrency != null) throwNotImplemented("run({ concurrency })", 5090); + if (opts.files == null) + throwNotImplemented("run() default file discovery", 5090, "Pass { files: [...] } explicitly."); + + runFiles(opts, reporter); + return reporter; +} + +function makeRunCounts() { + return { + __proto__: null, + tests: 0, + failed: 0, + passed: 0, + cancelled: 0, + skipped: 0, + todo: 0, + topLevel: 0, + suites: 0, + } as unknown as Record; +} + +function addRunCounts(into: Record, from: Record) { + for (const key of Object.keys(from)) into[key] += from[key]; +} + +function emitRunDiagnostics(reporter: TestsStream, counts: Record, durationMs: number) { + reporter.diagnostic({ __proto__: null, nesting: 0, message: `tests ${counts.tests}` }); + reporter.diagnostic({ __proto__: null, nesting: 0, message: `suites ${counts.suites}` }); + reporter.diagnostic({ __proto__: null, nesting: 0, message: `pass ${counts.passed}` }); + reporter.diagnostic({ __proto__: null, nesting: 0, message: `fail ${counts.failed}` }); + reporter.diagnostic({ __proto__: null, nesting: 0, message: `cancelled ${counts.cancelled}` }); + reporter.diagnostic({ __proto__: null, nesting: 0, message: `skipped ${counts.skipped}` }); + reporter.diagnostic({ __proto__: null, nesting: 0, message: `todo ${counts.todo}` }); + reporter.diagnostic({ __proto__: null, nesting: 0, message: `duration_ms ${durationMs}` }); +} + +// Runs each file in its own `bun test` child and republishes the child's events +// on the parent's stream, then emits the run-level plan/diagnostics/summary. +async function runFiles(opts: ReturnType, reporter: TestsStream) { + const started = Date.now(); + const counts = makeRunCounts(); + + // run() returns the stream before any file starts, and callers attach their + // listeners synchronously on the returned stream. Yield first so the earliest + // events (the file node's enqueue/dequeue) are not emitted into no listeners. + await Promise.resolve(); + + try { + if (typeof opts.setup === "function") await opts.setup(reporter); + + const files = opts.files ?? []; + let i = 0; + for (; i < files.length; i++) { + if (opts.signal?.aborted) break; + await runOneFile(files[i], opts, reporter, counts); + } + // Node cancels each not-yet-started FileTest with cancelledByParent rather + // than silently dropping it; an aborted run must not report success:true. + for (; i < files.length; i++) { + reportCancelledFile(files[i], opts, reporter, counts); + } + + reporter.plan({ __proto__: null, nesting: 0, count: counts.topLevel }); + const durationMs = Date.now() - started; + emitRunDiagnostics(reporter, counts, durationMs); + reporter.summary({ + __proto__: null, + success: counts.failed === 0 && counts.cancelled === 0, + counts, + duration_ms: durationMs, + file: undefined, + }); + } catch (err) { + reporter.destroy(err as Error); + return; + } + reporter.endStream(); +} + +function reportCancelledFile( + file: string, + opts: ReturnType, + reporter: TestsStream, + counts: Record, +) { + const path = require("node:path"); + const absolute = path.resolve(opts.cwd as string, file); + const fileNode = { + nesting: 0, + name: file, + type: "test", + testId: 1, + parentId: 0, + tags: [], + line: 1, + column: 1, + file: absolute, + }; + const error = makeTestFailure("test did not finish before its parent and was cancelled", "cancelledByParent"); + const details = { __proto__: null, duration_ms: 0, type: "test", error }; + reporter.enqueue({ __proto__: null, ...fileNode }); + reporter.dequeue({ __proto__: null, ...fileNode }); + reporter.complete({ + __proto__: null, + ...fileNode, + type: undefined, + testNumber: 1, + details: { ...details, passed: false }, + }); + reporter.fail({ __proto__: null, ...fileNode, type: undefined, testNumber: 1, details }); + counts.tests++; + counts.cancelled++; + counts.topLevel++; +} + +async function runOneFile( + file: string, + opts: ReturnType, + reporter: TestsStream, + counts: Record, +) { + const path = require("node:path"); + const absolute = path.resolve(opts.cwd as string, file); + // Node's getRunArgs builds [...execArgv, path, ...argv] so runtime flags land + // in the child's process.execArgv; bun's CLI likewise takes runtime flags + // before the `test` keyword and user args after the path. + const args = [process.execPath, ...(opts.execArgv as string[]), "test", absolute, ...(opts.argv as string[])]; + const fileStarted = Date.now(); + const fileCounts = makeRunCounts(); + + // Under process isolation node models the file itself as a top-level test, + // named by the path as it was passed in and located at 1:1. + const fileNode = { + nesting: 0, + name: file, + type: "test", + testId: 1, + parentId: 0, + tags: [], + line: 1, + column: 1, + file: absolute, + }; + reporter.enqueue({ __proto__: null, ...fileNode }); + reporter.dequeue({ __proto__: null, ...fileNode }); + + const proc = Bun.spawn({ + cmd: args, + cwd: opts.cwd as string, + env: { ...(opts.env ?? process.env), BUN_TEST_DRAIN_EVENT_LOOP: "1", [kRunChildEnv]: kRunChildEnvValue }, + stdout: "pipe", + stderr: "pipe", + signal: opts.signal, + }); + + let drainStderr: Promise | undefined; + try { + let stderrText = ""; + drainStderr = (async () => { + const decoder = new TextDecoder(); + let carry = ""; + for await (const chunk of proc.stderr as any) { + const text = typeof chunk === "string" ? chunk : decoder.decode(chunk, { stream: true }); + stderrText += text; + carry += text; + let nl; + while ((nl = carry.indexOf("\n")) !== -1) { + const line = carry.slice(0, nl); + carry = carry.slice(nl + 1); + if (line.length > 0) reporter.stderr({ __proto__: null, file: absolute, message: line + "\n" }); + } + } + if (carry.length > 0) reporter.stderr({ __proto__: null, file: absolute, message: carry + "\n" }); + })(); + // Defuse: a throwing test:stderr listener rejects this while the stdout + // loop is still suspended on I/O, before the finally's .catch attaches. + drainStderr.catch(() => {}); + + const handleStdoutLine = (line: string) => { + if (line.length === 0) return; + // bun:test's own reporter can leave an unterminated line, so the marker is + // not always at column 0; take everything from the marker on. + const marker = line.indexOf(kRunEventPrefix); + if (marker === -1) { + reporter.stdout({ __proto__: null, file: absolute, message: line + "\n" }); + return; + } + if (marker > 0) { + const before = line.slice(0, marker).trimEnd(); + if (before.length > 0) { + reporter.stdout({ __proto__: null, file: absolute, message: before + "\n" }); + } + } + let event; + try { + event = JSON.parse(line.slice(marker + kRunEventPrefix.length)); + } catch { + return; + } + if (event == null || typeof event.data !== "object" || event.data === null) return; + republishChildEvent(event, absolute, reporter, fileCounts); + }; + + const decoder = new TextDecoder(); + let carry = ""; + for await (const chunk of proc.stdout as any) { + carry += typeof chunk === "string" ? chunk : decoder.decode(chunk, { stream: true }); + let nl; + while ((nl = carry.indexOf("\n")) !== -1) { + const line = carry.slice(0, nl); + carry = carry.slice(nl + 1); + handleStdoutLine(line); + } + } + if (carry.length > 0) handleStdoutLine(carry); + + await drainStderr; + const exitCode = await proc.exited; + + // A nonzero exit with no child-reported failures means the file itself died + // (top-level throw); child-reported failures are already covered by the + // republished events and need no file-level verdict. + const fileFailed = exitCode !== 0 && fileCounts.failed === 0; + const fileDuration = Date.now() - fileStarted; + // Node's FileTest.#skipReporting(): no file-level complete/pass/fail when + // the child reported at least one test and the only error is subtestsFailed + // (or none); here that is `reportedChildren > 0 && !fileFailed`. + const reportedChildren = fileCounts.tests + fileCounts.suites; + let error: Error | undefined; + + // Count the file-node before emitting the per-file summary so a synchronous + // test:summary listener sees the same totals the run-level summary will. + fileCounts.topLevel++; + const reportFileNode = reportedChildren === 0 || fileFailed; + if (reportFileNode) { + fileCounts.tests++; + if (fileFailed) fileCounts.failed++; + else fileCounts.passed++; + } + + if (fileFailed) { + error = makeTestFailure(stderrText.trim() || `Test file failed with exit code ${exitCode}`, "testCodeFailure"); + } else { + reporter.summary({ + __proto__: null, + success: fileCounts.failed === 0, + counts: { __proto__: null, ...fileCounts }, + duration_ms: fileDuration, + file: absolute, + }); + } + + if (reportFileNode) { + reporter.complete({ + __proto__: null, + ...fileNode, + type: undefined, + testNumber: 1, + details: { + __proto__: null, + duration_ms: fileDuration, + type: "test", + passed: !fileFailed, + error, + }, + }); + const details = { __proto__: null, duration_ms: fileDuration, type: "test", error }; + if (fileFailed) { + reporter.fail({ __proto__: null, ...fileNode, type: undefined, testNumber: 1, details }); + } else { + reporter.pass({ __proto__: null, ...fileNode, type: undefined, testNumber: 1, details }); + } + } + addRunCounts(counts, fileCounts); + } finally { + proc.kill(); + if (drainStderr !== undefined) await drainStderr.catch(() => {}); + } +} + +function rebuildError(serialized: any, depth = 0): Error { + const { message, stack, name, code, failureType, cause } = serialized; + const error = new Error(message); + error.stack = stack; + if (name !== undefined && name !== "Error") error.name = name; + if (code !== undefined) (error as any).code = code; + if (failureType !== undefined) (error as any).failureType = failureType; + if (cause !== undefined && depth < 8) (error as any).cause = rebuildError(cause, depth + 1); + return error; +} + +function republishChildEvent( + event: { type: string; data: any }, + file: string, + reporter: TestsStream, + counts: Record, +) { + const { type, data } = event; + Object.setPrototypeOf(data, null); + data.file = file; + data.nesting = (data.nesting ?? 0) + 1; + if (type === "test:pass" || type === "test:fail") { + const isSuite = data.type === "suite"; + // node counts a suite in `suites` and stops there: a skipped or todo suite + // never lands in skipped/todo/passed/tests (countCompletedTest, test.js). + if (isSuite) counts.suites++; + else { + counts.tests++; + if (data.skip) counts.skipped++; + else if (data.todo) counts.todo++; + else if (type === "test:pass") counts.passed++; + else counts.failed++; + } + // node carries the node kind on `details`, not on the event itself. + const detailType = isSuite ? "suite" : "test"; + const serialized = data.error; + if (serialized !== undefined) { + data.details = { + __proto__: null, + duration_ms: data.duration_ms, + type: detailType, + error: rebuildError(serialized), + }; + } else { + data.details = { __proto__: null, duration_ms: data.duration_ms, type: detailType }; + } + delete data.error; + delete data.duration_ms; + delete data.type; + if (type === "test:pass") reporter.pass(data); + else reporter.fail(data); + return; + } + reporter.republish(type, data); +} + +// Child side: with kRunChildEnv set, stream one JSON event per line so the +// spawning parent can rebuild node's event stream. +const runChildReporterEnabled = process.env[kRunChildEnv] !== undefined; + +function emitRunChildEvent(type: string, data: unknown) { + try { + process.stdout.write(kRunEventPrefix + JSON.stringify({ type, data }) + "\n"); + } catch {} +} + +// node's top-level tests are nesting 0, so the root node itself doesn't count. +function nestingOf(node: TestNode) { + let depth = 0; + for (let cur = node.parent; cur !== undefined && cur.parent !== undefined; cur = cur.parent) depth++; + return depth; +} + +// Errors cross the process boundary as plain JSON; the parent rebuilds an Error. +function serializeRunError(error: unknown, depth = 0) { + if (Error.isError(error)) { + const cause = (error as { cause?: unknown }).cause; + return { + __proto__: null, + message: (error as Error).message, + stack: (error as Error).stack, + code: (error as { code?: string }).code, + failureType: (error as { failureType?: string }).failureType, + name: (error as Error).name, + cause: cause !== undefined && depth < 8 ? serializeRunError(cause, depth + 1) : undefined, + }; + } + return { __proto__: null, message: String(error), stack: undefined, code: undefined, name: "Error" }; +} + +// A test or suite that bun:test will never invoke (the `skip` and `todo` +// options). Node still reports it as a pass carrying the directive. +function reportDirectiveOnlyNode(node: TestNode, mode: "skip" | "todo") { + if (!runChildReporterEnabled) return; + // `{ skip: true, todo: true }` reports as a skip: node checks `skipped` + // first and only then `isTodo` (test.js getReportDetails). + const skipped = node.skipped || mode === "skip"; + emitRunChildEvent("test:pass", { + __proto__: null, + name: node.name, + nesting: nestingOf(node), + testNumber: 0, + duration_ms: 0, + skip: skipped ? (node.message ?? true) : undefined, + todo: !skipped ? (node.message ?? true) : undefined, + type: node.isSuite ? "suite" : "test", + tags: node.tags, + error: undefined, + }); +} + +// Called for every test node as its result is finalized, so subtests report +// with the same shape as top-level tests. No-op outside a run() child. +function reportNodeToRunParent(node: TestNode, startedAt: number) { + if (!runChildReporterEnabled || node.isSuite) return; + const { skipped, todoFlag, expectFailure } = node; + // node reports the xfail label when there is one, otherwise `true`. + const xfail = !skipped && expectFailure ? (expectFailure.label ?? true) : undefined; + // node spreads a `directive` into the event: `skip: true` / `todo: true`, with + // the other key absent entirely. + emitRunChildEvent(node.passed ? "test:pass" : "test:fail", { + __proto__: null, + name: node.name, + nesting: nestingOf(node), + testNumber: 0, + duration_ms: performance.now() - startedAt, + skip: skipped ? (node.message ?? true) : undefined, + todo: !skipped && todoFlag ? (node.message ?? true) : undefined, + expectFailure: xfail, + tags: node.tags, + error: node.passed ? undefined : serializeRunError(node.error), + }); } // ----------------------------------------------------------------------------- @@ -676,9 +1379,10 @@ function buildContextAssert(node: TestNode, ctx: TestContext) { // Test plan // ----------------------------------------------------------------------------- -function makeTestFailure(message: string) { +function makeTestFailure(message: string, failureType?: string) { const error = new Error(message); (error as { code?: string }).code = "ERR_TEST_FAILURE"; + if (failureType !== undefined) (error as { failureType?: string }).failureType = failureType; return error; } @@ -761,7 +1465,6 @@ function planCount(node: TestNode) { // ----------------------------------------------------------------------------- const kEmptyTags: string[] = Object.freeze([]) as string[]; -let tagsExperimentalWarningEmitted = false; function canonicalizeTags(tags: unknown, name: string): string[] { validateArray(tags, name); @@ -774,10 +1477,7 @@ function canonicalizeTags(tags: unknown, name: string): string[] { } seen.add((tag as string).toLowerCase()); } - if (seen.size > 0 && !tagsExperimentalWarningEmitted) { - tagsExperimentalWarningEmitted = true; - process.emitWarning("Test tags is an experimental feature and might change at any time", "ExperimentalWarning"); - } + if (seen.size > 0) emitTagsExperimentalWarning(); return Array.from(seen); } @@ -835,6 +1535,8 @@ class TestNode { mockTracker: MockTracker | null = null; skipped = false; todoFlag = false; + message: string | undefined = undefined; + expectFailure: ExpectFailure = false; started = false; finished = false; passed = false; @@ -866,8 +1568,12 @@ class TestNode { // (under `bun test` with multiple files, Bun.main is the file currently // being collected); nested tests inherit their parent's file. this.filePath = parent !== undefined && parent.parent !== undefined ? parent.filePath : Bun.main; - this.skipped = !!options.skip; - this.todoFlag = !!options.todo; + const { skip, todo } = options; + this.skipped = !!skip; + this.todoFlag = !!todo || (parent?.todoFlag ?? false); + if (typeof skip === "string") this.message = skip; + else if (typeof todo === "string") this.message = todo; + this.expectFailure = parseExpectFailure(options.expectFailure) || parent?.expectFailure || false; } get tags(): string[] { @@ -1033,12 +1739,14 @@ class TestContext { throwNotImplemented("runOnly()", 5090, "Use `bun:test` in the interim."); } - skip(_message?: string) { + skip(message?: string) { this.#node.skipped = true; + if (typeof message === "string") this.#node.message = message; } - todo(_message?: string) { + todo(message?: string) { this.#node.todoFlag = true; + if (typeof message === "string") this.#node.message = message; } before(arg0: unknown, arg1: unknown) { @@ -1174,6 +1882,7 @@ type TestOptions = { timeout?: number; plan?: number; tags?: string[]; + expectFailure?: unknown; }; type HookOptions = { @@ -1239,6 +1948,73 @@ function validateTimeoutAndSignal(options: TestOptions | HookOptions) { } } +// Port of Node's parseExpectFailure (test.js:528). A string is a label, a +// function or RegExp validates the error, an object may carry both, and any +// other object is itself the validation. +type ExpectFailure = false | { label?: string; match?: unknown }; + +function parseExpectFailure(expectFailure: unknown): ExpectFailure { + if (expectFailure === undefined || expectFailure === false) return false; + if (typeof expectFailure === "string") return { __proto__: null, label: expectFailure, match: undefined } as any; + if (typeof expectFailure === "function" || $isRegExpObject(expectFailure)) { + return { __proto__: null, label: undefined, match: expectFailure } as any; + } + if (typeof expectFailure !== "object") { + return { __proto__: null, label: undefined, match: undefined } as any; + } + // `null` reaches Object.keys and throws, exactly as it does in node. + const keys = Object.keys(expectFailure as object); + if (keys.length === 0) { + throw $ERR_INVALID_ARG_VALUE("options.expectFailure", expectFailure, "must not be an empty object"); + } + if (keys.every(k => k === "match" || k === "label")) { + return { + __proto__: null, + label: (expectFailure as { label?: string }).label, + match: (expectFailure as { match?: unknown }).match, + } as any; + } + return { __proto__: null, label: undefined, match: expectFailure } as any; +} + +// Node inverts the verdict of an expectFailure test: a failure is the expected +// outcome, and passing is itself a failure (test.js:1120-1184). +function applyExpectFailure(node: TestNode, failure: unknown): unknown { + const expectation = node.expectFailure; + if (!expectation) return failure; + + if (failure !== undefined) { + const validation = expectation.match; + if (validation !== undefined) { + // Only a wrapped test-code failure has an inner cause to validate; a bare + // ERR_TEST_FAILURE (a timeout, a plan mismatch) has none and is itself + // the error to check. + const wrapped = failure as { code?: string; failureType?: string; cause?: unknown }; + const unwrap = + wrapped?.code === "ERR_TEST_FAILURE" && + wrapped.failureType === "testCodeFailure" && + wrapped.cause !== undefined; + const errorToCheck = unwrap ? wrapped.cause : failure; + try { + nodeAssert.throws(() => { + throw errorToCheck; + }, validation); + } catch (e) { + const error = makeTestFailure( + "The test failed, but the error did not match the expected validation", + "testCodeFailure", + ); + (error as { cause?: unknown }).cause = e; + return error; + } + } + return undefined; + } + + if (node.skipped) return undefined; + return makeTestFailure("test was expected to fail but passed", "expectedFailure"); +} + function validateTestOptions(options: TestOptions): { ownTags: string[] | undefined } { const { concurrency, tags, plan } = options; @@ -1482,6 +2258,7 @@ async function executeTestNode(node: TestNode, fn: TestFn): Promise { // body, pending subtests, the plan check, inherited afterEach hooks, and the // test's own after hooks. Returns the failure (if any) instead of throwing. node.started = true; + const started = runChildReporterEnabled ? performance.now() : 0; const ctx = node.getCtx(); const ancestors = ancestorChain(node); let failure: unknown; @@ -1554,7 +2331,10 @@ async function executeTestNode(node: TestNode, fn: TestFn): Promise { const { failedSubtests, firstSubtestError } = node; if (failure === undefined && failedSubtests > 0) { - const error = makeTestFailure(`${failedSubtests} subtest${failedSubtests > 1 ? "s" : ""} failed`); + const error = makeTestFailure( + `${failedSubtests} subtest${failedSubtests > 1 ? "s" : ""} failed`, + "subtestsFailed", + ); if (firstSubtestError !== undefined) { (error as { cause?: unknown }).cause = firstSubtestError; } @@ -1562,11 +2342,15 @@ async function executeTestNode(node: TestNode, fn: TestFn): Promise { } } + const bodyFailure = failure; + failure = applyExpectFailure(node, failure); + const acceptedXfail = bodyFailure !== undefined && failure === undefined; + // Node sets passed/error before running afterEach/after so hooks can // introspect the outcome (nodejs/node lib/internal/test_runner/test.js // pass()/fail() precede afterEach). node.passed = failure === undefined; - node.error = failure ?? null; + node.error = failure ?? (acceptedXfail ? bodyFailure : null); // Mark finished before hooks so a late t.test() from an after/afterEach // hook hits addTest()'s parentAlreadyFinished path (Node cancels these). node.finished = true; @@ -1577,7 +2361,7 @@ async function executeTestNode(node: TestNode, fn: TestFn): Promise { try { await runHook(hook, ancestor, ctx); } catch (err) { - failure ??= err; + if (!acceptedXfail) failure ??= err; } } } @@ -1586,22 +2370,23 @@ async function executeTestNode(node: TestNode, fn: TestFn): Promise { try { await runHook(hook, node, ctx); } catch (err) { - failure ??= err; + if (!acceptedXfail) failure ??= err; } } try { node.mockTracker?.reset(); } catch (err) { - failure ??= err; + if (!acceptedXfail) failure ??= err; } node.passed = failure === undefined; - node.error = failure ?? null; + node.error = failure ?? (acceptedXfail ? bodyFailure : null); + reportNodeToRunParent(node, started); return failure; } -function scheduleSubtest(parent: TestNode, child: TestNode, fn: TestFn): Promise { +function scheduleSubtest(parent: TestNode, child: TestNode, fn: TestFn, ownTodo: boolean): Promise { const run = async () => { if (child.options.skip) { child.finished = true; @@ -1615,7 +2400,10 @@ function scheduleSubtest(parent: TestNode, child: TestNode, fn: TestFn): Promise } catch (err) { failure = err; } - if (failure !== undefined && !child.todoFlag && !child.skipped) { + // Check the child's own todo declaration (options.todo or test.todo(...)), + // not the inherited todoFlag: a subtest that threw must still fail a + // {todo:true} parent so bun:test under --todo reports Todo (failure rolls up). + if (failure !== undefined && !ownTodo && !child.skipped) { parent.failedSubtests++; parent.firstSubtestError ??= failure; } @@ -1642,7 +2430,7 @@ async function drainSubtestChain(node: TestNode) { } while (chain !== node.subtestChain); } -function scheduleSuiteSubtest(parent: TestNode, suite: TestNode, build: unknown): Promise { +function scheduleSuiteSubtest(parent: TestNode, suite: TestNode, build: unknown, ownTodo: boolean): Promise { // A describe()/suite() created while a test is running becomes a suite // subtest: its children were collected eagerly when the callback ran and are // already chained on the suite's own subtestChain; failures roll up here. @@ -1673,8 +2461,28 @@ function scheduleSuiteSubtest(parent: TestNode, suite: TestNode, build: unknown) } suite.finished = true; suite.passed = suite.failedSubtests === 0; + if (runChildReporterEnabled) { + emitRunChildEvent(suite.passed ? "test:pass" : "test:fail", { + __proto__: null, + name: suite.name, + nesting: nestingOf(suite), + testNumber: 0, + duration_ms: 0, + type: "suite", + tags: suite.tags, + todo: suite.todoFlag ? (suite.message ?? true) : undefined, + error: suite.passed + ? undefined + : serializeRunError( + makeTestFailure( + `${suite.failedSubtests} subtest${suite.failedSubtests > 1 ? "s" : ""} failed`, + "subtestsFailed", + ), + ), + }); + } // A todo suite's failures do not fail the owning test (Node). - if (suite.failedSubtests > 0 && !suite.todoFlag) { + if (suite.failedSubtests > 0 && !ownTodo) { parent.failedSubtests++; parent.firstSubtestError ??= suite.firstSubtestError; } @@ -1721,6 +2529,11 @@ function createTopLevelTestRunner(node: TestNode, fn: TestFn, declaredTodo = fal // bun:test invokes this with a `done` callback because the function declares // one parameter. return (done: (error?: unknown) => void) => { + // Under plain bun:test a describe.todo scope already handles its children's + // todo verdict (FailBecauseTodoPassed under --todo), so don't override when + // the flag was only inherited; under a run() child the suite registers as a + // plain describe so bun:test has no todo scope to consult. + const todoBefore = node.todoFlag; executeTestNode(node, fn).then( failure => { // A runtime t.skip()/t.todo() overrides bun:test's pass/fail accounting @@ -1728,7 +2541,7 @@ function createTopLevelTestRunner(node: TestNode, fn: TestFn, declaredTodo = fal // todo body's failure must reach bun:test's own todo accounting instead. if (node.skipped) { markCurrentResult(false, done); - } else if (node.todoFlag && !declaredTodo) { + } else if (node.todoFlag && !declaredTodo && (runChildReporterEnabled || !todoBefore)) { markCurrentResult(true, done); } else { done(failure); @@ -1760,13 +2573,18 @@ function addTest( } if (runningNode.isRunning()) { // Subtest of a running test (or of an inline suite created inside one). - if (mode === "skip" || options.skip) { - return Promise.resolve(undefined); - } const child = new TestNode(name, runningNode, options, false, true); child.ownTags = ownTags; - if (mode === "todo") child.todoFlag = true; - return scheduleSubtest(runningNode, child, fn); + if (mode === "skip" || options.skip) { + // Chain onto subtestChain so the directive lands after earlier siblings. + const chained = (runningNode.subtestChain = runningNode.subtestChain.then(() => + reportDirectiveOnlyNode(child, "skip"), + )); + return chained.then(() => undefined); + } + const ownTodo = mode === "todo" || !!options.todo; + if (ownTodo) child.todoFlag = true; + return scheduleSubtest(runningNode, child, fn, ownTodo); } } @@ -1778,9 +2596,36 @@ function addTest( const { test } = bunTest(); const passOptions = bunTestOptions(options); - const effectiveMode = mode ?? (options.todo ? "todo" : options.skip ? "skip" : undefined); + // Node merges .todo()/.skip() into the options and checks skip first, so + // test.todo(name, { skip: true }, fn) is a skip. + const effectiveMode = mode === "skip" || options.skip ? "skip" : mode === "todo" || options.todo ? "todo" : undefined; if (effectiveMode === "todo" || effectiveMode === "skip") { + // Under a run() child, register skip as an ordinary test so its directive + // event fires in execution order (not at collection time). + if (runChildReporterEnabled && effectiveMode === "skip") { + const runner = function (done: (err?: unknown) => void) { + reportDirectiveOnlyNode(node, "skip"); + markCurrentResult(false, done); + done(undefined); + }; + if (passOptions !== undefined) test(name, runner, passOptions); + else test(name, runner); + return Promise.resolve(undefined); + } + // Node runs a todo body, so `t.skip()` inside one still changes the + // directive it reports. bun:test only runs todo bodies under --todo, so a + // run() child registers them as ordinary tests and marks the result at the + // end (what createTopLevelTestRunner already does for a runtime t.todo()). + if (runChildReporterEnabled && effectiveMode === "todo") { + // The test.todo() spelling carries the directive in `mode`, not in the + // options, so the node has to be marked for the runner to report it. + node.todoFlag = true; + const runner = createTopLevelTestRunner(node, fn); + if (passOptions !== undefined) test(name, runner, passOptions); + else test(name, runner); + return Promise.resolve(undefined); + } const register = effectiveMode === "todo" ? test.todo : test.skip; // Node runs todo bodies; bun:test only does so under --todo. const body = effectiveMode === "todo" ? createTopLevelTestRunner(node, fn, true) : kDefaultFunction; @@ -1827,9 +2672,14 @@ function addSuite( const suite = new TestNode(name, runningNode, options, true, true); suite.ownTags = ownTags; if (mode === "skip" || options.skip) { - return Promise.resolve(undefined); - } - if (mode === "todo") suite.todoFlag = true; + // Chain onto subtestChain so the directive lands after earlier siblings. + const chained = (runningNode.subtestChain = runningNode.subtestChain.then(() => + reportDirectiveOnlyNode(suite, "skip"), + )); + return chained.then(() => undefined); + } + const ownTodo = mode === "todo" || !!options.todo; + if (ownTodo) suite.todoFlag = true; // The suite's children must run after the parent's previously scheduled // subtests AND after the describe callback's own returned promise settles // (Node's Suite.run awaits buildPromise before iterating subtests). The @@ -1855,7 +2705,7 @@ function addSuite( gate.resolve(); build = undefined; } - return scheduleSuiteSubtest(runningNode, suite, build); + return scheduleSuiteSubtest(runningNode, suite, build, ownTodo); } const parent = currentCollectionParent(); @@ -1863,16 +2713,29 @@ function addSuite( suiteNode.ownTags = ownTags; const { describe } = bunTest(); - const wrapped = () => { - return runWithNode(suiteNode, () => fn(suiteNode.getSuiteCtx())); - }; - const effectiveMode = mode ?? (options.todo ? "todo" : options.skip ? "skip" : undefined); + // Node merges .todo()/.skip() into the options and checks skip first, so + // describe.todo(name, { skip: true }, fn) is a skip. + const effectiveMode = mode === "skip" || options.skip ? "skip" : mode === "todo" || options.todo ? "todo" : undefined; + + // Node never invokes a skipped suite's callback (it does run a todo one), so + // the children are never declared and side effects in the body never happen. + const wrapped = + effectiveMode === "skip" + ? kDefaultFunction + : () => { + return runWithNode(suiteNode, () => fn(suiteNode.getSuiteCtx())); + }; + const passOptions = bunTestOptions(options); let register: Function = describe; if (effectiveMode === "skip") register = describe.skip; - else if (effectiveMode === "todo") register = describe.todo; + else if (effectiveMode === "todo") { + suiteNode.todoFlag = true; + register = runChildReporterEnabled ? describe : describe.todo; + } + if (effectiveMode !== undefined) reportDirectiveOnlyNode(suiteNode, effectiveMode); if (passOptions !== undefined) { register(name, wrapped, passOptions); diff --git a/src/runtime/cli/test_command.rs b/src/runtime/cli/test_command.rs index bf7c64deb186..c3d0481ee43a 100644 --- a/src/runtime/cli/test_command.rs +++ b/src/runtime/cli/test_command.rs @@ -900,6 +900,14 @@ impl JunitReporter { } } +/// Drain the event loop after a file's tests finish, like a node process +/// would before exiting; the vendored-node-test runner opts in via +/// `BUN_TEST_DRAIN_EVENT_LOOP=1` so mustCall()-style exit checks see +/// completed async work. Off by default: bun suites keep exit-after-tests. +pub(crate) fn should_drain_event_loop() -> bool { + env_var::BUN_TEST_DRAIN_EVENT_LOOP.get().unwrap_or(false) +} + pub struct CommandLineReporter { // `TestRunner<'a>` borrows `TestOptions`/regex from the CLI ctx; the // reporter is held in a `Box` local to `TestCommand::exec` which never @@ -3045,7 +3053,18 @@ impl TestCommand { { vm.exit_handler.exit_code = 1; } - vm.is_shutting_down = true; + // Run `process.on('exit')` handlers like `bun run` does. Node's test + // harness verifies mustCall() counts from one, so skipping them made + // those assertions silently pass. Must precede the GC-root release + // below: handlers are user JS and may touch still-live state. + { + let vm_ptr: *mut VirtualMachine = vm; + // SAFETY: `vm_ptr` reborrows the live `&mut VirtualMachine`; + // `run_with_api_lock` takes `&self` only, so the closure holds the + // unique mutable access on this single-threaded path. + vm.run_with_api_lock(|| unsafe { (*vm_ptr).on_exit() }); + } + // on_exit() already set is_shutting_down; global_exit() asserts it. // Release `bun:test` GC roots before `global_exit()` so // `destructOnExit()`'s `collectNow()` can reach the closures they pin // (preload hooks, per-file describe/test callbacks). Clear `RUNNER` @@ -3358,6 +3377,14 @@ impl TestCommand { let el = vm.event_loop(); // SAFETY: el is the VM-owned event loop; vm is passed back as *mut. unsafe { (*el).tick_immediate_tasks(vm) }; + + // Node parity: a node test file exits only when its loop drains. + // on_before_exit() drains and dispatches 'beforeExit' like `bun run`; + // it early-returns when unhandled_error_counter > 0, which is fine + // here since such a file already failed. Opt-in; one file per process. + if should_drain_event_loop() { + vm.on_before_exit(); + } drop(buntest_strong); } diff --git a/test/cli/test/bun-test.test.ts b/test/cli/test/bun-test.test.ts index f720630d88c4..292564167e2f 100644 --- a/test/cli/test/bun-test.test.ts +++ b/test/cli/test/bun-test.test.ts @@ -1456,6 +1456,51 @@ describe("bun test", () => { expect(output).toContain("1 pass"); expect(output).toContain("app message"); }); + + test("runs process.on('exit') handlers", async () => { + using dir = tempDir("bun-test-exit-handler", { + "exit.test.ts": ` + import { test } from "bun:test"; + process.on("exit", () => console.log("exit handler ran")); + test("a test", () => {}); + `, + }); + + await using proc = Bun.spawn({ + cmd: [bunExe(), "test", "exit.test.ts"], + env: bunEnv, + cwd: String(dir), + stderr: "pipe", + }); + + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + expect(stdout).toContain("exit handler ran"); + expect(stderr).toContain("1 pass"); + expect(exitCode).toBe(0); + }); + + test("an exit handler can fail the run, like node's common.mustCall()", async () => { + using dir = tempDir("bun-test-exit-handler-code", { + "exit-code.test.ts": ` + import { test } from "bun:test"; + process.on("exit", () => process.exit(1)); + test("a passing test", () => {}); + `, + }); + + await using proc = Bun.spawn({ + cmd: [bunExe(), "test", "exit-code.test.ts"], + env: bunEnv, + cwd: String(dir), + stderr: "pipe", + }); + + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + // Windows prints the banner to stdout; only assert nothing test-shaped leaks. + expect(stdout).not.toContain("pass"); + expect(stderr).toContain("1 pass"); + expect(exitCode).toBe(1); + }); }); function createTest(input?: string | (string | { filename: string; contents: string })[], filename?: string): string { diff --git a/test/js/node/test/.gitignore b/test/js/node/test/.gitignore index d758bbb62cae..d8c99a70d948 100644 --- a/test/js/node/test/.gitignore +++ b/test/js/node/test/.gitignore @@ -1,7 +1,9 @@ fixtures/wpt fixtures/tools fixtures/v8-coverage -fixtures/test-runner +fixtures/test-runner/* +!fixtures/test-runner/index.js +!fixtures/test-runner/tagged.js fixtures/source-map fixtures/snapshot fixtures/repl* diff --git a/test/js/node/test/fixtures/test-runner/index.js b/test/js/node/test/fixtures/test-runner/index.js new file mode 100644 index 000000000000..fcf4b4d8eaa0 --- /dev/null +++ b/test/js/node/test/fixtures/test-runner/index.js @@ -0,0 +1,2 @@ +'use strict'; +throw new Error('thrown from index.js'); diff --git a/test/js/node/test/fixtures/test-runner/tagged.js b/test/js/node/test/fixtures/test-runner/tagged.js new file mode 100644 index 000000000000..7d0e184ad1dd --- /dev/null +++ b/test/js/node/test/fixtures/test-runner/tagged.js @@ -0,0 +1,22 @@ +'use strict'; +const { describe, it, test } = require('node:test'); + +describe('db suite', { tags: ['db'] }, () => { + it('db only', () => {}); + it('db plus integration', { tags: ['integration'] }, () => {}); + it('db flaky', { tags: ['flaky'] }, () => {}); +}); + +describe('unit suite', { tags: ['unit'] }, () => { + it('unit only', () => {}); + it('unit slow', { tags: ['slow'] }, () => {}); +}); + +test('untagged', () => {}); +test('only flaky', { tags: ['flaky'] }, () => {}); +test('db wildcard match', { tags: ['db:postgres'] }, () => {}); + +describe('plain suite', () => { + it('lonely child', { tags: ['lonely'] }, () => {}); + it('plain sibling', () => {}); +}); diff --git a/test/js/node/test/parallel/test-runner-expect-error-but-pass.js b/test/js/node/test/parallel/test-runner-expect-error-but-pass.js new file mode 100644 index 000000000000..e300b705432e --- /dev/null +++ b/test/js/node/test/parallel/test-runner-expect-error-but-pass.js @@ -0,0 +1,17 @@ +'use strict'; +const common = require('../common'); +const assert = require('node:assert'); +const { run, test } = require('node:test'); + +if (!process.env.NODE_TEST_CONTEXT) { + const stream = run({ files: [__filename] }); + + stream.on('test:pass', common.mustNotCall()); + stream.on('test:fail', common.mustCall((event) => { + assert.strictEqual(event.details.error.code, 'ERR_TEST_FAILURE'); + assert.strictEqual(event.details.error.failureType, 'expectedFailure'); + assert.strictEqual(event.details.error.message, 'test was expected to fail but passed'); + }, 1)); +} else { + test('passing test', { expectFailure: true }, () => {}); +} diff --git a/test/js/node/test/parallel/test-runner-expect-error.js b/test/js/node/test/parallel/test-runner-expect-error.js new file mode 100644 index 000000000000..6185c91df43c --- /dev/null +++ b/test/js/node/test/parallel/test-runner-expect-error.js @@ -0,0 +1,15 @@ +'use strict'; +const common = require('../common'); +const assert = require('node:assert'); +const { run, test } = require('node:test'); + +if (!process.env.NODE_TEST_CONTEXT) { + const stream = run({ files: [__filename] }); + + stream.on('test:fail', common.mustNotCall()); + stream.on('test:pass', common.mustCall((event) => { + assert.strictEqual(event.expectFailure, true); + }, 1)); +} else { + test('failing test', { expectFailure: true }, () => assert.fail('should not pass')); +} diff --git a/test/js/node/test/parallel/test-runner-filetest-location.js b/test/js/node/test/parallel/test-runner-filetest-location.js new file mode 100644 index 000000000000..0b43198a743d --- /dev/null +++ b/test/js/node/test/parallel/test-runner-filetest-location.js @@ -0,0 +1,20 @@ +'use strict'; +const common = require('../common'); +const fixtures = require('../common/fixtures'); +const assert = require('node:assert'); +const { relative } = require('node:path'); +const { run } = require('node:test'); +const fixture = fixtures.path('test-runner', 'index.js'); +const relativePath = relative(process.cwd(), fixture); +const stream = run({ + files: [relativePath], + timeout: common.platformTimeout(100), +}); + +stream.on('test:fail', common.mustCall((result) => { + assert.strictEqual(result.name, relativePath); + assert.strictEqual(result.details.error.failureType, 'testCodeFailure'); + assert.strictEqual(result.line, 1); + assert.strictEqual(result.column, 1); + assert.strictEqual(result.file, fixture); +})); diff --git a/test/js/node/test/parallel/test-runner-tags-experimental-warning.mjs b/test/js/node/test/parallel/test-runner-tags-experimental-warning.mjs new file mode 100644 index 000000000000..522cc781ed3e --- /dev/null +++ b/test/js/node/test/parallel/test-runner-tags-experimental-warning.mjs @@ -0,0 +1,97 @@ +import { expectWarning } from '../common/index.mjs'; +import * as fixtures from '../common/fixtures.mjs'; +import assert from 'node:assert'; +import { spawnSync } from 'node:child_process'; +import { it, test } from 'node:test'; + +const fixture = fixtures.path('test-runner', 'tagged.js'); + +function runChild(args, opts = {}) { + // Strip NODE_TEST_CONTEXT so a child run() works as a top-level invocation. + const env = { ...process.env, ...(opts.env || {}) }; + delete env.NODE_TEST_CONTEXT; + delete env.NODE_TEST_WORKER_ID; + const child = spawnSync(process.execPath, args, { + stdio: ['ignore', 'pipe', 'pipe'], + env, + }); + return { + status: child.status, + stdout: child.stdout.toString(), + stderr: child.stderr.toString(), + }; +} + +function countWarnings(stderr) { + let count = 0; + let idx = 0; + while (true) { + const next = stderr.indexOf('ExperimentalWarning: Test tags', idx); + if (next === -1) break; + count++; + idx = next + 1; + } + return count; +} + +// In-process: when tags are registered with no other trigger, the warning +// fires exactly once for the lifetime of the process. +expectWarning('ExperimentalWarning', 'Test tags is an experimental feature and might change at any time'); + +test('the warning fires once per process for tag registration', () => { + it('first', { tags: ['db'] }, () => {}); + it('second', { tags: ['unit'] }, () => {}); + it('third', { tags: ['integration'] }, () => {}); + // expectWarning's mustCall(_, 1) will fail if the warning is emitted a + // second time for any of the registrations above. +}); + +test('--experimental-test-tag-filter fires the warning', () => { + const { stderr } = runChild([ + '--test', + '--test-reporter=tap', + '--experimental-test-tag-filter=db', + '--test-isolation=none', + fixture, + ]); + assert.strictEqual(countWarnings(stderr), 1, stderr); +}); + +test('programmatic testTagFilters fires the warning', () => { + const driver = ` + 'use strict'; + const { run } = require('node:test'); + run({ + files: [${JSON.stringify(fixture)}], + isolation: 'none', + testTagFilters: ['db'], + }).resume(); + `; + const { stderr } = runChild(['-e', driver]); + assert.strictEqual(countWarnings(stderr), 1, stderr); +}); + +test('tags: [] does not fire the warning', () => { + const driver = ` + 'use strict'; + const { it } = require('node:test'); + it('a', { tags: [] }, () => {}); + it('b', { tags: [] }, () => {}); + it('c', () => {}); + `; + const { stderr } = runChild(['-e', driver]); + assert.strictEqual(countWarnings(stderr), 0, stderr); +}); + +test('multiple triggers in one process emit the warning once', () => { + const driver = ` + 'use strict'; + const { it, run } = require('node:test'); + // Trigger 1: tag registration. + it('a', { tags: ['db'] }, () => {}); + // Trigger 2: programmatic testTagFilters. + run({ files: [${JSON.stringify(fixture)}], isolation: 'none', testTagFilters: ['db'] }).resume(); + `; + const { stderr } = runChild(['-e', driver]); + assert.strictEqual(countWarnings(stderr), 1, stderr); +}); diff --git a/test/js/node/test/parallel/test-runner-tags-validation.mjs b/test/js/node/test/parallel/test-runner-tags-validation.mjs new file mode 100644 index 000000000000..4e0249479627 --- /dev/null +++ b/test/js/node/test/parallel/test-runner-tags-validation.mjs @@ -0,0 +1,117 @@ +import { expectWarning } from '../common/index.mjs'; +import assert from 'node:assert'; +import { test, suite, describe, it, before, beforeEach, after, afterEach, run } from 'node:test'; + +// Silence the one-shot ExperimentalWarning that fires the first time tags are +// registered. Validation tests focus on the throw, not the warning. +expectWarning('ExperimentalWarning', 'Test tags is an experimental feature and might change at any time'); + +test('non-array tags throws ERR_INVALID_ARG_TYPE', () => { + for (const bad of ['db', 42, {}, true, Symbol('s'), null]) { + assert.throws( + () => test('x', { tags: bad }, () => {}), + { code: 'ERR_INVALID_ARG_TYPE' }, + `expected throw for tags=${String(bad)}`, + ); + } +}); + +test('non-string element throws ERR_INVALID_ARG_TYPE', () => { + for (const bad of [42, {}, true, null, undefined, Symbol('s')]) { + assert.throws( + () => test('x', { tags: ['db', bad] }, () => {}), + { code: 'ERR_INVALID_ARG_TYPE' }, + `expected throw for element=${String(bad)}`, + ); + } +}); + +test('empty-string tag throws ERR_INVALID_ARG_VALUE', () => { + assert.throws( + () => test('x', { tags: [''] }, () => {}), + { code: 'ERR_INVALID_ARG_VALUE' }, + ); + assert.throws( + () => test('x', { tags: ['db', ''] }, () => {}), + { code: 'ERR_INVALID_ARG_VALUE' }, + ); +}); + +test('Unicode and most punctuation are allowed', () => { + // None of these should throw. + test('unicode-1', { tags: ['café'] }, () => {}); + test('unicode-2', { tags: ['日本語'] }, () => {}); + test('punct-1', { tags: ['db:postgres'] }, () => {}); + test('punct-2', { tags: ['db-1'] }, () => {}); + test('punct-3', { tags: ['db.fast'] }, () => {}); + test('punct-4', { tags: ['db_fast'] }, () => {}); + test('punct-5', { tags: ['#critical'] }, () => {}); +}); + +test('tags: [] is a no-op (does not throw)', () => { + test('empty-tags', { tags: [] }, () => {}); +}); + +test('case dedup: ["db", "DB", "Db"] collapses to single canonical entry', async (t) => { + await t.test('child', { tags: ['db', 'DB', 'Db'] }, (ct) => { + assert.deepStrictEqual(ct.tags, ['db']); + }); +}); + +test('declaration order is preserved on dedup', async (t) => { + await t.test('child', { tags: ['Z', 'a', 'z', 'A'] }, (ct) => { + assert.deepStrictEqual(ct.tags, ['z', 'a']); + }); +}); + +test('hooks silently ignore the tags option', () => { + // Hooks must not throw if a caller mistakenly passes tags — the API has no + // tags concept for hooks, but it should be tolerated. The validator only + // runs on Test/Suite construction, not in the hook factory. + before(() => {}, { tags: ['db'] }); + after(() => {}, { tags: ['db'] }); + beforeEach(() => {}, { tags: ['db'] }); + afterEach(() => {}, { tags: ['db'] }); +}); + +test('suite() and describe() validate tags identically', () => { + assert.throws( + () => suite('s', { tags: 'db' }, () => {}), + { code: 'ERR_INVALID_ARG_TYPE' }, + ); + assert.throws( + () => describe('s', { tags: [''] }, () => {}), + { code: 'ERR_INVALID_ARG_VALUE' }, + ); +}); + +test('it() validates tags identically to test()', () => { + assert.throws( + () => it('i', { tags: [42] }, () => {}), + { code: 'ERR_INVALID_ARG_TYPE' }, + ); +}); + +test('run() rejects non-string testTagFilters values', () => { + for (const bad of [[42], [{}], [null], [undefined], [true], [Symbol('s')]]) { + assert.throws( + () => run({ testTagFilters: bad }), + { code: 'ERR_INVALID_ARG_TYPE' }, + `expected throw for testTagFilters=${JSON.stringify(bad)}`, + ); + } +}); + +test('run() accepts a bare-string testTagFilters and normalizes it', async () => { + // The string form is converted to a single-element array internally and + // must not throw. + const stream = run({ files: [], testTagFilters: 'db' }); + // eslint-disable-next-line no-unused-vars + for await (const _ of stream); +}); + +test('run() treats an empty testTagFilters array as a no-op', async () => { + const stream = run({ files: [], testTagFilters: [] }); + // eslint-disable-next-line no-unused-vars + for await (const _ of stream); +}); diff --git a/test/js/node/test/parallel/test-runner-todo-skip-tests.js b/test/js/node/test/parallel/test-runner-todo-skip-tests.js new file mode 100644 index 000000000000..3cabe55723b5 --- /dev/null +++ b/test/js/node/test/parallel/test-runner-todo-skip-tests.js @@ -0,0 +1,32 @@ +'use strict'; +const common = require('../common'); +const assert = require('node:assert'); +const { run, suite, test } = require('node:test'); + +if (!process.env.NODE_TEST_CONTEXT) { + const stream = run({ files: [__filename] }); + + stream.on('test:fail', common.mustNotCall()); + stream.on('test:pass', common.mustCall((event) => { + assert.strictEqual(event.skip, true); + assert.strictEqual(event.todo, undefined); + }, 4)); +} else { + test('test options only', { skip: true, todo: true }, common.mustNotCall()); + + test('test context calls only', common.mustCall((t) => { + t.todo(); + t.skip(); + })); + + test('todo test with context skip', { todo: true }, common.mustCall((t) => { + t.skip(); + })); + + // Note - there is no test for the skip option and t.todo() because the skip + // option prevents the test from running at all. This is verified by other + // tests. + + // Suites don't have the context methods, so only test the options combination. + suite('suite options only', { skip: true, todo: true }, common.mustNotCall()); +} diff --git a/test/js/node/test_runner/fixtures/25-expect-failure.js b/test/js/node/test_runner/fixtures/25-expect-failure.js new file mode 100644 index 000000000000..59c3fdf5d7eb --- /dev/null +++ b/test/js/node/test_runner/fixtures/25-expect-failure.js @@ -0,0 +1,18 @@ +const assert = require("node:assert"); +const { test } = require("node:test"); + +test("a failing body is the expected outcome", { expectFailure: true }, () => { + assert.fail("boom"); +}); + +test("a label is allowed in place of true", { expectFailure: "known broken" }, () => { + throw new Error("still broken"); +}); + +test("a RegExp validates the error", { expectFailure: /boom/ }, () => { + throw new Error("boom"); +}); + +test("an object may carry both label and match", { expectFailure: { label: "x", match: /nope/ } }, () => { + throw new Error("nope"); +}); diff --git a/test/js/node/test_runner/fixtures/26-skipped-suite-body.js b/test/js/node/test_runner/fixtures/26-skipped-suite-body.js new file mode 100644 index 000000000000..c5810b9d6d57 --- /dev/null +++ b/test/js/node/test_runner/fixtures/26-skipped-suite-body.js @@ -0,0 +1,17 @@ +const { suite, test } = require("node:test"); + +// Node never invokes a skipped suite's callback, and treats { skip, todo } as a +// skip, so neither of these may print. A todo suite's callback does run. +suite("skipped suite", { skip: true }, () => { + console.log("[suite body ran: skip-only]"); +}); + +suite("skip wins over todo", { skip: true, todo: true }, () => { + console.log("[suite body ran: both-flags]"); +}); + +suite("todo suite", { todo: true }, () => { + console.log("[suite body ran: pending-only]"); +}); + +test("sanity", () => {}); diff --git a/test/js/node/test_runner/fixtures/27-expect-failure-but-passes.js b/test/js/node/test_runner/fixtures/27-expect-failure-but-passes.js new file mode 100644 index 000000000000..d5d8b040bee1 --- /dev/null +++ b/test/js/node/test_runner/fixtures/27-expect-failure-but-passes.js @@ -0,0 +1,4 @@ +const { test } = require("node:test"); + +// Node fails a test that was expected to fail but did not. +test("passes unexpectedly", { expectFailure: true }, () => {}); diff --git a/test/js/node/test_runner/fixtures/28-expect-failure-inherited.js b/test/js/node/test_runner/fixtures/28-expect-failure-inherited.js new file mode 100644 index 000000000000..276415fe65c5 --- /dev/null +++ b/test/js/node/test_runner/fixtures/28-expect-failure-inherited.js @@ -0,0 +1,10 @@ +const { test } = require("node:test"); + +// The subtest inherits expectFailure, so its throw is the expected outcome and +// it passes — which leaves the parent passing when it was expected to fail. +// Verified against node v26.3.0: 1 fail, "test was expected to fail but passed". +test("expectFailure is inherited by subtests", { expectFailure: true }, async t => { + await t.test("child inherits", () => { + throw new Error("child boom"); + }); +}); diff --git a/test/js/node/test_runner/fixtures/29-expect-failure-mismatch.js b/test/js/node/test_runner/fixtures/29-expect-failure-mismatch.js new file mode 100644 index 000000000000..e82c1a58bfd9 --- /dev/null +++ b/test/js/node/test_runner/fixtures/29-expect-failure-mismatch.js @@ -0,0 +1,5 @@ +const { test } = require("node:test"); + +test("the thrown error does not satisfy the validator", { expectFailure: /expected message/ }, () => { + throw new Error("a different message entirely"); +}); diff --git a/test/js/node/test_runner/node-test.test.ts b/test/js/node/test_runner/node-test.test.ts index 2b480dddb050..2a27963203c0 100644 --- a/test/js/node/test_runner/node-test.test.ts +++ b/test/js/node/test_runner/node-test.test.ts @@ -4,21 +4,32 @@ import { bunEnv, bunExe } from "harness"; import { join } from "node:path"; describe("node:test", () => { - test("should run basic tests", async () => { - const { exitCode, stderr } = await runTests(["01-harness.js"]); - expect({ exitCode, stderr }).toMatchObject({ - exitCode: 0, - stderr: expect.stringContaining("0 fail"), - }); - }); - - test("should run hooks in the right order", async () => { - const { exitCode, stderr } = await runTests(["02-hooks.js"]); - expect({ exitCode, stderr }).toMatchObject({ - exitCode: 0, - stderr: expect.stringContaining("0 fail"), - }); - }); + // These three drive the largest fixtures (01-harness has 32 node:test cases); + // a debug+ASAN `bun test` child takes several seconds to start, so give them + // headroom and let them spawn in parallel instead of serially. + test.concurrent( + "should run basic tests", + async () => { + const { exitCode, stderr } = await runTests(["01-harness.js"]); + expect({ exitCode, stderr }).toMatchObject({ + exitCode: 0, + stderr: expect.stringContaining("0 fail"), + }); + }, + 30_000, + ); + + test.concurrent( + "should run hooks in the right order", + async () => { + const { exitCode, stderr } = await runTests(["02-hooks.js"]); + expect({ exitCode, stderr }).toMatchObject({ + exitCode: 0, + stderr: expect.stringContaining("0 fail"), + }); + }, + 30_000, + ); test("should run tests with different variations", async () => { const { exitCode, stderr } = await runTests(["03-test-variations.js"]); @@ -36,14 +47,18 @@ describe("node:test", () => { }); }); - test("should run all tests from multiple files", async () => { - const { exitCode, stderr } = await runTests(["01-harness.js", "02-hooks.js"]); - expect({ exitCode, stderr }).toMatchObject({ - exitCode: 0, - // 32 from 01-harness + 3 from 02-hooks - stderr: expect.stringContaining("35 pass"), - }); - }); + test.concurrent( + "should run all tests from multiple files", + async () => { + const { exitCode, stderr } = await runTests(["01-harness.js", "02-hooks.js"]); + expect({ exitCode, stderr }).toMatchObject({ + exitCode: 0, + // 32 from 01-harness + 3 from 02-hooks + stderr: expect.stringContaining("35 pass"), + }); + }, + 30_000, + ); test("should run test() and describe() called inside another test() as subtests", async () => { const { exitCode, stderr } = await runTests(["05-test-in-test.js"]); @@ -194,6 +209,56 @@ describe("node:test", () => { }); }); + test("should treat a failing expectFailure test as a pass", async () => { + const { exitCode, stderr } = await runTests(["25-expect-failure.js"]); + expect({ exitCode, stderr }).toMatchObject({ + exitCode: 0, + stderr: expect.stringContaining("0 fail"), + }); + }); + + test("should fail an expectFailure test that passes", async () => { + const { exitCode, stderr } = await runTests(["27-expect-failure-but-passes.js"]); + expect(stderr).toContain("test was expected to fail but passed"); + expect({ exitCode, stderr }).toMatchObject({ + exitCode: 1, + stderr: expect.stringContaining("1 fail"), + }); + }); + + test("should fail an expectFailure test whose error does not match the validator", async () => { + const { exitCode, stderr } = await runTests(["29-expect-failure-mismatch.js"]); + expect(stderr).toContain("the error did not match the expected validation"); + expect({ exitCode, stderr }).toMatchObject({ + exitCode: 1, + stderr: expect.stringContaining("1 fail"), + }); + }); + + test("should inherit expectFailure into subtests", async () => { + // Matches node v26.3.0: the subtest inherits the expectation and passes, so + // the parent is the one that fails for not failing. + const { exitCode, stderr } = await runTests(["28-expect-failure-inherited.js"]); + expect(stderr).toContain("test was expected to fail but passed"); + expect({ exitCode, stderr }).toMatchObject({ + exitCode: 1, + stderr: expect.stringContaining("1 fail"), + }); + }); + + test("should not run a skipped suite's callback", async () => { + const { exitCode, stdout, stderr } = await runTests(["26-skipped-suite-body.js"]); + expect(stdout).not.toContain("[suite body ran: skip-only]"); + // { skip: true, todo: true } is a skip in Node, so this body is skipped too. + expect(stdout).not.toContain("[suite body ran: both-flags]"); + // A todo suite's callback does run. + expect(stdout).toContain("[suite body ran: pending-only]"); + expect({ exitCode, stderr }).toMatchObject({ + exitCode: 0, + stderr: expect.stringContaining("0 fail"), + }); + }); + test("should reset the module-level mock tracker between --rerun-each iterations", async () => { // ESM entry: --rerun-each currently only re-evaluates ESM entry files. const { exitCode, stderr } = await runTests(["17-rerun-mock-reset.mjs"], {}, ["--rerun-each=3"]);