diff --git a/src/jsc/VmHandle.rs b/src/jsc/VmHandle.rs index 0e4d5cce3321..ee6247c0da75 100644 --- a/src/jsc/VmHandle.rs +++ b/src/jsc/VmHandle.rs @@ -409,6 +409,14 @@ impl VmHandle { self.0.state() == State::Open } + /// Is the wait over? Past it nothing started for the VM runs any more: + /// [`VirtualMachine::ticket`] panics and a queued task is only released. + /// Any thread; meaningful on the JS thread (a final collection's finalizer). + #[inline] + pub fn closed(&self) -> bool { + self.0.state() == State::Closed + } + pub(crate) fn tickets_outstanding(&self) -> u32 { self.0.tickets.load(Ordering::SeqCst) } @@ -503,19 +511,30 @@ impl VmHandle { impl VirtualMachine { /// JS thread: a ticket for work about to leave this thread — this VM, and /// the loop it is currently ticking. Hold it in the in-flight operation - /// and drop it after the completion is posted. Infallible until the wait - /// has finished (after which nothing on this thread starts off-thread work). + /// and drop it after the completion is posted. + /// + /// Panics, in release too, once the VM is [`closed`](Self::closed): the + /// wait is over, so nothing would wait for this ticket and its completion + /// would land on a loop teardown is freeing. Callers that can run that late + /// (finalizers) check `closed()` and refuse the work; the panic names one + /// that did not. #[track_caller] #[inline] pub fn ticket(&self) -> Ticket { let h = self.handle_ref(); h.assert_js_thread(); - debug_assert!( - h.0.state() != State::Closed, + assert!( + !h.closed(), "off-thread work started after the VM finished draining" ); Ticket::issue(&h.0, self.current_loop_kind()) } + + /// [`VmHandle::closed`] for this VM. + #[inline] + pub fn closed(&self) -> bool { + self.handle_ref().closed() + } } // ── Test suite only: deterministic late completions ─────────────────────── diff --git a/src/jsc/event_loop.rs b/src/jsc/event_loop.rs index 6437e63f9456..e0bab29150a4 100644 --- a/src/jsc/event_loop.rs +++ b/src/jsc/event_loop.rs @@ -49,6 +49,9 @@ pub struct EventLoop { /// Set when teardown releases the queue: from then on `enqueue_task` /// releases instead of parking (nothing will tick this loop again). closed_for_tasks: bool, + /// Set while `release_queued_tasks` is draining: a task enqueued by one + /// it releases is parked for that same drain (see `enqueue_task`). + releasing_tasks: bool, /// setImmediate() gets it's own two task queues /// When you call `setImmediate` in JS, it queues to the start of the next tick @@ -119,6 +122,7 @@ impl Default for EventLoop { Self { tasks: Queue::init(), closed_for_tasks: false, + releasing_tasks: false, immediate_tasks: Vec::new(), next_immediate_tasks: Vec::new(), yield_tasks: Vec::new(), @@ -719,10 +723,14 @@ impl EventLoop { } pub fn enqueue_task(&mut self, task: Task) { - if self.closed_for_tasks { + if self.closed_for_tasks && !self.releasing_tasks { // Teardown already released the queue and this loop never ticks // again: release the task now, as `release_queued_tasks` would have // — the queue owns refusal, like `VmHandle::post` does off-thread. + // While that release is still draining, the task is parked for it + // instead, so a released task that enqueues another (a napi + // `complete` queueing more work) has it released after itself + // rather than inside itself. // SAFETY: JS thread, JSC heap alive (teardown phase B/C). unsafe { __bun_release_task_unrun(task) }; return; @@ -758,16 +766,35 @@ impl EventLoop { /// JSC heap alive, children joined): called on every turn of the wait, and /// once more after `Closed`. pub fn release_queued_tasks(&mut self) { - self.closed_for_tasks = true; - self.take_concurrent_tasks(); - let _ = self.promote_yield_tasks(); - while let Some(task) = self.tasks.read_item() { + // R-2 noalias mitigation — see `run_callback` above. `&mut self` is + // `noalias` and `__bun_release_task_unrun` receives nothing derived + // from it, but a release re-enters this loop through + // `vm.event_loop_mut()`: `enqueue_task` reads the two flags stored + // below and, while we are draining, pushes onto `tasks`, which the next + // `read_item` must observe. So everything here goes through a + // laundered pointer, re-escaped after every release. + let mut this: *mut Self = core::hint::black_box(core::ptr::from_mut(self)); + // SAFETY: `this` is the unique live `EventLoop`; every access through + // it is a short-lived borrow that ends before a release runs. + unsafe { + (*this).closed_for_tasks = true; + (*this).take_concurrent_tasks(); + let _ = (*this).promote_yield_tasks(); + (*this).releasing_tasks = true; + } + // SAFETY: as above. + while let Some(task) = unsafe { (*this).tasks.read_item() } { // SAFETY: JS thread, heap alive; `task` just left the queue. unsafe { __bun_release_task_unrun(task) }; + this = core::hint::black_box(this); + } + // SAFETY: as above. Pending immediates likewise: cancelling one drops + // its keep-alive on this thread's loop, so it happens now, not after + // the loop is gone. + unsafe { + (*this).releasing_tasks = false; + (*this).release_pending_immediates(); } - // Pending immediates likewise: cancelling one drops its keep-alive on - // this thread's loop, so it happens now, not after the loop is gone. - self.release_pending_immediates(); } /// Cancel (never run) every queued ImmediateObject; each cancel drops the diff --git a/src/runtime/api/JSTranspiler.rs b/src/runtime/api/JSTranspiler.rs index d99b774bd830..2ffb38b895f6 100644 --- a/src/runtime/api/JSTranspiler.rs +++ b/src/runtime/api/JSTranspiler.rs @@ -651,7 +651,8 @@ impl Config { // This is going to be hard to not leak /// `transpiler.transform()` off the JS thread. The parse/print state points /// into the owning `JSTranspiler`'s config (its `Transpiler` is bit-copied), -/// which the job's Js side keeps alive and the pool borrow keeps valid. +/// which the `Strong` on the job's Js side keeps alive until the job is back +/// on the JS thread; `run` reads it while the job's ticket keeps the VM alive. pub(crate) struct TransformTask { pub input_code: bun_jsc::ThreadSafe, pub output_code: BunString, @@ -663,8 +664,8 @@ pub(crate) struct TransformTask { pub loader: Loader, pub replace_exports: bun_ast::runtime::ReplaceableExportMap, } -// SAFETY: see the type doc — VM-owned config is read only under the pool -// borrow; everything else is owned. +// SAFETY: see the type doc — the wrapper's config (behind `transpiler` and +// `tsconfig`) is read only in `run`, under the job's ticket; the rest is owned. unsafe impl Send for TransformTask {} #[derive(bun_jsc::JsAffine)] diff --git a/src/runtime/image/Image.rs b/src/runtime/image/Image.rs index 6a90ee95521e..95c24253b8d5 100644 --- a/src/runtime/image/Image.rs +++ b/src/runtime/image/Image.rs @@ -1398,8 +1398,10 @@ pub struct PipelineTask { auto_orient: bool, result: TaskResult, } -// SAFETY: `input` borrows bytes that are pinned (`Pin`) or owned by the Image -// the job's Js side keeps alive; read only under the pool borrow. The rest is owned. +// SAFETY: `input` borrows bytes pinned by `Pin` or, like its path, owned by the +// Image `PendingTask` holds; both sit on the job's Js side, dropped there only +// after the pool has posted this task back, and the pool reads them in `run`, +// under the job's ticket. The rest is owned. unsafe impl Send for PipelineTask {} /// The JS-thread half of a scheduled `PipelineTask`. diff --git a/src/runtime/napi/napi_body.rs b/src/runtime/napi/napi_body.rs index f25df298ddf8..1f946f8795c8 100644 --- a/src/runtime/napi/napi_body.rs +++ b/src/runtime/napi/napi_body.rs @@ -43,9 +43,10 @@ impl JSValueNapiExt for JSValue { // `Taskable` impls for the napi heap tasks dispatched through the JS event loop. impl Taskable for napi_async_work { const TAG: TaskTag = task_tag::NapiAsyncWork; - /// Work the pool handed back during teardown: its `complete` callback is - /// how the addon learns the outcome and frees the work (Node calls it from - /// environment cleanup too); script it tries to run is refused at the boundary. + /// Work handed back during teardown (by the pool, or by `schedule` itself + /// once script was forbidden): its `complete` callback is how the addon + /// learns the outcome and frees the work (Node calls it from environment + /// cleanup too); script it tries to run is refused at the boundary. unsafe fn release_unrun(this: *mut Self) { let vm = VirtualMachine::get().as_mut(); let global = vm.global(); @@ -143,6 +144,12 @@ impl NapiEnv { Self::set_last_error(Some(self), NapiStatus::pending_exception) } + /// Node's status for a call the environment can no longer serve because it + /// is shutting down. + pub(crate) fn cannot_run_js(&self) -> napi_status { + Self::set_last_error(Some(self), NapiStatus::cannot_run_js) + } + /// Checks both `env->m_pendingException` (set by `napi_throw*`) and the JSC /// VM exception slot. This is the gate Node.js's `NAPI_PREAMBLE` enforces. pub(crate) fn has_pending_exception(&self) -> bool { @@ -1851,17 +1858,44 @@ impl napi_async_work { drop(unsafe { bun_core::heap::take(this) }); } - pub(crate) fn schedule(&mut self) { - if self.scheduled { - return; + /// `false`: refused untouched, the work stays the addon's to delete. + /// + /// # Safety + /// `this` is a live work on its JS thread. It may be gone when this + /// returns `true`: its `complete` (which addons delete the work from) can + /// run inside the call. + pub(crate) unsafe fn schedule(this: *mut Self) -> bool { + // SAFETY: fn contract. The work is only ever reached through `this`, + // one statement at a time, so no reference to it is live across + // `enqueue_task`, which may free it. + unsafe { + if (*this).scheduled { + return true; + } + let vm = VirtualMachine::get(); + if vm.closed() { + // A finalizer of the collection destroying the heap: + // `vm.ticket()` would panic, and the closed queue would run + // `complete` right here, mid-collection. + return false; + } + (*this).scheduled = true; + if !vm.script_allowed() { + // The pool would only hand it back cancelled; skip the pool and + // the ticket. Whether the queue runs it or teardown releases + // it, `complete` gets `napi_cancelled` as before. + let _ = (*this).cancel(); + vm.event_loop_mut().enqueue_task(Task::init(this)); + return true; + } + (*this).poll_ref.ref_(bun_io::js_vm_ctx()); + // The work object belongs to the addon and `execute` receives this + // env, so the VM waits for it (Node likewise settles its threadpool + // requests before an environment is freed). + (*this).ticket = Some(vm.ticket()); + WorkPool::schedule(&raw mut (*this).task); + true } - self.scheduled = true; - self.poll_ref.ref_(bun_io::js_vm_ctx()); - // The work object belongs to the addon and `execute` receives this - // env, so the VM waits for it (Node likewise settles its threadpool - // requests before an environment is freed). - self.ticket = Some(self.global.bun_vm().ticket()); - WorkPool::schedule(&raw mut self.task); } pub(crate) unsafe fn run_from_thread_pool(task: *mut WorkPoolTask) { @@ -2222,12 +2256,18 @@ extern "C" fn napi_delete_async_work(env_: napi_env, work_: *mut napi_async_work extern "C" fn napi_queue_async_work(env_: napi_env, work_: *mut napi_async_work) -> napi_status { bun_output::scoped_log!(napi, "napi_queue_async_work"); let env = get_env!(env_); - // SAFETY: `work_` is null or the `napi_async_work` we allocated in `napi_create_async_work`. - let Some(work) = (unsafe { work_.as_mut() }) else { + if work_.is_null() { return env.invalid_arg(); - }; - debug_assert!(core::ptr::eq(env.to_js(), work.global.as_ptr())); - work.schedule(); + } + // SAFETY: non-null `work_` is the `napi_async_work` we allocated in + // `napi_create_async_work`, live until `schedule` (which may free it: its + // `complete` can run inside) takes it; no reference to it is formed here. + unsafe { + debug_assert!(core::ptr::eq(env.to_js(), (*work_).global.as_ptr())); + if !napi_async_work::schedule(work_) { + return env.cannot_run_js(); + } + } env.ok() } diff --git a/src/runtime/webcore/CompressionStreamCoder.rs b/src/runtime/webcore/CompressionStreamCoder.rs index 2115bcc30545..7f9e809cb75f 100644 --- a/src/runtime/webcore/CompressionStreamCoder.rs +++ b/src/runtime/webcore/CompressionStreamCoder.rs @@ -579,7 +579,8 @@ pub(crate) enum AsyncInput { Owned(Vec), } // SAFETY: `Pinned.ptr` is a backing store pinned + protected by the paired -// `PinnedChunk` for as long as the job lives; read only under the pool borrow. +// `PinnedChunk` on the job's Js side, dropped there only after the pool has +// posted the job back; the pool reads it in `run`, under the job's ticket. unsafe impl Send for AsyncInput {} /// The pin + GC protection on a chunk whose bytes went to the pool; released diff --git a/test/js/web/workers/worker-late-completion-napi-fixture.c b/test/js/web/workers/worker-late-completion-napi-fixture.c new file mode 100644 index 000000000000..267ec50a5328 --- /dev/null +++ b/test/js/web/workers/worker-late-completion-napi-fixture.c @@ -0,0 +1,127 @@ +// The napi rows of worker-late-completion.test.ts. Built against Bun's own +// N-API headers as a NAPI_VERSION_EXPERIMENTAL addon, so its finalizers run +// inline during garbage collection (including the collection that destroys +// the worker's heap) and only the basic-env subset of the API is allowed there; +// napi_queue_async_work is in that subset. Every entry point starts one +// napi_async_work at a different point in the worker's life, and every work +// reports the status its `complete` callback is eventually handed. +#define NAPI_EXPERIMENTAL +#include +#include + +typedef struct Work { + const char *name; + napi_async_work work; + // Queued by this work's `complete`, i.e. while the worker is already + // tearing down (queueFromComplete). + struct Work *then; +} Work; + +static Work first = {"first", NULL, NULL}; +static Work second = {"second", NULL, NULL}; +static Work late = {"late", NULL, NULL}; + +static void report(const char *what, const Work *w, napi_status status) { + switch (status) { + case napi_ok: + printf("%s %s: ok\n", what, w->name); + break; + case napi_cancelled: + printf("%s %s: cancelled\n", what, w->name); + break; + case napi_cannot_run_js: + printf("%s %s: cannot run js\n", what, w->name); + break; + default: + printf("%s %s: napi_status %d\n", what, w->name, (int)status); + break; + } + fflush(stdout); +} + +static void execute(napi_env env, void *data) { + (void)env; + (void)data; +} + +static void complete(napi_env env, napi_status status, void *data) { + Work *w = data; + report("complete", w, status); + if (w->then != NULL) { + report("queued from complete", w->then, + napi_queue_async_work(env, w->then->work)); + } + napi_delete_async_work(env, w->work); +} + +// A failure here shows up in the row's output as its own line. +static void create(napi_env env, Work *w) { + napi_value name; + napi_status status = + napi_create_string_utf8(env, w->name, NAPI_AUTO_LENGTH, &name); + if (status == napi_ok) { + status = + napi_create_async_work(env, NULL, name, execute, complete, w, &w->work); + } + if (status != napi_ok) { + report("created", w, status); + } +} + +// The ordinary path: queued while the worker runs. The completion comes back +// from the pool during the teardown's wait and is released there. +static napi_value queue(napi_env env, napi_callback_info info) { + (void)info; + create(env, &first); + report("queued", &first, napi_queue_async_work(env, first.work)); + return NULL; +} + +// `first` completes during the wait; its `complete` then queues `second`. +static napi_value queue_from_complete(napi_env env, napi_callback_info info) { + create(env, &second); + first.then = &second; + return queue(env, info); +} + +// Runs during the collection that destroys the heap, after the wait is over: +// the worker keeps the external reachable until it exits. The work itself is +// created up front because napi_create_async_work is not a basic-env API. +// Refused work is still the addon's; deleting it here (a plain free in Bun, +// though the API is not declared basic, hence the cast) keeps the leak +// checker quiet. +static void finalize(node_api_basic_env env, void *data, void *hint) { + (void)hint; + Work *w = data; + napi_status status = napi_queue_async_work(env, w->work); + report("queued from finalizer", w, status); + if (status != napi_ok) { + napi_delete_async_work((napi_env)env, w->work); + } +} + +static napi_value queue_from_finalizer(napi_env env, napi_callback_info info) { + (void)info; + create(env, &late); + napi_value external; + napi_create_external(env, &late, finalize, NULL, &external); + return external; +} + +// One napi_create_function + napi_set_named_property per export rather than +// napi_define_properties: the ASAN lane runs this file with +// BUN_JSC_validateExceptionChecks=1, which napi_define_properties' method +// path does not pass yet. +static void export_function(napi_env env, napi_value exports, const char *name, + napi_callback callback) { + napi_value function; + napi_create_function(env, name, NAPI_AUTO_LENGTH, callback, NULL, &function); + napi_set_named_property(env, exports, name, function); +} + +NAPI_MODULE_INIT() { + export_function(env, exports, "queue", queue); + export_function(env, exports, "queueFromComplete", queue_from_complete); + export_function(env, exports, "queueFromFinalizer", queue_from_finalizer); + return exports; +} diff --git a/test/js/web/workers/worker-late-completion.test.ts b/test/js/web/workers/worker-late-completion.test.ts index 7d27ea293c9a..fcad8bb2afa7 100644 --- a/test/js/web/workers/worker-late-completion.test.ts +++ b/test/js/web/workers/worker-late-completion.test.ts @@ -14,10 +14,14 @@ // line appeared (the work really was on another thread, really came back // during teardown, and — for ticketed work — was taken through the door at // the expected site) and the process exited cleanly; on the ASAN build the -// release paths are also checked for use-after-free and leaks. Builds with -// debug assertions only (debug, ASAN): the gate does not exist in release. +// release paths are also checked for use-after-free and leaks. The napi rows +// also cover the opposite obligation: work queued once the teardown is under +// way (from a completion the wait releases, or from a finalizer in the +// collection that destroys the heap) is settled or refused on the worker's own +// thread rather than given a ticket nothing would wait for. Builds with debug +// assertions only (debug, ASAN): the gate does not exist in release. import { describe, expect, test } from "bun:test"; -import { bunEnv, bunExe, isAndroid, isASAN, isDebug, isLinux, isWindows, tempDir } from "harness"; +import { bunEnv, bunExe, isAndroid, isASAN, isDebug, isLinux, isMacOS, isWindows, tempDir } from "harness"; import fs from "node:fs"; import path from "node:path"; @@ -31,15 +35,33 @@ type Row = { parent?: string; // Runs in the parent when the worker has exited (release what the prelude holds). onExit?: string; + // A C source built into `/addon.node` (against Bun's own N-API headers) before the worker starts. + addon?: string; + // Exactly what the row's native fixture must have printed, in order; a + // RegExp where more than one outcome is correct. + stdout?: (string | RegExp)[]; env?: Record; files?: Record; skip?: boolean; } & (Ticketed | Weak); // Ticketed work: substring of the site (file) the ticket was taken at, as -// logged by "[vm] late completion from :". -type Ticketed = { ticket: string; weak?: never }; +// logged by "[vm] late completion from :". `lateCompletions` pins +// how many completions came back through the door in total; 0 for a row whose +// work is queued so late that it must be settled or refused on the worker's +// own thread instead of taking a ticket. +type Ticketed = ({ ticket: string; lateCompletions?: number } | { ticket?: never; lateCompletions: number }) & { + weak?: never; +}; // Weak posters: the task tag logged by "[vm] late post: (...)". -type Weak = { weak: string; ticket?: never }; +type Weak = { weak: string; ticket?: never; lateCompletions?: never }; + +// The napi rows need a C compiler; Windows has none on PATH in CI (the napi +// suite proper goes through node-gyp for that), so they skip there. +const cc = isWindows ? null : process.env.CC || Bun.which("cc") || Bun.which("gcc") || Bun.which("clang"); +const napiFixture = path.join(import.meta.dir, "worker-late-completion-napi-fixture.c"); +const napiAddon = `require(require("node:path").join(workerData.dir, "addon.node"))`; +// See the first napi row. +const FIRST_SETTLED = /^complete first: (ok|cancelled)$/; const ROWS: Row[] = [ // ── thread pool: bun_jsc::Job ──────────────────────────────────────────── @@ -192,6 +214,50 @@ const ROWS: Row[] = [ ticket: "js_bundle_completion_task.rs", files: { "entry.ts": `export default 1;\n` }, }, + // ── thread pool: napi_async_work (worker-late-completion-napi-fixture.c) ─ + { + // Queued while the worker runs: the work goes to the pool under a ticket + // and the wait releases the completion, which for napi means running the + // addon's `complete` (that is how it frees the work). Its status depends on + // whether the pool reached the work before the worker began stopping + // (`execute` ran: ok) or after (it did not: cancelled); the gate only fixes + // when the completion arrives, so either is correct here. + name: "napi_queue_async_work", + addon: napiFixture, + worker: `${napiAddon}.queue();`, + ticket: "napi_body.rs", + stdout: ["queued first: ok", FIRST_SETTLED], + skip: !cc, + }, + { + // `first`'s complete runs from the wait's release, with script already + // forbidden, and queues `second`. Work queued that late is handed back on + // this thread (complete with napi_cancelled) without going near the pool, + // so only `first` ever crossed the door; and it is released after the + // complete that queued it returns, not inside it (the order below), so a + // chain of such works is released link by link rather than recursively. + name: "napi_queue_async_work from a complete callback during the wait", + addon: napiFixture, + worker: `${napiAddon}.queueFromComplete();`, + ticket: "napi_body.rs", + lateCompletions: 1, + stdout: ["queued first: ok", FIRST_SETTLED, "queued from complete second: ok", "complete second: cancelled"], + skip: !cc, + }, + { + // The addon is NAPI_VERSION_EXPERIMENTAL, so the external's finalizer runs + // inline in the collection that destroys the worker's heap: after the wait, + // when a ticket would be waited for by nobody and nothing is left to run + // `complete` either, so queueing is refused (napi_cannot_run_js) and the + // addon keeps the work. The external is kept reachable so that collection + // is the one that finalizes it. + name: "napi_queue_async_work from a finalizer in the final collection", + addon: napiFixture, + worker: `globalThis.keep = ${napiAddon}.queueFromFinalizer();`, + lateCompletions: 0, + stdout: ["queued from finalizer late: cannot run js"], + skip: !cc, + }, // ── weak posters (no ticket): delivered while draining, or refused ─────── { name: "child exit reported by the waiter thread", @@ -242,10 +308,34 @@ function host(row: Row, dir: string) { `; } +// The napi_* symbols stay undefined in the addon and resolve against the bun +// that dlopens it (as node-gyp builds do); the headers are Bun's own, as in the +// in-tree header test of test/napi/napi-value-ffi.test.ts. +async function buildAddon(source: string, dir: string) { + await using proc = Bun.spawn({ + cmd: [ + cc!, + "-shared", + "-fPIC", + ...(isMacOS ? ["-undefined", "dynamic_lookup"] : []), + `-I${path.resolve(import.meta.dir, "../../../../src/runtime/napi")}`, + source, + "-o", + path.join(dir, "addon.node"), + ], + env: bunEnv, + stdout: "ignore", + stderr: "pipe", + }); + const [stderr, exitCode] = await Promise.all([proc.stderr.text(), proc.exited]); + expect({ exitCode, stderr: exitCode === 0 ? "" : stderr }).toEqual({ exitCode: 0, stderr: "" }); +} + describe.skipIf(!isDebug && !isASAN)("work that comes back after its worker began tearing down", () => { for (const row of ROWS) { test.concurrent.skipIf(!!row.skip)(row.name, async () => { using dir = tempDir("worker-late-completion", row.files ?? {}); + if (row.addon) await buildAddon(row.addon, String(dir)); await using proc = Bun.spawn({ cmd: [bunExe(), "-e", host(row, String(dir))], env: { ...bunEnv, ...row.env, BUN_DEBUG_TEST_WORKER_TEARDOWN_GATE: "1" }, @@ -254,16 +344,26 @@ describe.skipIf(!isDebug && !isASAN)("work that comes back after its worker bega }); const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); const lines = stderr.split("\n").filter(l => l.startsWith("[vm] ")); + const completions = lines.filter(l => l.startsWith("[vm] late completion from ")); + const { ticket, lateCompletions, weak } = row; const seen = - "ticket" in row && row.ticket - ? lines.some(l => l.startsWith("[vm] late completion from ") && l.includes(row.ticket)) - : lines.some(l => l.startsWith(`[vm] late post: ${row.weak} (`)); + weak !== undefined + ? lines.some(l => l.startsWith(`[vm] late post: ${weak} (`)) + : (ticket === undefined || completions.some(l => l.includes(ticket))) && + (lateCompletions === undefined || completions.length === lateCompletions); + const printed = stdout.split("\n").filter(Boolean); + const printedAsExpected = + row.stdout === undefined || + (printed.length === row.stdout.length && + row.stdout.every((want, i) => (typeof want === "string" ? printed[i] === want : want.test(printed[i])))); + const ok = exitCode === 0 && seen && printedAsExpected; expect({ exitCode, seen, + printedAsExpected, // On a failure, everything the host printed. - detail: exitCode === 0 && seen ? "" : stdout + stderr, - }).toEqual({ exitCode: 0, seen: true, detail: "" }); + detail: ok ? "" : stdout + stderr, + }).toEqual({ exitCode: 0, seen: true, printedAsExpected: true, detail: "" }); }); } });