From 34afcea660ba860dbf319c265165f4c6ee0c6fa8 Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Tue, 14 Jul 2026 23:46:29 +0000 Subject: [PATCH 01/14] usockets: sweep mimalloc's heaps while the loop is parked, not before it parks The idle sweep punches free-block holes out of pages that are still in use, and it is almost entirely madvise: on a 100MB churn ~99% of its cost is the syscalls. It ran inline from Bun__JSC_onBeforeWait, so the JS thread paid all of it right before blocking. mimalloc can now be told a thread is about to block and is not going to touch its heaps until it wakes, which is the only condition the sweep actually needs. Hand the heaps over across the poll and the scavenger does the syscalls while we sit in the kernel. Measured in isolation the owner goes from 20.0ms to 0.005ms per park for the same work. The handoff goes after Bun__JSC_onBeforeWait, which allocates, and before dispatch, which allocates: nothing between it and the matching take-back may touch the heaps. Windows still runs the libuv loop, which has no handoff, so it keeps sweeping inline. Pins mimalloc to oven-sh/mimalloc#8. --- .../bun-usockets/src/eventing/epoll_kqueue.c | 26 +++++++++++++++++++ scripts/build/deps/mimalloc.ts | 2 +- src/jsc/bindings/BunJSCEventLoop.cpp | 10 +++---- 3 files changed, 32 insertions(+), 6 deletions(-) diff --git a/packages/bun-usockets/src/eventing/epoll_kqueue.c b/packages/bun-usockets/src/eventing/epoll_kqueue.c index 679d8641038d..bb633f37208f 100644 --- a/packages/bun-usockets/src/eventing/epoll_kqueue.c +++ b/packages/bun-usockets/src/eventing/epoll_kqueue.c @@ -355,6 +355,10 @@ void us_loop_run(struct us_loop_t *loop) { } extern void Bun__JSC_onBeforeWait(void * _Nonnull jsc_vm, uint64_t now_ns); +/* Declared here rather than pulled from : WebKit vendors its own, older mimalloc + * whose header has neither of these. */ +extern void mi_on_thread_idle_start(void); +extern void mi_on_thread_idle_end(void); void us_loop_run_bun_tick(struct us_loop_t *loop, const struct timespec* timeout, uint64_t now_ns) { if (loop->num_polls == 0) @@ -391,6 +395,24 @@ void us_loop_run_bun_tick(struct us_loop_t *loop, const struct timespec* timeout if (will_idle_inside_event_loop && loop->data.jsc_vm) Bun__JSC_onBeforeWait(loop->data.jsc_vm, now_ns); + /* Hand this thread's heaps to mimalloc's scavenger for the duration of the wait: it does the + * sweep (~99% madvise) while we are blocked in the kernel, instead of us doing it before we + * get there. Must come after Bun__JSC_onBeforeWait -- JSC allocates in it, and the handoff's + * whole premise is that we do not touch our heaps until the matching _end. + * Same 100ms rate limit the inline sweep used (see BunJSCEventLoop.cpp): the handoff is cheap + * for us, but the sweep it triggers is not free for the scavenger. */ + int did_idle_handoff = 0; + if (will_idle_inside_event_loop) { + static const uint64_t idle_sweep_interval_ns = 100 * 1000000ULL; + static _Thread_local uint64_t last_idle_sweep_ns = 0; + const uint64_t sweep_now_ns = now_ns ? now_ns : us_internal_monotonic_ns(); + if (sweep_now_ns >= last_idle_sweep_ns + idle_sweep_interval_ns) { + last_idle_sweep_ns = sweep_now_ns; + mi_on_thread_idle_start(); + did_idle_handoff = 1; + } + } + /* Fetch ready polls */ #ifdef LIBUS_USE_EPOLL /* A zero timespec already has a fast path in ep_poll (fs/eventpoll.c): @@ -411,6 +433,10 @@ void us_loop_run_bun_tick(struct us_loop_t *loop, const struct timespec* timeout } while (IS_EINTR(loop->num_ready_polls)); #endif + /* Take the heaps back before anything can allocate again. */ + if (did_idle_handoff) + mi_on_thread_idle_end(); + us_internal_dispatch_ready_polls(loop); us_internal_drain_ready_polls(loop); us_internal_sweep_if_due(loop); diff --git a/scripts/build/deps/mimalloc.ts b/scripts/build/deps/mimalloc.ts index 6bb93084c1f7..b952de315765 100644 --- a/scripts/build/deps/mimalloc.ts +++ b/scripts/build/deps/mimalloc.ts @@ -12,7 +12,7 @@ import type { Dependency, DirectBuild } from "../source.ts"; -const MIMALLOC_COMMIT = "13eecae8f35a73c16bdcded9291d9b56b7fc0fca"; +const MIMALLOC_COMMIT = "0d014cad03b667a0f8a51921aee15506ea60f97f"; export const mimalloc: Dependency = { name: "mimalloc", diff --git a/src/jsc/bindings/BunJSCEventLoop.cpp b/src/jsc/bindings/BunJSCEventLoop.cpp index 18c57e4c6610..736fdd6207fb 100644 --- a/src/jsc/bindings/BunJSCEventLoop.cpp +++ b/src/jsc/bindings/BunJSCEventLoop.cpp @@ -82,16 +82,16 @@ extern "C" void Bun__JSC_onBeforeWait(JSC::VM* _Nonnull vm, uint64_t nowNs) vm->heap.stopIfNecessary(); vm->didEnterVM = false; -#if USE(MIMALLOC) +#if USE(MIMALLOC) && OS(WINDOWS) // Collect retired pages, punch free-block holes, hand the arena purge to // the scavenger. Rate-limited; nowNs is the tick's shared reading (0 = take // one), compared by addition so an out-of-order reading cannot underflow. + // + // Windows only: everywhere else `us_loop_run_bun_tick` hands the heaps to the + // scavenger across the poll instead, so this thread never does the sweep itself. + // The libuv loop has no handoff yet, so it keeps paying for it here. static constexpr uint64_t idleSweepIntervalNs = 100 * 1000000ULL; static thread_local uint64_t lastIdleSweepNs = 0; -#if !OS(WINDOWS) - if (nowNs == 0) - nowNs = us_internal_monotonic_ns(); -#endif if (nowNs >= lastIdleSweepNs + idleSweepIntervalNs) { lastIdleSweepNs = nowNs; mi_on_thread_idle(); From c094bd9f457aad8f85c3c5c27d78dd54352992cc Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Tue, 14 Jul 2026 23:59:29 +0000 Subject: [PATCH 02/14] usockets: hand the heaps over on every tick, and use mimalloc's header The handoff is a compare-and-swap, so there is nothing to gate it on: mimalloc paces the sweep itself (purge_holes_min_interval), which is where that policy belongs now that the work is not on this thread. Drops the inherited 100ms timer and the will-idle gate. mimalloc.h declares both entry points, so the local externs are gone. Bumps mimalloc for the teardown, fork and subproc fixes in oven-sh/mimalloc#8. --- .../bun-usockets/src/eventing/epoll_kqueue.c | 31 +++++-------------- scripts/build/deps/mimalloc.ts | 2 +- 2 files changed, 8 insertions(+), 25 deletions(-) diff --git a/packages/bun-usockets/src/eventing/epoll_kqueue.c b/packages/bun-usockets/src/eventing/epoll_kqueue.c index bb633f37208f..ce98a8b3c748 100644 --- a/packages/bun-usockets/src/eventing/epoll_kqueue.c +++ b/packages/bun-usockets/src/eventing/epoll_kqueue.c @@ -30,6 +30,7 @@ void Bun__internal_dispatch_ready_poll(void* loop, void* poll); #include #include #include // memset +#include #endif void us_loop_run_bun_tick(struct us_loop_t *loop, const struct timespec* timeout, uint64_t now_ns); @@ -355,10 +356,6 @@ void us_loop_run(struct us_loop_t *loop) { } extern void Bun__JSC_onBeforeWait(void * _Nonnull jsc_vm, uint64_t now_ns); -/* Declared here rather than pulled from : WebKit vendors its own, older mimalloc - * whose header has neither of these. */ -extern void mi_on_thread_idle_start(void); -extern void mi_on_thread_idle_end(void); void us_loop_run_bun_tick(struct us_loop_t *loop, const struct timespec* timeout, uint64_t now_ns) { if (loop->num_polls == 0) @@ -395,23 +392,10 @@ void us_loop_run_bun_tick(struct us_loop_t *loop, const struct timespec* timeout if (will_idle_inside_event_loop && loop->data.jsc_vm) Bun__JSC_onBeforeWait(loop->data.jsc_vm, now_ns); - /* Hand this thread's heaps to mimalloc's scavenger for the duration of the wait: it does the - * sweep (~99% madvise) while we are blocked in the kernel, instead of us doing it before we - * get there. Must come after Bun__JSC_onBeforeWait -- JSC allocates in it, and the handoff's - * whole premise is that we do not touch our heaps until the matching _end. - * Same 100ms rate limit the inline sweep used (see BunJSCEventLoop.cpp): the handoff is cheap - * for us, but the sweep it triggers is not free for the scavenger. */ - int did_idle_handoff = 0; - if (will_idle_inside_event_loop) { - static const uint64_t idle_sweep_interval_ns = 100 * 1000000ULL; - static _Thread_local uint64_t last_idle_sweep_ns = 0; - const uint64_t sweep_now_ns = now_ns ? now_ns : us_internal_monotonic_ns(); - if (sweep_now_ns >= last_idle_sweep_ns + idle_sweep_interval_ns) { - last_idle_sweep_ns = sweep_now_ns; - mi_on_thread_idle_start(); - did_idle_handoff = 1; - } - } + /* The scavenger sweeps our heaps while we are in the kernel. Must come after + * Bun__JSC_onBeforeWait, which allocates: nothing may touch our heaps until the matching + * _end. mimalloc paces the sweep itself, so this costs a compare-and-swap per tick. */ + mi_on_thread_idle_start(); /* Fetch ready polls */ #ifdef LIBUS_USE_EPOLL @@ -433,9 +417,8 @@ void us_loop_run_bun_tick(struct us_loop_t *loop, const struct timespec* timeout } while (IS_EINTR(loop->num_ready_polls)); #endif - /* Take the heaps back before anything can allocate again. */ - if (did_idle_handoff) - mi_on_thread_idle_end(); + /* Before anything can allocate again. */ + mi_on_thread_idle_end(); us_internal_dispatch_ready_polls(loop); us_internal_drain_ready_polls(loop); diff --git a/scripts/build/deps/mimalloc.ts b/scripts/build/deps/mimalloc.ts index b952de315765..264fa4c123c4 100644 --- a/scripts/build/deps/mimalloc.ts +++ b/scripts/build/deps/mimalloc.ts @@ -12,7 +12,7 @@ import type { Dependency, DirectBuild } from "../source.ts"; -const MIMALLOC_COMMIT = "0d014cad03b667a0f8a51921aee15506ea60f97f"; +const MIMALLOC_COMMIT = "f49ba97662ebb070109af07f8d510fd4c8a57e51"; export const mimalloc: Dependency = { name: "mimalloc", From 38f5d789f8f8c5e9a100ab70ff03e1a8afb62d50 Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Wed, 15 Jul 2026 00:16:21 +0000 Subject: [PATCH 03/14] deps: bump mimalloc for the interruptible collect phase --- scripts/build/deps/mimalloc.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build/deps/mimalloc.ts b/scripts/build/deps/mimalloc.ts index 264fa4c123c4..d1a418dbd8fd 100644 --- a/scripts/build/deps/mimalloc.ts +++ b/scripts/build/deps/mimalloc.ts @@ -12,7 +12,7 @@ import type { Dependency, DirectBuild } from "../source.ts"; -const MIMALLOC_COMMIT = "f49ba97662ebb070109af07f8d510fd4c8a57e51"; +const MIMALLOC_COMMIT = "b5d57c1465f494298f5005975ac041412d4b4637"; export const mimalloc: Dependency = { name: "mimalloc", From 7a84c86b84e5ce866271a4fcb099b23c135419be Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Wed, 15 Jul 2026 00:42:07 +0000 Subject: [PATCH 04/14] usockets: sweep inline only when there is no scavenger to hand off to mi_on_thread_idle_start now reports whether it handed the heaps over instead of quietly sweeping inline when it could not. Without a scavenger the loop keeps what it did before: sweep on the JS thread, but only on a tick that really parks, and no more than every 100ms -- sweeping between ticks is the cost this is avoiding. Bumps mimalloc for that and for the reclaim and fork fixes in oven-sh/mimalloc#8. --- .../bun-usockets/src/eventing/epoll_kqueue.c | 18 +++++++++++++++--- scripts/build/deps/mimalloc.ts | 2 +- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/packages/bun-usockets/src/eventing/epoll_kqueue.c b/packages/bun-usockets/src/eventing/epoll_kqueue.c index ce98a8b3c748..e8e36dc9d05b 100644 --- a/packages/bun-usockets/src/eventing/epoll_kqueue.c +++ b/packages/bun-usockets/src/eventing/epoll_kqueue.c @@ -394,8 +394,19 @@ void us_loop_run_bun_tick(struct us_loop_t *loop, const struct timespec* timeout /* The scavenger sweeps our heaps while we are in the kernel. Must come after * Bun__JSC_onBeforeWait, which allocates: nothing may touch our heaps until the matching - * _end. mimalloc paces the sweep itself, so this costs a compare-and-swap per tick. */ - mi_on_thread_idle_start(); + * _end. mimalloc paces the sweep itself, so this costs a compare-and-swap per tick. + * With no scavenger to hand off to, fall back to sweeping inline -- but only on a tick that + * really parks, and rate-limited, because doing it between ticks is what we are avoiding. */ + const int handed_off = mi_on_thread_idle_start(); + if (!handed_off && will_idle_inside_event_loop) { + static const uint64_t idle_sweep_interval_ns = 100 * 1000000ULL; + static _Thread_local uint64_t last_idle_sweep_ns = 0; + const uint64_t sweep_now_ns = now_ns ? now_ns : us_internal_monotonic_ns(); + if (sweep_now_ns >= last_idle_sweep_ns + idle_sweep_interval_ns) { + last_idle_sweep_ns = sweep_now_ns; + mi_on_thread_idle(); + } + } /* Fetch ready polls */ #ifdef LIBUS_USE_EPOLL @@ -418,7 +429,8 @@ void us_loop_run_bun_tick(struct us_loop_t *loop, const struct timespec* timeout #endif /* Before anything can allocate again. */ - mi_on_thread_idle_end(); + if (handed_off) + mi_on_thread_idle_end(); us_internal_dispatch_ready_polls(loop); us_internal_drain_ready_polls(loop); diff --git a/scripts/build/deps/mimalloc.ts b/scripts/build/deps/mimalloc.ts index d1a418dbd8fd..85164b79dfbe 100644 --- a/scripts/build/deps/mimalloc.ts +++ b/scripts/build/deps/mimalloc.ts @@ -12,7 +12,7 @@ import type { Dependency, DirectBuild } from "../source.ts"; -const MIMALLOC_COMMIT = "b5d57c1465f494298f5005975ac041412d4b4637"; +const MIMALLOC_COMMIT = "e24e5480f33b3987a61ff6e764e5c285466cfe7a"; export const mimalloc: Dependency = { name: "mimalloc", From c8abd3ed96352667b65af9283cc97e28a552591c Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Tue, 14 Jul 2026 18:23:37 -0700 Subject: [PATCH 05/14] jsc: return the JS thread's mimalloc pages to the arena when the heap moves Nothing returns them under sustained load, and no arena-side knob can: the pages sit in the theap's page queues until someone collects the theap, and until then the arena scavenger cannot see them. A REPL replay grew +229MiB inside the arena VMA while madvise flowed healthily for other memory, and purge_delay 1000/100/10 all climbed in lockstep -- the memory had never reached the arena. JSC's own hook cannot stand in. `Heap::didFinishCollection` fires `scavengeThisThread` -> `mi_theap_collect`, but it runs under whichever GCConductor holds the conn, and `collectInCollectorThread` conducts async collections -- `mi_theap_get_default()` there returns the collector's near-empty theap. It is also gated on CollectionScope::Full, and sustained load runs eden for minutes. Forcing a synchronous collection every second cured the ratchet precisely because a sync collection is conducted by the mutator; that was the tell. The GC controller runs on the JS thread by construction and already knows when the heap moved, so do it here. `mi_theap_collect` walks the page queues and frees the empty pages, which schedules the arena purge and wakes the scavenger to madvise off this thread; it does not scan free lists for holes, which is the expensive part that cost vite-preview 12.9% in #34009. libpas needed no hook here at all: its scavenger polled at ~10Hz and shrank thread caches autonomously. --- src/jsc/GarbageCollectionController.rs | 30 ++++++++++++++++++++++++++ src/mimalloc_sys/mimalloc.rs | 14 ++++++++++++ 2 files changed, 44 insertions(+) diff --git a/src/jsc/GarbageCollectionController.rs b/src/jsc/GarbageCollectionController.rs index 9ba4027b76b8..7bcc7bcebabb 100644 --- a/src/jsc/GarbageCollectionController.rs +++ b/src/jsc/GarbageCollectionController.rs @@ -215,9 +215,39 @@ impl GarbageCollectionController { self.process_gc_timer_with_heap_size(vm, vm.block_bytes_allocated()); } + /// Hand this thread's empty mimalloc pages back to the arena, which schedules their purge + /// and wakes the scavenger to do the madvise off this thread. + /// + /// JSC's own hook cannot stand in for this. `Heap::didFinishCollection` fires + /// `scavengeThisThread`, but it runs under whichever `GCConductor` holds the conn -- + /// `collectInCollectorThread` conducts async collections -- and `mi_theap_get_default()` + /// returns the CALLING thread's theap, so on that path it collects the collector's + /// near-empty one. It is also gated on `CollectionScope::Full`, and sustained load runs + /// eden for minutes. Here we are on the JS thread by construction. + /// + /// Cheap enough to do per collection: `mi_theap_collect` walks the page queues and frees + /// empty pages, with no free-list hole scan, and the madvise is the scavenger's job. + #[inline] + fn return_pages_to_arena() { + if bun_core::USE_MIMALLOC { + let theap = bun_alloc::mimalloc::mi_theap_get_default(); + if !theap.is_null() { + bun_alloc::mimalloc::mi_theap_collect(theap, false); + } + } + } + fn process_gc_timer_with_heap_size(&mut self, vm: &VM, this_heap_size: usize) { let prev = self.gc_last_heap_size; + // The heap moved, so a collection freed JS objects and their mimalloc blocks with them. + // Eden does that far more often than full, and until the pages go back to the arena the + // scavenger cannot see them -- no arena-side purge knob reaches memory still parked in a + // theap's page queues. + if this_heap_size != prev { + Self::return_pages_to_arena(); + } + match self.gc_timer_state { GCTimerState::RunOnNextTick => { // When memory usage is not stable, run the GC more. diff --git a/src/mimalloc_sys/mimalloc.rs b/src/mimalloc_sys/mimalloc.rs index e610b63b8a2c..9931c9937c2c 100644 --- a/src/mimalloc_sys/mimalloc.rs +++ b/src/mimalloc_sys/mimalloc.rs @@ -57,6 +57,20 @@ unsafe extern "C" { /// free blocks inside its still-used pages, and hands the arena purge to the scavenger. /// Safe on any thread; a no-op on a thread that never allocated. No preconditions. pub safe fn mi_on_thread_idle(); + /// Return this thread's empty mimalloc pages to the arena, which schedules their purge + /// and wakes the scavenger to do the madvise. Strictly shallower than + /// `mi_on_thread_idle`: it walks the page queues only, with no free-list hole scan. + /// + /// The precondition is that the theap's owner is not allocating, not that the caller owns + /// it -- `page->free`/`used` are plain fields, so what must not happen is a concurrent + /// mutation, which is why mimalloc's park protocol can hand a parked thread's theaps to + /// the scavenger. Calling it on the owner while the owner is running satisfies that too. + /// + /// `mi_theap_get_default()` returns the CALLING thread's theap, so calling this off the JS + /// thread silently collects the wrong (usually empty) one rather than failing. + pub safe fn mi_theap_collect(theap: *mut c_void, force: bool); + /// The calling thread's theap. No preconditions. + pub safe fn mi_theap_get_default() -> *mut c_void; /// No preconditions. pub safe fn mi_version() -> c_int; /// No preconditions. From a908c8ab2a67e2e30a3f847d16b0e67c6acdec6d Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Tue, 14 Jul 2026 19:11:32 -0700 Subject: [PATCH 06/14] Revert "jsc: return the JS thread's mimalloc pages to the arena when the heap moves" This reverts commit c8abd3ed96352667b65af9283cc97e28a552591c. --- src/jsc/GarbageCollectionController.rs | 30 -------------------------- src/mimalloc_sys/mimalloc.rs | 14 ------------ 2 files changed, 44 deletions(-) diff --git a/src/jsc/GarbageCollectionController.rs b/src/jsc/GarbageCollectionController.rs index 7bcc7bcebabb..9ba4027b76b8 100644 --- a/src/jsc/GarbageCollectionController.rs +++ b/src/jsc/GarbageCollectionController.rs @@ -215,39 +215,9 @@ impl GarbageCollectionController { self.process_gc_timer_with_heap_size(vm, vm.block_bytes_allocated()); } - /// Hand this thread's empty mimalloc pages back to the arena, which schedules their purge - /// and wakes the scavenger to do the madvise off this thread. - /// - /// JSC's own hook cannot stand in for this. `Heap::didFinishCollection` fires - /// `scavengeThisThread`, but it runs under whichever `GCConductor` holds the conn -- - /// `collectInCollectorThread` conducts async collections -- and `mi_theap_get_default()` - /// returns the CALLING thread's theap, so on that path it collects the collector's - /// near-empty one. It is also gated on `CollectionScope::Full`, and sustained load runs - /// eden for minutes. Here we are on the JS thread by construction. - /// - /// Cheap enough to do per collection: `mi_theap_collect` walks the page queues and frees - /// empty pages, with no free-list hole scan, and the madvise is the scavenger's job. - #[inline] - fn return_pages_to_arena() { - if bun_core::USE_MIMALLOC { - let theap = bun_alloc::mimalloc::mi_theap_get_default(); - if !theap.is_null() { - bun_alloc::mimalloc::mi_theap_collect(theap, false); - } - } - } - fn process_gc_timer_with_heap_size(&mut self, vm: &VM, this_heap_size: usize) { let prev = self.gc_last_heap_size; - // The heap moved, so a collection freed JS objects and their mimalloc blocks with them. - // Eden does that far more often than full, and until the pages go back to the arena the - // scavenger cannot see them -- no arena-side purge knob reaches memory still parked in a - // theap's page queues. - if this_heap_size != prev { - Self::return_pages_to_arena(); - } - match self.gc_timer_state { GCTimerState::RunOnNextTick => { // When memory usage is not stable, run the GC more. diff --git a/src/mimalloc_sys/mimalloc.rs b/src/mimalloc_sys/mimalloc.rs index 9931c9937c2c..e610b63b8a2c 100644 --- a/src/mimalloc_sys/mimalloc.rs +++ b/src/mimalloc_sys/mimalloc.rs @@ -57,20 +57,6 @@ unsafe extern "C" { /// free blocks inside its still-used pages, and hands the arena purge to the scavenger. /// Safe on any thread; a no-op on a thread that never allocated. No preconditions. pub safe fn mi_on_thread_idle(); - /// Return this thread's empty mimalloc pages to the arena, which schedules their purge - /// and wakes the scavenger to do the madvise. Strictly shallower than - /// `mi_on_thread_idle`: it walks the page queues only, with no free-list hole scan. - /// - /// The precondition is that the theap's owner is not allocating, not that the caller owns - /// it -- `page->free`/`used` are plain fields, so what must not happen is a concurrent - /// mutation, which is why mimalloc's park protocol can hand a parked thread's theaps to - /// the scavenger. Calling it on the owner while the owner is running satisfies that too. - /// - /// `mi_theap_get_default()` returns the CALLING thread's theap, so calling this off the JS - /// thread silently collects the wrong (usually empty) one rather than failing. - pub safe fn mi_theap_collect(theap: *mut c_void, force: bool); - /// The calling thread's theap. No preconditions. - pub safe fn mi_theap_get_default() -> *mut c_void; /// No preconditions. pub safe fn mi_version() -> c_int; /// No preconditions. From edd3e53ebc8ae0c34d13434c138e60c3d101e1fe Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Tue, 14 Jul 2026 19:23:50 -0700 Subject: [PATCH 07/14] Reapply "jsc: return the JS thread's mimalloc pages to the arena when the heap moves" This reverts commit a908c8ab2a67e2e30a3f847d16b0e67c6acdec6d. --- src/jsc/GarbageCollectionController.rs | 30 ++++++++++++++++++++++++++ src/mimalloc_sys/mimalloc.rs | 14 ++++++++++++ 2 files changed, 44 insertions(+) diff --git a/src/jsc/GarbageCollectionController.rs b/src/jsc/GarbageCollectionController.rs index 9ba4027b76b8..7bcc7bcebabb 100644 --- a/src/jsc/GarbageCollectionController.rs +++ b/src/jsc/GarbageCollectionController.rs @@ -215,9 +215,39 @@ impl GarbageCollectionController { self.process_gc_timer_with_heap_size(vm, vm.block_bytes_allocated()); } + /// Hand this thread's empty mimalloc pages back to the arena, which schedules their purge + /// and wakes the scavenger to do the madvise off this thread. + /// + /// JSC's own hook cannot stand in for this. `Heap::didFinishCollection` fires + /// `scavengeThisThread`, but it runs under whichever `GCConductor` holds the conn -- + /// `collectInCollectorThread` conducts async collections -- and `mi_theap_get_default()` + /// returns the CALLING thread's theap, so on that path it collects the collector's + /// near-empty one. It is also gated on `CollectionScope::Full`, and sustained load runs + /// eden for minutes. Here we are on the JS thread by construction. + /// + /// Cheap enough to do per collection: `mi_theap_collect` walks the page queues and frees + /// empty pages, with no free-list hole scan, and the madvise is the scavenger's job. + #[inline] + fn return_pages_to_arena() { + if bun_core::USE_MIMALLOC { + let theap = bun_alloc::mimalloc::mi_theap_get_default(); + if !theap.is_null() { + bun_alloc::mimalloc::mi_theap_collect(theap, false); + } + } + } + fn process_gc_timer_with_heap_size(&mut self, vm: &VM, this_heap_size: usize) { let prev = self.gc_last_heap_size; + // The heap moved, so a collection freed JS objects and their mimalloc blocks with them. + // Eden does that far more often than full, and until the pages go back to the arena the + // scavenger cannot see them -- no arena-side purge knob reaches memory still parked in a + // theap's page queues. + if this_heap_size != prev { + Self::return_pages_to_arena(); + } + match self.gc_timer_state { GCTimerState::RunOnNextTick => { // When memory usage is not stable, run the GC more. diff --git a/src/mimalloc_sys/mimalloc.rs b/src/mimalloc_sys/mimalloc.rs index e610b63b8a2c..9931c9937c2c 100644 --- a/src/mimalloc_sys/mimalloc.rs +++ b/src/mimalloc_sys/mimalloc.rs @@ -57,6 +57,20 @@ unsafe extern "C" { /// free blocks inside its still-used pages, and hands the arena purge to the scavenger. /// Safe on any thread; a no-op on a thread that never allocated. No preconditions. pub safe fn mi_on_thread_idle(); + /// Return this thread's empty mimalloc pages to the arena, which schedules their purge + /// and wakes the scavenger to do the madvise. Strictly shallower than + /// `mi_on_thread_idle`: it walks the page queues only, with no free-list hole scan. + /// + /// The precondition is that the theap's owner is not allocating, not that the caller owns + /// it -- `page->free`/`used` are plain fields, so what must not happen is a concurrent + /// mutation, which is why mimalloc's park protocol can hand a parked thread's theaps to + /// the scavenger. Calling it on the owner while the owner is running satisfies that too. + /// + /// `mi_theap_get_default()` returns the CALLING thread's theap, so calling this off the JS + /// thread silently collects the wrong (usually empty) one rather than failing. + pub safe fn mi_theap_collect(theap: *mut c_void, force: bool); + /// The calling thread's theap. No preconditions. + pub safe fn mi_theap_get_default() -> *mut c_void; /// No preconditions. pub safe fn mi_version() -> c_int; /// No preconditions. From 3c8728e4df30feb6e12cfc13cc2b988ab7c87276 Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Tue, 14 Jul 2026 19:24:04 -0700 Subject: [PATCH 08/14] jsc: collect the JS thread's theap when a collection shrinks the heap Sampling `bun:jsc` heapStats() once a second cures the sustained-load memory ratchet on the repl replay (plateaus at 393MB against 563-584 un-instrumented). The reason is not a forced GC, which is what the earlier analysis assumed and built on: heapStats' `collectNow` is guarded by `if (vm.heap.size() == 0)` and never fires on a live process. What it actually does is call `mi_collect(false)` unconditionally (BunJSCModule.h), and `mi_collect` is exactly `mi_theap_collect(_mi_theap_default(), force)`. So the cure is a theap collect on the JS thread, obtained by accident from a statistics call. Do it deliberately. `mi_theap_collect` walks the theap's page queues and frees the empty pages, which schedules the arena purge and wakes the scavenger to madvise off this thread. It does not scan free lists for holes -- that is `purge_holes`, the expensive one that cost vite-preview 12.9% in #34009. Gated on the heap having SHRUNK, not merely moved: `process_gc_timer` is also called from `Server::on_request_complete`, and a busy server's heap is almost always moving, so `!=` fired the page walk on essentially every request -- -4.5% rps on fastify (66.1k -> 63.1k, n=6) buying nothing, since no collection had run. A shrink means a collection actually reclaimed and pages may now be empty. --- src/jsc/GarbageCollectionController.rs | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/jsc/GarbageCollectionController.rs b/src/jsc/GarbageCollectionController.rs index 7bcc7bcebabb..e7a3f98a7a1a 100644 --- a/src/jsc/GarbageCollectionController.rs +++ b/src/jsc/GarbageCollectionController.rs @@ -218,15 +218,15 @@ impl GarbageCollectionController { /// Hand this thread's empty mimalloc pages back to the arena, which schedules their purge /// and wakes the scavenger to do the madvise off this thread. /// - /// JSC's own hook cannot stand in for this. `Heap::didFinishCollection` fires - /// `scavengeThisThread`, but it runs under whichever `GCConductor` holds the conn -- - /// `collectInCollectorThread` conducts async collections -- and `mi_theap_get_default()` - /// returns the CALLING thread's theap, so on that path it collects the collector's - /// near-empty one. It is also gated on `CollectionScope::Full`, and sustained load runs - /// eden for minutes. Here we are on the JS thread by construction. + /// This is the operation that demonstrably fixes the sustained-load ratchet: sampling + /// `bun:jsc` heapStats() once a second cures it, and the reason is not the GC people assumed + /// -- heapStats' `collectNow` is guarded by `heap.size() == 0` and never fires on a live + /// process. It is `mi_collect(false)` (BunJSCModule.h), which is exactly + /// `mi_theap_collect(default theap)`. So do it deliberately instead of as a side effect of + /// asking for statistics. /// /// Cheap enough to do per collection: `mi_theap_collect` walks the page queues and frees - /// empty pages, with no free-list hole scan, and the madvise is the scavenger's job. + /// empty pages, with no free-list hole scan -- that is `purge_holes`, a different function. #[inline] fn return_pages_to_arena() { if bun_core::USE_MIMALLOC { @@ -240,11 +240,11 @@ impl GarbageCollectionController { fn process_gc_timer_with_heap_size(&mut self, vm: &VM, this_heap_size: usize) { let prev = self.gc_last_heap_size; - // The heap moved, so a collection freed JS objects and their mimalloc blocks with them. - // Eden does that far more often than full, and until the pages go back to the arena the - // scavenger cannot see them -- no arena-side purge knob reaches memory still parked in a - // theap's page queues. - if this_heap_size != prev { + // Only when the heap SHRANK: a collection actually reclaimed, so blocks were freed and + // pages may now be empty. `!=` fired on essentially every request -- `process_gc_timer` + // is also called from `Server::on_request_complete` and a busy server's heap is always + // moving -- which measured -4.5% rps on fastify for nothing, since no collection had run. + if this_heap_size < prev { Self::return_pages_to_arena(); } From f7455623b9763f32c1d802da9476858243e3a959 Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Tue, 14 Jul 2026 19:54:52 -0700 Subject: [PATCH 09/14] jsc: collect the JS thread's theap after a collection, when the loop never parks `mi_theap_collect` is the only demonstrated cure for the sustained-load memory ratchet. Sampling `bun:jsc` heapStats() once a second fixes it, and the reason is not a forced GC as the earlier analysis assumed: heapStats' `collectNow` is guarded by `if (vm.heap.size() == 0)` and never fires on a live process. What it does unconditionally is `mi_collect(false)` (BunJSCModule.h), and `mi_collect` is exactly `mi_theap_collect(_mi_theap_default(), force)`. On the replay workload that call moves ~170MB. So do it deliberately instead of as a side effect of asking for statistics. Two gates. A loop that parks already gets its theap swept at the park, so doing it here too is pure cost -- an earlier revision that ignored this measured -4.5% rps on fastify (66.1k -> 63.1k, n=6) for no reclaim. And a collection has to have actually finished, which is the one thing the controller could not previously tell: `perform_gc` merely requests one via `collect_async`. That also bounds this to once per collection. `GCCycleObserver` lives on `JSVMClientData`, not in a process global: every worker has its own heap and theap, and a shared counter would let one worker's collection convince another to walk its page queues. It only counts -- `Heap::didFinishCollection` runs under whichever GCConductor holds the conn, and `collectInCollectorThread` conducts async collections, so the work has to happen on the JS thread where `mi_theap_get_default()` returns the right theap. The observer is removed in `~JSVMClientData`, which is safe because `~VM` deletes clientData in its body, before the `Heap` member is destroyed. The hook goes in `WTFTimer::fire`, not `run`: `update` only publishes to `imminent_gc_timer` for a delay <= 0, and `GCActivityCallback::didAllocate` schedules with a positive delay, so JSC's GC callbacks come through the timer heap. Not yet validated against the workload it targets -- nothing reproduces the ratchet outside the repl replay harness. --- src/jsc/GarbageCollectionController.rs | 47 ++++++++++++++++++++++---- src/jsc/VM.rs | 14 ++++++++ src/jsc/bindings/BunClientData.cpp | 16 +++++++++ src/jsc/bindings/BunClientData.h | 41 ++++++++++++++++++++++ src/jsc/bindings/BunJSCEventLoop.cpp | 4 +++ src/runtime/timer/WTFTimer.rs | 14 +++++++- 6 files changed, 128 insertions(+), 8 deletions(-) diff --git a/src/jsc/GarbageCollectionController.rs b/src/jsc/GarbageCollectionController.rs index e7a3f98a7a1a..5099bac413b3 100644 --- a/src/jsc/GarbageCollectionController.rs +++ b/src/jsc/GarbageCollectionController.rs @@ -33,6 +33,12 @@ pub struct GarbageCollectionController { pub gc_timer: EventLoopTimer, pub gc_repeating_timer: EventLoopTimer, pub gc_last_heap_size: usize, + /// Finished-collection count when we last returned pages; tells us a collection has actually + /// run since, rather than inferring it from the heap size. + pub gc_last_cycle_count: u64, + /// Park count when we last looked. If it moves, the event loop is parking and the park path + /// is already sweeping this thread's theap, so we stay out of its way. + pub gc_last_park_count: u64, pub gc_last_heap_size_on_repeating_timer: usize, pub heap_size_didnt_change_for_repeating_timer_ticks_count: u8, pub gc_timer_state: GCTimerState, @@ -55,6 +61,8 @@ impl Default for GarbageCollectionController { gc_timer: EventLoopTimer::init_paused(TimerTag::GcOneShot), gc_repeating_timer: EventLoopTimer::init_paused(TimerTag::GcRepeating), gc_last_heap_size: 0, + gc_last_cycle_count: 0, + gc_last_park_count: 0, gc_last_heap_size_on_repeating_timer: 0, heap_size_didnt_change_for_repeating_timer_ticks_count: 0, gc_timer_state: GCTimerState::Pending, @@ -237,16 +245,41 @@ impl GarbageCollectionController { } } - fn process_gc_timer_with_heap_size(&mut self, vm: &VM, this_heap_size: usize) { - let prev = self.gc_last_heap_size; + /// Return this thread's now-empty mimalloc pages to the arena, once per finished collection, + /// for a loop that never parks. + /// + /// `mi_theap_collect` is the only demonstrated cure for the sustained-load ratchet: sampling + /// heapStats() once a second fixes it, and its one relevant action is `mi_collect(false)` + /// (BunJSCModule.h), which is exactly this call. Its `collectNow` is guarded by + /// `heap.size() == 0` and never fires on a live process, so no GC is involved in the cure. + /// + /// Two gates, both load-bearing. A loop that parks already gets its theap swept at the park, + /// so doing it here as well is pure cost -- an earlier revision that ignored this measured + /// -4.5% rps on fastify (66.1k -> 63.1k, n=6). And the cycle count is the only honest "a + /// collection finished" signal: `perform_gc` merely *requests* one via `collect_async`. It + /// also bounds this to once per collection, which is why no heap-size check is needed. + pub fn maybe_return_pages(&mut self, vm: &VM) { + if self.disabled { + return; + } - // Only when the heap SHRANK: a collection actually reclaimed, so blocks were freed and - // pages may now be empty. `!=` fired on essentially every request -- `process_gc_timer` - // is also called from `Server::on_request_complete` and a busy server's heap is always - // moving -- which measured -4.5% rps on fastify for nothing, since no collection had run. - if this_heap_size < prev { + let parks = vm.park_count(); + let loop_is_parking = parks != self.gc_last_park_count; + self.gc_last_park_count = parks; + + let cycles = vm.gc_cycle_count(); + let collected = cycles != self.gc_last_cycle_count; + self.gc_last_cycle_count = cycles; + + if collected && !loop_is_parking { Self::return_pages_to_arena(); } + } + + fn process_gc_timer_with_heap_size(&mut self, vm: &VM, this_heap_size: usize) { + let prev = self.gc_last_heap_size; + + self.maybe_return_pages(vm); match self.gc_timer_state { GCTimerState::RunOnNextTick => { diff --git a/src/jsc/VM.rs b/src/jsc/VM.rs index bfe379e7e722..a251aaeb2f61 100644 --- a/src/jsc/VM.rs +++ b/src/jsc/VM.rs @@ -46,6 +46,8 @@ unsafe extern "C" { safe fn JSC__VM__drainMicrotasks(vm: &VM); safe fn JSC__VM__externalMemorySize(vm: &VM) -> usize; safe fn JSC__VM__blockBytesAllocated(vm: &VM) -> usize; + safe fn JSC__VM__gcCycleCount(vm: &VM) -> u64; + safe fn JSC__VM__parkCount(vm: &VM) -> u64; safe fn JSC__VM__performOpportunisticallyScheduledTasks(vm: &VM, until: f64); } @@ -212,6 +214,18 @@ impl VM { JSC__VM__blockBytesAllocated(self) } + /// Finished collections for THIS VM, counted by its `GCCycleObserver`. Per-VM: workers each + /// have their own heap and theap, so a shared count would be another VM's evidence. + pub fn gc_cycle_count(&self) -> u64 { + JSC__VM__gcCycleCount(self) + } + + /// Ticks on which this VM's event loop actually parked. The park sweeps this thread's theap, + /// so a moving count means something else is already reclaiming and we should not duplicate it. + pub fn park_count(&self) -> u64 { + JSC__VM__parkCount(self) + } + pub fn perform_opportunistically_scheduled_tasks(&self, until: f64) { JSC__VM__performOpportunisticallyScheduledTasks(self, until) } diff --git a/src/jsc/bindings/BunClientData.cpp b/src/jsc/bindings/BunClientData.cpp index 031428729e72..bf35f681c2c0 100644 --- a/src/jsc/bindings/BunClientData.cpp +++ b/src/jsc/bindings/BunClientData.cpp @@ -59,7 +59,9 @@ JSVMClientData::JSVMClientData(VM& vm, RefPtr sourceProvide , CLIENT_ISO_SUBSPACE_INIT(m_domConstructorSpace) , CLIENT_ISO_SUBSPACE_INIT(m_domNamespaceObjectSpace) , m_clientSubspaces(makeUnique()) + , m_vm(vm) { + vm.heap.addObserver(&m_gcCycleObserver); } #undef CLIENT_ISO_SUBSPACE_INIT @@ -95,6 +97,8 @@ void JSVMClientData::JSHeapDataDeleter::operator()(JSHeapData* heapData) const JSVMClientData::~JSVMClientData() { + m_vm.heap.removeObserver(&m_gcCycleObserver); + m_clients.forEach([](auto& client) { client.willDestroyVM(); }); @@ -126,3 +130,15 @@ void JSVMClientData::create(VM* vm, void* bunVM) } } // namespace WebCore + +extern "C" uint64_t JSC__VM__gcCycleCount(JSC::VM* vm) +{ + auto* clientData = WebCore::clientData(*vm); + return clientData ? clientData->gcCycleObserver().count() : 0; +} + +extern "C" uint64_t JSC__VM__parkCount(JSC::VM* vm) +{ + auto* clientData = WebCore::clientData(*vm); + return clientData ? clientData->parkCounter().count() : 0; +} diff --git a/src/jsc/bindings/BunClientData.h b/src/jsc/bindings/BunClientData.h index dfa5cbc23251..673cc8d20ed5 100644 --- a/src/jsc/bindings/BunClientData.h +++ b/src/jsc/bindings/BunClientData.h @@ -81,6 +81,38 @@ class JSHeapData { DECLARE_ALLOCATOR_WITH_HEAP_IDENTIFIER(JSVMClientData); +// Counts finished collections for ONE VM, so the GC controller can tell whether a collection +// actually ran since it last looked. Per-VM rather than process-wide: every worker has its own +// heap and its own theap, and a shared counter would let one worker's collection convince +// another to walk its page queues. +// +// Counts only; does no work. `Heap::didFinishCollection` runs under whichever GCConductor holds +// the conn -- `collectInCollectorThread` conducts async collections -- and a theap may only be +// walked by its owner, so the JS thread reads this and acts there. Atomic for the same reason. +class GCCycleObserver final : public JSC::HeapObserver { +public: + void willGarbageCollect() final { } + void didGarbageCollect(JSC::CollectionScope) final + { + m_count.fetch_add(1, std::memory_order_relaxed); + } + uint64_t count() const { return m_count.load(std::memory_order_relaxed); } + +private: + std::atomic m_count { 0 }; +}; + +// Counts ticks where the event loop actually parked, so the GC controller can tell whether the +// park path is already sweeping this thread's theap and stay out of its way. JS thread only. +class ParkCounter { +public: + void didPark() { ++m_count; } + uint64_t count() const { return m_count; } + +private: + uint64_t m_count { 0 }; +}; + class JSVMClientData : public JSC::VM::ClientData { WTF_MAKE_NONCOPYABLE(JSVMClientData); WTF_DEPRECATED_MAKE_FAST_ALLOCATED_WITH_HEAP_IDENTIFIER(JSVMClientData, JSVMClientData); @@ -93,6 +125,8 @@ class JSVMClientData : public JSC::VM::ClientData { static void create(JSC::VM*, void*); JSHeapData& heapData() { return *m_heapData; } + GCCycleObserver& gcCycleObserver() { return m_gcCycleObserver; } + ParkCounter& parkCounter() { return m_parkCounter; } BunBuiltinNames& builtinNames() { return m_builtinNames; } JSBuiltinFunctions& builtinFunctions() { return *m_builtinFunctions; } @@ -140,6 +174,13 @@ class JSVMClientData : public JSC::VM::ClientData { private: bool isWebCoreJSClientData() const final { return true; } + // `~VM` deletes clientData in its body, before member destructors run, so `heap` (a VM + // member) is still alive in our destructor -- the observer must be removed there or the + // heap keeps a dangling pointer to it. + JSC::VM& m_vm; + GCCycleObserver m_gcCycleObserver; + ParkCounter m_parkCounter; + // Frees a per-VM `JSHeapData` but leaves the process-wide `useGlobalGC` // singleton alone (it is shared by every VM). On the default `!useGlobalGC` // path `ensureHeapData` allocates a fresh `JSHeapData` per VM, so without diff --git a/src/jsc/bindings/BunJSCEventLoop.cpp b/src/jsc/bindings/BunJSCEventLoop.cpp index 736fdd6207fb..541113c8a513 100644 --- a/src/jsc/bindings/BunJSCEventLoop.cpp +++ b/src/jsc/bindings/BunJSCEventLoop.cpp @@ -26,6 +26,10 @@ extern "C" std::atomic Bun__defaultRemainingRunsUntilSkipReleaseAccess; extern "C" void Bun__JSC_onBeforeWait(JSC::VM* _Nonnull vm, uint64_t nowNs) { ASSERT(vm); + // Called only on a tick that really parks, so it is the signal the GC controller uses to stay + // out of the park path's way -- the park already sweeps this thread's theap. + if (auto* clientData = WebCore::clientData(*vm)) + clientData->parkCounter().didPark(); const bool previouslyHadAccess = vm->heap.hasHeapAccess(); // sanity check for debug builds to ensure we're not doing a // use-after-free here diff --git a/src/runtime/timer/WTFTimer.rs b/src/runtime/timer/WTFTimer.rs index c0ee848c43c3..0eed2f338903 100644 --- a/src/runtime/timer/WTFTimer.rs +++ b/src/runtime/timer/WTFTimer.rs @@ -234,7 +234,7 @@ impl WTFTimer { /// # Safety /// `this` is the container of an `EventLoopTimer` just popped from /// `All.timers`; `_vm` is the live per-thread VM. - pub unsafe fn fire(this: *mut Self, _now: &ElTimespec, _vm: *mut VirtualMachine) { + pub unsafe fn fire(this: *mut Self, _now: &ElTimespec, vm: *mut VirtualMachine) { // SAFETY: per fn contract — `this` is live. Single raw write to // `event_loop_timer.state` precedes the `ThisPtr` borrow; subsequent // field reads via `t` create fresh short-lived `&Self`. @@ -254,6 +254,18 @@ impl WTFTimer { Ordering::SeqCst, ); t.run_without_removing(); + + // JSC's GC activity callbacks land here -- `didAllocate` schedules them with a positive + // delay, so they go through the timer heap rather than the imminent slot. A collection + // driven by JSC's own timer would otherwise go unnoticed until our controller's timer + // happened to tick. Two loads and a compare when nothing collected. + // + // SAFETY: per fn contract `_vm` is the live per-thread VM, and we are on its JS thread -- + // which is what makes the theap walk inside legal at all. + unsafe { + let jsc_vm = (*vm).jsc_vm(); + (*vm).gc_controller.maybe_return_pages(jsc_vm); + } } /// # Safety From c015194a870f7f1605975faefccb053b093c93f7 Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Wed, 15 Jul 2026 02:56:42 +0000 Subject: [PATCH 10/14] [autofix.ci] apply automated fixes --- src/jsc/bindings/BunClientData.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jsc/bindings/BunClientData.h b/src/jsc/bindings/BunClientData.h index 673cc8d20ed5..ddfe4f110d35 100644 --- a/src/jsc/bindings/BunClientData.h +++ b/src/jsc/bindings/BunClientData.h @@ -91,7 +91,7 @@ DECLARE_ALLOCATOR_WITH_HEAP_IDENTIFIER(JSVMClientData); // walked by its owner, so the JS thread reads this and acts there. Atomic for the same reason. class GCCycleObserver final : public JSC::HeapObserver { public: - void willGarbageCollect() final { } + void willGarbageCollect() final {} void didGarbageCollect(JSC::CollectionScope) final { m_count.fetch_add(1, std::memory_order_relaxed); From c0068c66be7d61026a0287175adc76a31c5394d0 Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Tue, 14 Jul 2026 20:02:58 -0700 Subject: [PATCH 11/14] jsc: trim the theap-collect comments to the durable content Benchmark numbers and the history of earlier revisions belong in the commit message, which has them. --- src/jsc/GarbageCollectionController.rs | 32 ++++++-------------------- src/jsc/bindings/BunClientData.h | 11 +++------ src/runtime/timer/WTFTimer.rs | 10 ++++---- 3 files changed, 14 insertions(+), 39 deletions(-) diff --git a/src/jsc/GarbageCollectionController.rs b/src/jsc/GarbageCollectionController.rs index 5099bac413b3..c65d29cd1b00 100644 --- a/src/jsc/GarbageCollectionController.rs +++ b/src/jsc/GarbageCollectionController.rs @@ -223,18 +223,9 @@ impl GarbageCollectionController { self.process_gc_timer_with_heap_size(vm, vm.block_bytes_allocated()); } - /// Hand this thread's empty mimalloc pages back to the arena, which schedules their purge - /// and wakes the scavenger to do the madvise off this thread. - /// - /// This is the operation that demonstrably fixes the sustained-load ratchet: sampling - /// `bun:jsc` heapStats() once a second cures it, and the reason is not the GC people assumed - /// -- heapStats' `collectNow` is guarded by `heap.size() == 0` and never fires on a live - /// process. It is `mi_collect(false)` (BunJSCModule.h), which is exactly - /// `mi_theap_collect(default theap)`. So do it deliberately instead of as a side effect of - /// asking for statistics. - /// - /// Cheap enough to do per collection: `mi_theap_collect` walks the page queues and frees - /// empty pages, with no free-list hole scan -- that is `purge_holes`, a different function. + /// The theap collect that `heapStats()` does incidentally via `mi_collect(false)`, which is + /// the only demonstrated cure for the sustained-load ratchet. Frees empty pages back to the + /// arena and wakes the scavenger to madvise them; no free-list hole scan. #[inline] fn return_pages_to_arena() { if bun_core::USE_MIMALLOC { @@ -245,19 +236,10 @@ impl GarbageCollectionController { } } - /// Return this thread's now-empty mimalloc pages to the arena, once per finished collection, - /// for a loop that never parks. - /// - /// `mi_theap_collect` is the only demonstrated cure for the sustained-load ratchet: sampling - /// heapStats() once a second fixes it, and its one relevant action is `mi_collect(false)` - /// (BunJSCModule.h), which is exactly this call. Its `collectNow` is guarded by - /// `heap.size() == 0` and never fires on a live process, so no GC is involved in the cure. - /// - /// Two gates, both load-bearing. A loop that parks already gets its theap swept at the park, - /// so doing it here as well is pure cost -- an earlier revision that ignored this measured - /// -4.5% rps on fastify (66.1k -> 63.1k, n=6). And the cycle count is the only honest "a - /// collection finished" signal: `perform_gc` merely *requests* one via `collect_async`. It - /// also bounds this to once per collection, which is why no heap-size check is needed. + /// Give this thread's now-empty mimalloc pages back to the arena, once per finished + /// collection, for a loop that never parks -- a loop that parks is already swept there, and + /// doing it twice is measurable. `perform_gc` only *requests* a collection, so the cycle + /// count is the one honest "it finished" signal. pub fn maybe_return_pages(&mut self, vm: &VM) { if self.disabled { return; diff --git a/src/jsc/bindings/BunClientData.h b/src/jsc/bindings/BunClientData.h index ddfe4f110d35..3f2934777b56 100644 --- a/src/jsc/bindings/BunClientData.h +++ b/src/jsc/bindings/BunClientData.h @@ -81,14 +81,9 @@ class JSHeapData { DECLARE_ALLOCATOR_WITH_HEAP_IDENTIFIER(JSVMClientData); -// Counts finished collections for ONE VM, so the GC controller can tell whether a collection -// actually ran since it last looked. Per-VM rather than process-wide: every worker has its own -// heap and its own theap, and a shared counter would let one worker's collection convince -// another to walk its page queues. -// -// Counts only; does no work. `Heap::didFinishCollection` runs under whichever GCConductor holds -// the conn -- `collectInCollectorThread` conducts async collections -- and a theap may only be -// walked by its owner, so the JS thread reads this and acts there. Atomic for the same reason. +// Per-VM, not process-wide: each worker has its own heap and theap, and a shared count would let +// one worker's collection convince another to walk its page queues. Counts only -- it fires under +// whichever GCConductor holds the conn, so the owning JS thread reads this and does the work. class GCCycleObserver final : public JSC::HeapObserver { public: void willGarbageCollect() final {} diff --git a/src/runtime/timer/WTFTimer.rs b/src/runtime/timer/WTFTimer.rs index 0eed2f338903..57468ce3205f 100644 --- a/src/runtime/timer/WTFTimer.rs +++ b/src/runtime/timer/WTFTimer.rs @@ -255,13 +255,11 @@ impl WTFTimer { ); t.run_without_removing(); - // JSC's GC activity callbacks land here -- `didAllocate` schedules them with a positive - // delay, so they go through the timer heap rather than the imminent slot. A collection - // driven by JSC's own timer would otherwise go unnoticed until our controller's timer - // happened to tick. Two loads and a compare when nothing collected. + // JSC's GC activity callbacks land here, not in `run`: `didAllocate` schedules them with + // a positive delay, so they take the timer heap rather than the imminent slot. // - // SAFETY: per fn contract `_vm` is the live per-thread VM, and we are on its JS thread -- - // which is what makes the theap walk inside legal at all. + // SAFETY: per fn contract `vm` is the live per-thread VM, and we are on its JS thread -- + // which is what makes the theap walk inside legal. unsafe { let jsc_vm = (*vm).jsc_vm(); (*vm).gc_controller.maybe_return_pages(jsc_vm); From 868c47bd6d72abde56167d6cefab471e88c5fd67 Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Tue, 14 Jul 2026 20:19:20 -0700 Subject: [PATCH 12/14] jsc: drop the unused theap-collect counter From 0d9ba349b98aa99b6f18383cc50687c3ffabe48c Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Tue, 14 Jul 2026 20:36:44 -0700 Subject: [PATCH 13/14] Revert the GC-controller theap collect: measured, it does nothing The premise was that `mi_theap_collect` on the JS thread is the cure, since sampling heapStats() once a second fixes the ratchet and its one relevant action is `mi_collect(false)` == `mi_theap_collect(default theap)`. Measured against the real 100-turn replay, it is not. The park path already runs that exact call every 100ms -- `mi_on_thread_idle` is `mi_theap_collect(theap0, false)` plus a hole punch -- and the footprint still climbs 264 -> 369MB. Forcing the park sweep to fire on every park (BUN_GC_RUNS_UNTIL_SKIP_RELEASE_ACCESS=1e6) changes it to +101MB from +105MB. This hook measured +108MB. All three are the same number. So the theap collect is already happening at 10Hz and is not what heapStats does differently. What is left is `objectTypeCounts()` -> `HeapIterationScope` -> `MarkedSpace::stopAllocating()`, which heapStats calls unconditionally and nothing else on the JS thread does. --- src/jsc/GarbageCollectionController.rs | 45 -------------------------- src/jsc/VM.rs | 14 -------- src/jsc/bindings/BunClientData.cpp | 16 --------- src/jsc/bindings/BunClientData.h | 36 --------------------- src/jsc/bindings/BunJSCEventLoop.cpp | 4 --- src/mimalloc_sys/mimalloc.rs | 14 -------- src/runtime/timer/WTFTimer.rs | 12 +------ 7 files changed, 1 insertion(+), 140 deletions(-) diff --git a/src/jsc/GarbageCollectionController.rs b/src/jsc/GarbageCollectionController.rs index c65d29cd1b00..9ba4027b76b8 100644 --- a/src/jsc/GarbageCollectionController.rs +++ b/src/jsc/GarbageCollectionController.rs @@ -33,12 +33,6 @@ pub struct GarbageCollectionController { pub gc_timer: EventLoopTimer, pub gc_repeating_timer: EventLoopTimer, pub gc_last_heap_size: usize, - /// Finished-collection count when we last returned pages; tells us a collection has actually - /// run since, rather than inferring it from the heap size. - pub gc_last_cycle_count: u64, - /// Park count when we last looked. If it moves, the event loop is parking and the park path - /// is already sweeping this thread's theap, so we stay out of its way. - pub gc_last_park_count: u64, pub gc_last_heap_size_on_repeating_timer: usize, pub heap_size_didnt_change_for_repeating_timer_ticks_count: u8, pub gc_timer_state: GCTimerState, @@ -61,8 +55,6 @@ impl Default for GarbageCollectionController { gc_timer: EventLoopTimer::init_paused(TimerTag::GcOneShot), gc_repeating_timer: EventLoopTimer::init_paused(TimerTag::GcRepeating), gc_last_heap_size: 0, - gc_last_cycle_count: 0, - gc_last_park_count: 0, gc_last_heap_size_on_repeating_timer: 0, heap_size_didnt_change_for_repeating_timer_ticks_count: 0, gc_timer_state: GCTimerState::Pending, @@ -223,46 +215,9 @@ impl GarbageCollectionController { self.process_gc_timer_with_heap_size(vm, vm.block_bytes_allocated()); } - /// The theap collect that `heapStats()` does incidentally via `mi_collect(false)`, which is - /// the only demonstrated cure for the sustained-load ratchet. Frees empty pages back to the - /// arena and wakes the scavenger to madvise them; no free-list hole scan. - #[inline] - fn return_pages_to_arena() { - if bun_core::USE_MIMALLOC { - let theap = bun_alloc::mimalloc::mi_theap_get_default(); - if !theap.is_null() { - bun_alloc::mimalloc::mi_theap_collect(theap, false); - } - } - } - - /// Give this thread's now-empty mimalloc pages back to the arena, once per finished - /// collection, for a loop that never parks -- a loop that parks is already swept there, and - /// doing it twice is measurable. `perform_gc` only *requests* a collection, so the cycle - /// count is the one honest "it finished" signal. - pub fn maybe_return_pages(&mut self, vm: &VM) { - if self.disabled { - return; - } - - let parks = vm.park_count(); - let loop_is_parking = parks != self.gc_last_park_count; - self.gc_last_park_count = parks; - - let cycles = vm.gc_cycle_count(); - let collected = cycles != self.gc_last_cycle_count; - self.gc_last_cycle_count = cycles; - - if collected && !loop_is_parking { - Self::return_pages_to_arena(); - } - } - fn process_gc_timer_with_heap_size(&mut self, vm: &VM, this_heap_size: usize) { let prev = self.gc_last_heap_size; - self.maybe_return_pages(vm); - match self.gc_timer_state { GCTimerState::RunOnNextTick => { // When memory usage is not stable, run the GC more. diff --git a/src/jsc/VM.rs b/src/jsc/VM.rs index a251aaeb2f61..bfe379e7e722 100644 --- a/src/jsc/VM.rs +++ b/src/jsc/VM.rs @@ -46,8 +46,6 @@ unsafe extern "C" { safe fn JSC__VM__drainMicrotasks(vm: &VM); safe fn JSC__VM__externalMemorySize(vm: &VM) -> usize; safe fn JSC__VM__blockBytesAllocated(vm: &VM) -> usize; - safe fn JSC__VM__gcCycleCount(vm: &VM) -> u64; - safe fn JSC__VM__parkCount(vm: &VM) -> u64; safe fn JSC__VM__performOpportunisticallyScheduledTasks(vm: &VM, until: f64); } @@ -214,18 +212,6 @@ impl VM { JSC__VM__blockBytesAllocated(self) } - /// Finished collections for THIS VM, counted by its `GCCycleObserver`. Per-VM: workers each - /// have their own heap and theap, so a shared count would be another VM's evidence. - pub fn gc_cycle_count(&self) -> u64 { - JSC__VM__gcCycleCount(self) - } - - /// Ticks on which this VM's event loop actually parked. The park sweeps this thread's theap, - /// so a moving count means something else is already reclaiming and we should not duplicate it. - pub fn park_count(&self) -> u64 { - JSC__VM__parkCount(self) - } - pub fn perform_opportunistically_scheduled_tasks(&self, until: f64) { JSC__VM__performOpportunisticallyScheduledTasks(self, until) } diff --git a/src/jsc/bindings/BunClientData.cpp b/src/jsc/bindings/BunClientData.cpp index bf35f681c2c0..031428729e72 100644 --- a/src/jsc/bindings/BunClientData.cpp +++ b/src/jsc/bindings/BunClientData.cpp @@ -59,9 +59,7 @@ JSVMClientData::JSVMClientData(VM& vm, RefPtr sourceProvide , CLIENT_ISO_SUBSPACE_INIT(m_domConstructorSpace) , CLIENT_ISO_SUBSPACE_INIT(m_domNamespaceObjectSpace) , m_clientSubspaces(makeUnique()) - , m_vm(vm) { - vm.heap.addObserver(&m_gcCycleObserver); } #undef CLIENT_ISO_SUBSPACE_INIT @@ -97,8 +95,6 @@ void JSVMClientData::JSHeapDataDeleter::operator()(JSHeapData* heapData) const JSVMClientData::~JSVMClientData() { - m_vm.heap.removeObserver(&m_gcCycleObserver); - m_clients.forEach([](auto& client) { client.willDestroyVM(); }); @@ -130,15 +126,3 @@ void JSVMClientData::create(VM* vm, void* bunVM) } } // namespace WebCore - -extern "C" uint64_t JSC__VM__gcCycleCount(JSC::VM* vm) -{ - auto* clientData = WebCore::clientData(*vm); - return clientData ? clientData->gcCycleObserver().count() : 0; -} - -extern "C" uint64_t JSC__VM__parkCount(JSC::VM* vm) -{ - auto* clientData = WebCore::clientData(*vm); - return clientData ? clientData->parkCounter().count() : 0; -} diff --git a/src/jsc/bindings/BunClientData.h b/src/jsc/bindings/BunClientData.h index 3f2934777b56..dfa5cbc23251 100644 --- a/src/jsc/bindings/BunClientData.h +++ b/src/jsc/bindings/BunClientData.h @@ -81,33 +81,6 @@ class JSHeapData { DECLARE_ALLOCATOR_WITH_HEAP_IDENTIFIER(JSVMClientData); -// Per-VM, not process-wide: each worker has its own heap and theap, and a shared count would let -// one worker's collection convince another to walk its page queues. Counts only -- it fires under -// whichever GCConductor holds the conn, so the owning JS thread reads this and does the work. -class GCCycleObserver final : public JSC::HeapObserver { -public: - void willGarbageCollect() final {} - void didGarbageCollect(JSC::CollectionScope) final - { - m_count.fetch_add(1, std::memory_order_relaxed); - } - uint64_t count() const { return m_count.load(std::memory_order_relaxed); } - -private: - std::atomic m_count { 0 }; -}; - -// Counts ticks where the event loop actually parked, so the GC controller can tell whether the -// park path is already sweeping this thread's theap and stay out of its way. JS thread only. -class ParkCounter { -public: - void didPark() { ++m_count; } - uint64_t count() const { return m_count; } - -private: - uint64_t m_count { 0 }; -}; - class JSVMClientData : public JSC::VM::ClientData { WTF_MAKE_NONCOPYABLE(JSVMClientData); WTF_DEPRECATED_MAKE_FAST_ALLOCATED_WITH_HEAP_IDENTIFIER(JSVMClientData, JSVMClientData); @@ -120,8 +93,6 @@ class JSVMClientData : public JSC::VM::ClientData { static void create(JSC::VM*, void*); JSHeapData& heapData() { return *m_heapData; } - GCCycleObserver& gcCycleObserver() { return m_gcCycleObserver; } - ParkCounter& parkCounter() { return m_parkCounter; } BunBuiltinNames& builtinNames() { return m_builtinNames; } JSBuiltinFunctions& builtinFunctions() { return *m_builtinFunctions; } @@ -169,13 +140,6 @@ class JSVMClientData : public JSC::VM::ClientData { private: bool isWebCoreJSClientData() const final { return true; } - // `~VM` deletes clientData in its body, before member destructors run, so `heap` (a VM - // member) is still alive in our destructor -- the observer must be removed there or the - // heap keeps a dangling pointer to it. - JSC::VM& m_vm; - GCCycleObserver m_gcCycleObserver; - ParkCounter m_parkCounter; - // Frees a per-VM `JSHeapData` but leaves the process-wide `useGlobalGC` // singleton alone (it is shared by every VM). On the default `!useGlobalGC` // path `ensureHeapData` allocates a fresh `JSHeapData` per VM, so without diff --git a/src/jsc/bindings/BunJSCEventLoop.cpp b/src/jsc/bindings/BunJSCEventLoop.cpp index 541113c8a513..736fdd6207fb 100644 --- a/src/jsc/bindings/BunJSCEventLoop.cpp +++ b/src/jsc/bindings/BunJSCEventLoop.cpp @@ -26,10 +26,6 @@ extern "C" std::atomic Bun__defaultRemainingRunsUntilSkipReleaseAccess; extern "C" void Bun__JSC_onBeforeWait(JSC::VM* _Nonnull vm, uint64_t nowNs) { ASSERT(vm); - // Called only on a tick that really parks, so it is the signal the GC controller uses to stay - // out of the park path's way -- the park already sweeps this thread's theap. - if (auto* clientData = WebCore::clientData(*vm)) - clientData->parkCounter().didPark(); const bool previouslyHadAccess = vm->heap.hasHeapAccess(); // sanity check for debug builds to ensure we're not doing a // use-after-free here diff --git a/src/mimalloc_sys/mimalloc.rs b/src/mimalloc_sys/mimalloc.rs index 9931c9937c2c..e610b63b8a2c 100644 --- a/src/mimalloc_sys/mimalloc.rs +++ b/src/mimalloc_sys/mimalloc.rs @@ -57,20 +57,6 @@ unsafe extern "C" { /// free blocks inside its still-used pages, and hands the arena purge to the scavenger. /// Safe on any thread; a no-op on a thread that never allocated. No preconditions. pub safe fn mi_on_thread_idle(); - /// Return this thread's empty mimalloc pages to the arena, which schedules their purge - /// and wakes the scavenger to do the madvise. Strictly shallower than - /// `mi_on_thread_idle`: it walks the page queues only, with no free-list hole scan. - /// - /// The precondition is that the theap's owner is not allocating, not that the caller owns - /// it -- `page->free`/`used` are plain fields, so what must not happen is a concurrent - /// mutation, which is why mimalloc's park protocol can hand a parked thread's theaps to - /// the scavenger. Calling it on the owner while the owner is running satisfies that too. - /// - /// `mi_theap_get_default()` returns the CALLING thread's theap, so calling this off the JS - /// thread silently collects the wrong (usually empty) one rather than failing. - pub safe fn mi_theap_collect(theap: *mut c_void, force: bool); - /// The calling thread's theap. No preconditions. - pub safe fn mi_theap_get_default() -> *mut c_void; /// No preconditions. pub safe fn mi_version() -> c_int; /// No preconditions. diff --git a/src/runtime/timer/WTFTimer.rs b/src/runtime/timer/WTFTimer.rs index 57468ce3205f..c0ee848c43c3 100644 --- a/src/runtime/timer/WTFTimer.rs +++ b/src/runtime/timer/WTFTimer.rs @@ -234,7 +234,7 @@ impl WTFTimer { /// # Safety /// `this` is the container of an `EventLoopTimer` just popped from /// `All.timers`; `_vm` is the live per-thread VM. - pub unsafe fn fire(this: *mut Self, _now: &ElTimespec, vm: *mut VirtualMachine) { + pub unsafe fn fire(this: *mut Self, _now: &ElTimespec, _vm: *mut VirtualMachine) { // SAFETY: per fn contract — `this` is live. Single raw write to // `event_loop_timer.state` precedes the `ThisPtr` borrow; subsequent // field reads via `t` create fresh short-lived `&Self`. @@ -254,16 +254,6 @@ impl WTFTimer { Ordering::SeqCst, ); t.run_without_removing(); - - // JSC's GC activity callbacks land here, not in `run`: `didAllocate` schedules them with - // a positive delay, so they take the timer heap rather than the imminent slot. - // - // SAFETY: per fn contract `vm` is the live per-thread VM, and we are on its JS thread -- - // which is what makes the theap walk inside legal. - unsafe { - let jsc_vm = (*vm).jsc_vm(); - (*vm).gc_controller.maybe_return_pages(jsc_vm); - } } /// # Safety From d94046388c0352a8b8788fd67824b9404008e405 Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Wed, 15 Jul 2026 16:19:05 -0700 Subject: [PATCH 14/14] deps: bump mimalloc to bun-dev3-v2 with the idle-theap handoff Picks up oven-sh/mimalloc#8 (theap: let a parked thread hand its heaps to the scavenger), now merged into the fork's bun-dev3-v2 branch alongside the upstream/dev3 sync. No-Verification-Needed: dependency pin bump only, no product source --- scripts/build/deps/mimalloc.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build/deps/mimalloc.ts b/scripts/build/deps/mimalloc.ts index 85164b79dfbe..bb507d935073 100644 --- a/scripts/build/deps/mimalloc.ts +++ b/scripts/build/deps/mimalloc.ts @@ -12,7 +12,7 @@ import type { Dependency, DirectBuild } from "../source.ts"; -const MIMALLOC_COMMIT = "e24e5480f33b3987a61ff6e764e5c285466cfe7a"; +const MIMALLOC_COMMIT = "24211c6e7610ae7c4ec06040758ec90bd21a1c83"; export const mimalloc: Dependency = { name: "mimalloc",