From 6b33f91b4b69d44d92e7c5e691842bc70af7787b Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Thu, 13 Aug 2026 19:00:09 +0000 Subject: [PATCH] Keep the idle sweep's state on the tld being swept instead of in __thread variables The hole-purging sweep kept its re-entrancy guard and per-sweep counters in __thread variables and read the guard from mi_page_free_collect_ex, inside the allocator. Where __thread is emulated (Android before API 29, bun's Android target) the first access on a thread calls malloc, so with mimalloc being malloc the allocator re-entered itself from inside a page collect and, whenever the size class emulated TLS asked for was itself on an exhausted page, recursed until the stack overflowed (oven-sh/bun#38051). The previous pin happened to write another __thread variable during thread init, which materialized each thread's emulated-TLS state while the thread had no pages yet; the dev3 sync removed it and the crash became frequent. The sweep always runs for one specific tld (the owner's own, or the parked thread's when the scavenger sweeps for it), so that tld now carries the state of the running sweep. The guard is read back through the calling thread's default theap, which every TLS model reads without allocating, and the sweep's own collects ask for no un-purging explicitly instead of relying on the guard. The only __thread variable left on the allocator's paths is the message-output guard in options.c, which is only reached when a warning is printed. test-emulated-tls builds the allocator once more with -femulated-tls, as malloc, in the pthreads TLS model, and makes a fresh thread's first page collect happen with the page of every small size bin exhausted in turn, and then has a fresh thread allocate into pages whose holes the scavenger has just purged, in every bin. Without this change it overflows the stack in optimized builds (part 1), overflows it in unoptimized builds (part 2), and trips mi_page_is_valid_init on the scavenger in assertion builds, because the emulated guard write gave the scavenger a theap of its own. --- CMakeLists.txt | 27 ++++++ include/mimalloc/internal.h | 8 +- include/mimalloc/types.h | 10 ++ src/arena.c | 16 ++-- src/page.c | 97 ++++++++++--------- src/theap.c | 18 ++-- test/test-emulated-tls.c | 184 ++++++++++++++++++++++++++++++++++++ 7 files changed, 298 insertions(+), 62 deletions(-) create mode 100644 test/test-emulated-tls.c diff --git a/CMakeLists.txt b/CMakeLists.txt index a1143193b..2e0957d4b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -869,6 +869,33 @@ if (MI_BUILD_TESTS) # and sweep every park, so the reclaim path is hit as hard as possible add_test(NAME test-park-handoff-eager COMMAND ${CMAKE_COMMAND} -E env MIMALLOC_PURGE_HOLES_MIN_INTERVAL=0 $) + # The allocator must never touch one of its own `__thread` variables from inside an allocation: + # where `__thread` is emulated (Android before API 29) the first access on a thread mallocs, and + # with the allocator being malloc that is a re-entrant allocation (oven-sh/bun#38051). Build the + # allocator once more with emulated TLS, as malloc, in the TLS model Android uses, and link + # test/test-emulated-tls.c against that build. Only compilers that can emulate TLS (clang); the + # sanitizers bring their own malloc. + if(MI_OVERRIDE AND NOT WIN32 AND NOT (MI_TRACK_ASAN OR MI_TRACK_VALGRIND OR MI_DEBUG_TSAN OR MI_DEBUG_UBSAN)) + include(CheckCCompilerFlag) + check_c_compiler_flag("-femulated-tls" MI_HAS_EMULATED_TLS) + if(MI_HAS_EMULATED_TLS) + set(mi_emutls_defines ${mi_defines}) + list(FILTER mi_emutls_defines EXCLUDE REGEX "^MI_TLS_MODEL_") + add_library(mimalloc-emulated-tls STATIC ${mi_sources}) + target_compile_definitions(mimalloc-emulated-tls PRIVATE ${mi_emutls_defines} MI_STATIC_LIB MI_MALLOC_OVERRIDE MI_TLS_MODEL_PTHREADS=1) + target_compile_options(mimalloc-emulated-tls PRIVATE ${mi_cflags} -femulated-tls) + target_include_directories(mimalloc-emulated-tls PUBLIC $) + target_link_libraries(mimalloc-emulated-tls PRIVATE ${mi_libraries}) + add_executable(mimalloc-test-emulated-tls test/test-emulated-tls.c) + target_compile_definitions(mimalloc-test-emulated-tls PRIVATE ${mi_emutls_defines}) + target_compile_options(mimalloc-test-emulated-tls PRIVATE ${mi_cflags}) + target_link_libraries(mimalloc-test-emulated-tls PRIVATE mimalloc-emulated-tls ${mi_libraries}) + add_test(NAME test-emulated-tls COMMAND mimalloc-test-emulated-tls) + else() + message(STATUS "The compiler cannot emulate TLS: skipping test-emulated-tls") + endif() + endif() + # heap snapshot reader CLI (standalone, no mimalloc dependency) add_executable(mi-heapview tools/mi-heapview.c) target_compile_options(mi-heapview PRIVATE ${mi_cflags}) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 88d943d10..a5798b0a6 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -889,7 +889,7 @@ static inline bool mi_page_all_free(const mi_page_t* page) { // Page hole purging (see the "Page hole purging" section in `page.c`) // ------------------------------------------------------ -void _mi_page_purge_holes(mi_page_t* page); +void _mi_page_purge_holes(mi_page_t* page, mi_tld_t* tld); // `tld`: the thread whose sweep this is (see `_mi_page_purge_holes_begin`) void _mi_page_purged_reset(mi_page_t* page); bool _mi_page_unpurge_run(mi_page_t* page); void _mi_page_unpurge_all(mi_page_t* page); @@ -898,12 +898,12 @@ void _mi_page_unpurge_unformed_upto(mi_page_t* page, uintptr_t end); size_t _mi_page_unformed_purged_bytes(const mi_page_t* page); // the bytes of this page's unformed tail that are discarded right now bool _mi_page_purge_os_page_blocks(size_t os_page_size, size_t block_size, uintptr_t page_start, size_t capacity, size_t k, size_t* first, size_t* last); -bool _mi_page_purge_holes_in_progress(void); +bool _mi_page_purge_holes_in_progress(void); // is the calling thread inside a sweep of its own heaps? void _mi_page_holes_count_page_freed(void); void _mi_page_holes_count_ineligible(const mi_page_t* page); void _mi_page_holes_reset_ineligible(void); -void _mi_page_purge_holes_begin(void); -void _mi_page_purge_holes_end(void); +void _mi_page_purge_holes_begin(mi_tld_t* tld); // around each pass of a sweep; `tld` is the thread being swept +void _mi_page_purge_holes_end(mi_tld_t* tld); void _mi_page_purge_holes_sweep_begin(mi_tld_t* tld); // once per idle sweep, before its passes // ------------------------------------------------------ diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index d7c2689be..11fa776bb 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -709,6 +709,16 @@ struct mi_tld_s { mi_tld_t* subproc_next; // list of tlds in the subproc, so the scavenger can find parked threads size_t holes_sweep_seq; // idle sweeps run over THIS tld's heaps (paces `purge_holes_full_every`) mi_msecs_t holes_sweep_last; // when this tld's heaps were last swept (paces `purge_holes_min_interval`) + + // State of the sweep that is currently running over this tld's heaps (on the owner, or on the + // scavenger while the owner is parked). It lives here and not in thread-locals of the sweeping + // thread: `_mi_page_purge_holes_in_progress` is read inside the allocator, and where `__thread` + // is emulated (Android before API 29) the first access to a thread-local on a thread calls + // malloc, which re-enters the very page collect that is reading it (oven-sh/bun#38051). + bool holes_sweeping; // a pass of the sweep is rewriting free lists right now (see `_mi_page_purge_holes_begin`) + bool holes_sweep_full; // this sweep ignores `page->swept_state` (see `_mi_page_purge_holes`) + size_t holes_sweep_skipped; // per-sweep counters, folded into the process-wide ones in `_mi_page_purge_holes_end` + size_t holes_sweep_visited; }; diff --git a/src/arena.c b/src/arena.c index af7e8fed5..7845abbe9 100644 --- a/src/arena.c +++ b/src/arena.c @@ -1329,7 +1329,7 @@ void _mi_arenas_page_unabandon(mi_page_t* page, mi_theap_t* current_theapx) { typedef struct mi_purge_holes_arg_s { mi_bitmap_t* bitmap; - mi_tld_t* tld; // whose park we are sweeping under; NULL when not a parked sweep + mi_tld_t* tld; // the thread whose sweep this is (its own, or the parked one the scavenger is sweeping for) } mi_purge_holes_arg_t; static bool mi_arena_page_purge_holes_at(size_t slice_index, size_t slice_count, mi_arena_t* arena, void* arg) { @@ -1337,7 +1337,7 @@ static bool mi_arena_page_purge_holes_at(size_t slice_index, size_t slice_count, mi_purge_holes_arg_t* const parg = (mi_purge_holes_arg_t*)arg; // this pass holds every page that ever became full, so it is most of a cold sweep: an owner // waiting in `_mi_park_leave` cannot allocate until we stop - if (parg->tld != NULL && mi_atomic_load_relaxed(&parg->tld->park_reclaim) != 0) return false; + if (mi_atomic_load_relaxed(&parg->tld->park_reclaim) != 0) return false; mi_bitmap_t* const bitmap = parg->bitmap; // Take the page out of the abandoned map first: this is the reader side of the protocol in @@ -1353,8 +1353,8 @@ static bool mi_arena_page_purge_holes_at(size_t slice_index, size_t slice_count, } // We own the page: no other thread can reclaim, unabandon, or free it now, and only the - // atomic `xthread_free` can still change under us. - _mi_page_free_collect(page, true); + // atomic `xthread_free` can still change under us. (No un-purging: we are about to purge.) + _mi_page_free_collect_no_unpurge(page, true); if (mi_page_all_free(page)) { mi_bitmap_set(bitmap, slice_index); // `_mi_arenas_page_unabandon` expects it in the map _mi_arenas_page_unabandon(page, NULL); @@ -1362,7 +1362,7 @@ static bool mi_arena_page_purge_holes_at(size_t slice_index, size_t slice_count, _mi_page_holes_count_page_freed(); return true; } - _mi_page_purge_holes(page); + _mi_page_purge_holes(page, parg->tld); mi_bitmap_set(bitmap, slice_index); // back in the map *before* unowning: unown may free the page mi_abandoned_page_unown(page, NULL); return true; @@ -1372,9 +1372,9 @@ static bool mi_arena_page_purge_holes_at(size_t slice_index, size_t slice_count, // A page abandoned while full is not mapped; it has no free blocks at that point, and once // enough blocks are freed in it, `_mi_arenas_page_try_reabandon_to_mapped` puts it in the map. void _mi_arenas_purge_abandoned_holes(mi_heap_t* heap, mi_tld_t* tld) { - if (heap == NULL) return; + if (heap == NULL || tld == NULL) return; if (!mi_option_is_enabled(mi_option_purge_holes)) return; - _mi_page_purge_holes_begin(); + _mi_page_purge_holes_begin(tld); mi_forall_arenas(heap, ((mi_arena_t*)NULL), 0, arena) { mi_arena_pages_t* const arena_pages = mi_heap_arena_pages(heap, arena); if (arena_pages != NULL) { @@ -1389,7 +1389,7 @@ void _mi_arenas_purge_abandoned_holes(mi_heap_t* heap, mi_tld_t* tld) { } } mi_forall_arenas_end(); - _mi_page_purge_holes_end(); + _mi_page_purge_holes_end(tld); } // The read-only counterpart of the sweep above: account for the holes in the abandoned pages diff --git a/src/page.c b/src/page.c index 382d7d92c..42a659ab5 100644 --- a/src/page.c +++ b/src/page.c @@ -472,44 +472,54 @@ static void mi_holes_count_reuse(size_t bytes, size_t blocks, bool reused) { mi_atomic_addi64_relaxed(&mi_holes_blocks, -(int64_t)blocks); } -// Re-entrancy guard: while the idle sweep is rewriting a page's free list and -// bitmap, a nested `mi_malloc` (only reachable through a user output function -// from a warning message) must not un-purge a hole from under it. -static mi_decl_thread bool mi_purging_holes; - -// Per-sweep counters. The sweep runs over every page of the thread, so a process-wide atomic -// per page would be a real cost on the very path we are making cheap: accumulate thread-locally -// and fold them in once per pass, in `_mi_page_purge_holes_end`. -static mi_decl_thread size_t mi_holes_sweep_skipped; -static mi_decl_thread size_t mi_holes_sweep_visited; - -// Whether THIS sweep ignores `page->swept_state` (see `_mi_page_purge_holes`). Set once per idle -// sweep, in `_mi_page_purge_holes_sweep_begin`; the sequence it is paced by lives on the tld being -// swept, since one scavenger thread runs the sweeps of many. -static mi_decl_thread bool mi_holes_sweep_full; - -bool _mi_page_purge_holes_in_progress(void) { return mi_purging_holes; } -void _mi_page_purge_holes_begin(void) { mi_assert_internal(!mi_purging_holes); mi_purging_holes = true; } - -void _mi_page_purge_holes_end(void) { - mi_assert_internal(mi_purging_holes); - mi_purging_holes = false; - if (mi_holes_sweep_skipped > 0) { - mi_atomic_addi64_relaxed(&mi_holes_pages_skipped, (int64_t)mi_holes_sweep_skipped); - mi_holes_sweep_skipped = 0; - } - if (mi_holes_sweep_visited > 0) { - mi_atomic_addi64_relaxed(&mi_holes_blocks_visited, (int64_t)mi_holes_sweep_visited); - mi_holes_sweep_visited = 0; - } -} - -// Called once per idle sweep of `tld`'s heaps, before its passes (`mi_purge_holes_of`). +// The state of a running sweep lives on the tld being swept (`tld->holes_sweep*`, see `types.h`), +// never in thread-locals of the sweeping thread. Besides the scavenger sweeping many tlds from one +// thread, this must not touch a `__thread` variable at all: `_mi_page_purge_holes_in_progress` is +// read from `mi_page_free_collect_ex`, inside the allocator, and on targets where `__thread` is +// emulated (Android before API 29) the first access on a thread allocates -- re-entering the +// collect that is reading it, without bound (oven-sh/bun#38051). The tld of the calling thread is +// reached through the default theap, which every TLS model can read without allocating. + +// Re-entrancy guard: while a sweep is rewriting a page's free list and bitmap, a nested `mi_malloc` +// on the sweeping thread (only reachable through a user output function from a warning message) +// must not un-purge a hole from under it. That nested allocation comes out of the calling thread's +// own theaps, so it is its own tld that matters here: on the owner that is the tld being swept; +// the scavenger has no theaps of its own being swept (it sweeps a parked thread's theaps, and the +// abandoned pages it touches are claimed), so un-purging there is harmless. +bool _mi_page_purge_holes_in_progress(void) { + mi_theap_t* const theap = _mi_theap_default(); + if (theap == NULL || theap->tld == NULL) return false; + return theap->tld->holes_sweeping; +} + +void _mi_page_purge_holes_begin(mi_tld_t* tld) { + mi_assert_internal(tld != NULL && !tld->holes_sweeping); + tld->holes_sweeping = true; +} + +// Also folds the per-sweep counters into the process-wide ones. The sweep runs over every page of +// the thread, so a process-wide atomic per page would be a real cost on the very path we are making +// cheap: they accumulate on the tld and are folded in once per pass. +void _mi_page_purge_holes_end(mi_tld_t* tld) { + mi_assert_internal(tld != NULL && tld->holes_sweeping); + tld->holes_sweeping = false; + if (tld->holes_sweep_skipped > 0) { + mi_atomic_addi64_relaxed(&mi_holes_pages_skipped, (int64_t)tld->holes_sweep_skipped); + tld->holes_sweep_skipped = 0; + } + if (tld->holes_sweep_visited > 0) { + mi_atomic_addi64_relaxed(&mi_holes_blocks_visited, (int64_t)tld->holes_sweep_visited); + tld->holes_sweep_visited = 0; + } +} + +// Called once per idle sweep of `tld`'s heaps, before its passes (`mi_purge_holes_of`): decides +// whether this sweep ignores `page->swept_state` (see `_mi_page_purge_holes`). void _mi_page_purge_holes_sweep_begin(mi_tld_t* tld) { const long every = mi_option_get(mi_option_purge_holes_full_every); const size_t seq = ++tld->holes_sweep_seq; - mi_holes_sweep_full = (every > 0 && (seq % (size_t)every) == 0); - if (mi_holes_sweep_full) { mi_atomic_addi64_relaxed(&mi_holes_full_sweeps, 1); } + tld->holes_sweep_full = (every > 0 && (seq % (size_t)every) == 0); + if (tld->holes_sweep_full) { mi_atomic_addi64_relaxed(&mi_holes_full_sweeps, 1); } } static inline bool mi_page_bits_at(const uint64_t* bits, size_t k) { @@ -678,7 +688,7 @@ void _mi_page_unpurge_unformed_upto(mi_page_t* page, uintptr_t end) { // Walk the free list of a page and discard every OS page in it that holds no live block. // Returns false if any discard failed: those blocks went straight back on the free list and the // page must be swept again, so the caller must not record it as swept. -static bool mi_page_purge_holes_walk(mi_page_t* page) { +static bool mi_page_purge_holes_walk(mi_page_t* page, mi_tld_t* tld) { if (page->free == NULL) return true; // nothing to take off the free list const size_t os_size = _mi_os_page_size(); @@ -703,7 +713,7 @@ static bool mi_page_purge_holes_walk(mi_page_t* page) { nfree[k]++; } } - mi_holes_sweep_visited += nvisited; // folded into the process-wide counter at the end of the sweep + tld->holes_sweep_visited += nvisited; // folded into the process-wide counter at the end of the pass // 2. an OS page can be discarded when *every* block overlapping it is free -- either on the // free list, or purged already. Of the blocks overlapping an OS page, only the first and @@ -789,26 +799,27 @@ static bool mi_page_purge_holes_walk(mi_page_t* page) { // regardless, which caps the delay of a missed discard at N parks for 1/N of the old cost. // (An exact "was anything freed in this page" bit is the alternative, and it costs a store in // `mi_free` itself -- the hot path this whole feature stays off.) -void _mi_page_purge_holes(mi_page_t* page) { - mi_assert_internal(page != NULL); +void _mi_page_purge_holes(mi_page_t* page, mi_tld_t* tld) { + mi_assert_internal(page != NULL && tld != NULL); + mi_assert_internal(tld->holes_sweeping); if (!mi_option_is_enabled(mi_option_purge_holes)) return; if (mi_page_all_free(page)) return; // the page itself is about to be freed if (mi_option_get(mi_option_purge_delay) < 0) return; // purging disabled mi_page_purge_unformed_tail(page); // the blocks that are not formed yet: resident, but never handed out if (!mi_page_can_purge_holes(page)) { _mi_page_holes_count_ineligible(page); return; } - if (!mi_holes_sweep_full && page->swept_state == mi_page_sweep_state(page)) { - mi_holes_sweep_skipped++; // nothing was allocated or freed in this page since we swept it + if (!tld->holes_sweep_full && page->swept_state == mi_page_sweep_state(page)) { + tld->holes_sweep_skipped++; // nothing was allocated or freed in this page since we swept it return; } // Record the state we LEAVE the page in, read back from the page: a nested `mi_malloc` (see - // `mi_purging_holes`) may have taken a block out of it while we walked. + // `_mi_page_purge_holes_in_progress`) may have taken a block out of it while we walked. // // Only if the walk got everything. A failed `_mi_os_discard` (ENOMEM under pressure) puts its // blocks straight back, and changes neither `capacity` nor `used` -- so recording here would // say "already swept" for a page that still has holes, and the skip check would then park them // until the next full sweep, or forever with `purge_holes_full_every=0`. - if (mi_page_purge_holes_walk(page)) { + if (mi_page_purge_holes_walk(page, tld)) { page->swept_state = mi_page_sweep_state(page); } } diff --git a/src/theap.c b/src/theap.c index 1a0a9463d..fc37cfcdd 100644 --- a/src/theap.c +++ b/src/theap.c @@ -165,20 +165,23 @@ void _mi_theap_collect_abandon(mi_theap_t* theap) { // that are still partially used. Meant to be called when the application knows // it is idle (e.g. from an event loop about to park): it costs a few madvise // calls and nothing on the alloc/free hot path. -static bool mi_theap_page_purge_holes(mi_theap_t* theap, mi_page_queue_t* pq, mi_page_t* page, void* arg1, void* arg2) { - MI_UNUSED(arg1); MI_UNUSED(arg2); +static bool mi_theap_page_purge_holes(mi_theap_t* theap, mi_page_queue_t* pq, mi_page_t* page, void* arg_tld, void* arg2) { + MI_UNUSED(arg2); + mi_tld_t* const tld = (mi_tld_t*)arg_tld; // the tld being swept (== theap->tld) // When the scavenger is doing this for a parked thread, the owner may wake at any moment and // has to wait for us. Stopping between pages bounds that wait to one page's walk; the pages we // skip are simply swept at the next park (`swept_state` makes the re-walk cheap). if (theap->tld != NULL && mi_atomic_load_relaxed(&theap->tld->park_reclaim) != 0) return false; - _mi_page_free_collect(page, true); // force: fold local_free (and thread_free) into `free` first + // force: fold local_free (and thread_free) into `free` first. Never un-purge here: we are about + // to purge, and a run brought back now would be discarded again right away (see `mi_theap_page_collect`). + _mi_page_free_collect_no_unpurge(page, true); if (mi_page_all_free(page)) { // the forced collect emptied the page: hand it back instead of leaving it resident _mi_page_holes_count_page_freed(); _mi_page_free(page, pq); return true; } - _mi_page_purge_holes(page); + _mi_page_purge_holes(page, tld); mi_assert_expensive(_mi_page_is_valid(page)); return true; // continue } @@ -193,9 +196,10 @@ static void mi_theap_purge_holes(mi_theap_t* theap) mi_attr_noexcept { if (theap->tld == NULL) return; if (theap->tld->thread_id != _mi_thread_id() && mi_atomic_load_acquire(&theap->tld->park_state) != MI_PARK_SWEEPING) return; - _mi_page_purge_holes_begin(); - mi_theap_visit_pages(theap, &mi_theap_page_purge_holes, true /* include full pages */, NULL, NULL); - _mi_page_purge_holes_end(); + mi_tld_t* const tld = theap->tld; + _mi_page_purge_holes_begin(tld); + mi_theap_visit_pages(theap, &mi_theap_page_purge_holes, true /* include full pages */, tld, NULL); + _mi_page_purge_holes_end(tld); } // Purge the holes in every page this thread may safely touch: diff --git a/test/test-emulated-tls.c b/test/test-emulated-tls.c new file mode 100644 index 000000000..277986a3c --- /dev/null +++ b/test/test-emulated-tls.c @@ -0,0 +1,184 @@ +// The allocator must not touch a `__thread` variable of its own from inside an allocation or a +// page collect. On targets where `__thread` is emulated (Android before API 29, which is what +// bun's Android build targets), every access goes through `__emutls_get_address`, and the first +// access on a thread materializes the variable -- and the thread's per-variable address array -- +// with malloc(). With mimalloc being malloc, that re-enters the allocator from wherever the access +// sits. The idle sweep's guard used to be such a variable, read from `mi_page_free_collect_ex`: +// the collect called into emulated TLS, emulated TLS called malloc, malloc had to collect a page +// of exactly the size class it was asked for, that collect read the (still unmaterialized) guard +// again, and so on until the stack was gone (oven-sh/bun#38051). +// +// This binary is linked against a build of the allocator compiled with `-femulated-tls` and +// malloc overriding (see CMakeLists.txt), so emulated TLS allocates from the allocator under test, +// exactly as on Android. It then arranges for a thread's first page collect to happen while the +// page of whichever size class emulated TLS is going to ask for is exhausted. Every small size +// class gets a turn, so the test does not depend on the sizes a particular emutls runtime asks for +// (compiler-rt and libgcc differ), and a bug of this kind shows up as a stack overflow instead of +// a failed check. With a correct allocator nothing here ever reaches emulated TLS at all. +#include +#include +#include +#include +#include +#include +#include +#include + +// Request sizes covering every size bin up to 1 KiB (several of these share a bin; part 2 below +// only uses one request size per bin). +static const size_t sizes[] = { 8, 16, 24, 32, 40, 48, 56, 64, 80, 96, 112, 128, 160, 192, 224, 256, + 320, 384, 448, 512, 640, 768, 896, 1024 }; +#define NSIZES (sizeof(sizes) / sizeof(sizes[0])) +// Per class: stays inside one (64 KiB) small page, yet spans several OS pages even with 16 KiB ones. +#define BYTES_PER_CLASS (48 * 1024) +#define MAX_BLOCKS (BYTES_PER_CLASS / 8) +#define EXTEND_BYTES 4096 // a page formats this many bytes of blocks at a time (MI_MAX_EXTEND_SIZE) + +static void* blocks[MAX_BLOCKS]; +static uintptr_t os_page; // the OS page size: the granularity at which the sweep discards + +static void run_thread(void* (*fun)(void*), void* arg) { + pthread_t th; + if (pthread_create(&th, NULL, fun, arg) != 0) { printf(" pthread_create failed\n"); exit(2); } + pthread_join(th, NULL); +} + +// --------------------------------------------------------------------------------------------- +// 1. A fresh thread allocates one size class until its page runs out of formatted blocks. The +// allocation that runs out collects the page (`mi_page_queue_find_free_ex`), which is the first +// collect on this thread. An optimizing compiler hoists a thread-local read in that collect +// ahead of the conditions guarding it, so with the bug this is where emulated TLS allocates, +// and the thread whose size class matches the emutls request recurses into the same exhausted +// page without bound. +// --------------------------------------------------------------------------------------------- +static void* exhaust_one_class(void* arg) { + const size_t size = *(const size_t*)arg; + const size_t n = BYTES_PER_CLASS / size; + for (size_t i = 0; i < n; i++) { + blocks[i] = malloc(size); + if (blocks[i] == NULL) { printf(" out of memory\n"); exit(2); } + memset(blocks[i], (int)i, size); + } + for (size_t i = 0; i < n; i++) free(blocks[i]); + return NULL; +} + +// --------------------------------------------------------------------------------------------- +// 2. The unoptimized variant of the same read only happens for a page with purged holes, so also +// produce those: a fresh thread leaves, in every size class, a page whose free blocks all sit in +// OS pages without a live block, parks, and lets the scavenger sweep it. The sweep discards +// those OS pages and takes every one of those blocks off the free lists. The next allocation in +// any class has to collect a page with purged holes and an empty free list; with the bug that +// reads the guard, emulated TLS allocates, and that allocation lands on another such page. +// --------------------------------------------------------------------------------------------- +static int cmp_ptr(const void* a, const void* b) { + const uintptr_t x = *(const uintptr_t*)a, y = *(const uintptr_t*)b; + return (x < y ? -1 : (x > y ? 1 : 0)); +} + +typedef struct { void** live; size_t count; } kept_t; + +// Allocate a fresh page's worth of one bin, then keep exactly the blocks lying entirely inside the +// first and the last OS page the blocks touch (those two OS pages can never be discarded, so their +// blocks must not be free either) and free everything else: every OS page in between then holds +// only free blocks. The count is a whole number of extensions so that no formatted block is left +// over next to the kept ones, which would stay on the free list and serve the allocation later. +static void prepare_class(size_t size, kept_t* kept) { + static uintptr_t addrs[MAX_BLOCKS]; + addrs[0] = (uintptr_t)malloc(size); + addrs[1] = (uintptr_t)malloc(size); + if (addrs[0] == 0 || addrs[1] == 0) { printf(" out of memory\n"); exit(2); } + const size_t stride = (addrs[1] > addrs[0] ? addrs[1] - addrs[0] : addrs[0] - addrs[1]); // the block size, padding included + if (stride < size || stride > 2 * size + 64) { printf(" unexpected block layout for size %zu (stride %zu)\n", size, stride); exit(2); } + const size_t per_extension = EXTEND_BYTES / stride; + const size_t n = (BYTES_PER_CLASS / EXTEND_BYTES) * per_extension; + for (size_t i = 2; i < n; i++) { + addrs[i] = (uintptr_t)malloc(size); + if (addrs[i] == 0) { printf(" out of memory\n"); exit(2); } + } + qsort(addrs, n, sizeof(addrs[0]), &cmp_ptr); + const uintptr_t first_page = addrs[0] & ~(os_page - 1); + const uintptr_t last_page = (addrs[n - 1] + stride - 1) & ~(os_page - 1); + kept->count = 0; + for (size_t i = 0; i < n; i++) { + const uintptr_t lo = addrs[i], hi = lo + stride; + const int in_first = (lo >= first_page && hi <= first_page + os_page); + const int in_last = (lo >= last_page && hi <= last_page + os_page); + if (in_first || in_last) { kept->live[kept->count++] = (void*)lo; } + else { free((void*)lo); } + } +} + +static void* purged_pages_in_every_class(void* arg) { + (void)arg; + static void* live[NSIZES][MAX_BLOCKS]; // what prepare_class keeps alive, per class + static size_t class_size[NSIZES]; // one request size per bin + kept_t kept[NSIZES]; + size_t nclasses = 0; + for (size_t s = 0; s < NSIZES; s++) { + if (s > 0 && mi_good_size(sizes[s]) == mi_good_size(sizes[s - 1])) continue; // same bin as the previous size + class_size[nclasses] = sizes[s]; + kept[nclasses].live = live[nclasses]; + prepare_class(sizes[s], &kept[nclasses]); + nclasses++; + } + + mi_purge_holes_stats_t before, now; + mi_purge_holes_stats_get(&before); + if (!mi_on_thread_idle_start()) { + printf(" (no scavenger to hand the heaps to: skipping the purged-pages part)\n"); + } + else { + // Wait for the scavenger to sweep us: until the discard count has risen and then stayed put + // for a while (taking the heaps back stops a sweep that is still running). Nothing in this + // loop may allocate. + const struct timespec ms = { 0, 1000 * 1000 }; + size_t last = before.purged_blocks; + int stable = 0, waited = 0; + do { + nanosleep(&ms, NULL); + mi_purge_holes_stats_get(&now); + if (now.purged_blocks == last) { if (last > before.purged_blocks) stable++; } + else { last = now.purged_blocks; stable = 0; } + } while (stable < 100 && ++waited < 20000); + mi_on_thread_idle_end(); + if (now.purged_blocks <= before.purged_blocks) { + printf(" (the scavenger did not sweep this thread within 20s: skipping the purged-pages part)\n"); + } + else { + printf(" swept: %zu blocks held off the free lists in %zu classes\n", (size_t)(now.purged_blocks - before.purged_blocks), nclasses); + for (size_t c = 0; c < nclasses; c++) { + void* p = malloc(class_size[c]); // collects a page with purged holes and an empty free list + if (p == NULL) { printf(" out of memory\n"); exit(2); } + memset(p, 1, class_size[c]); + free(p); + } + mi_purge_holes_stats_t after; + mi_purge_holes_stats_get(&after); + printf(" handed back: %zu hole runs\n", (size_t)(after.reuse_calls - now.reuse_calls)); + if (after.reuse_calls == now.reuse_calls) { + // then the allocations above never collected a purged page and this part tested nothing + printf(" FAILED: the allocations after the sweep did not reach a page with purged holes\n"); + exit(1); + } + } + } + for (size_t c = 0; c < nclasses; c++) { + for (size_t i = 0; i < kept[c].count; i++) free(kept[c].live[i]); + } + return NULL; +} + +int main(void) { + const long ps = sysconf(_SC_PAGESIZE); + os_page = (ps > 0 ? (uintptr_t)ps : 4096); + printf("exhausting one size class per fresh thread...\n"); + for (size_t s = 0; s < NSIZES; s++) { + size_t size = sizes[s]; + run_thread(&exhaust_one_class, &size); + } + printf("collecting pages with purged holes on a fresh thread...\n"); + run_thread(&purged_pages_in_every_class, NULL); + printf("ok\n"); + return 0; +}