diff --git a/CMakeLists.txt b/CMakeLists.txt index 96adcf5b1..6d4b2e15b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -874,7 +874,7 @@ if (MI_BUILD_TESTS) enable_testing() # static link tests - set(mi_static_tests api api-fill stress-heaps stress-subprocs stress heap-mt heap-delete-race heap-churn fork-user-heap snapshot prof prof-adversarial theap-sentinel purge-holes purge-zero park-handoff) + set(mi_static_tests api api-fill stress-heaps stress-subprocs stress heap-mt heap-delete-race heap-churn fork-user-heap snapshot prof prof-adversarial theap-sentinel purge-holes purge-zero park-handoff freelist-corruption) if (MI_DEBUG_FULL OR CMAKE_BUILD_TYPE MATCHES "Debug") list(APPEND mi_static_tests commit-fail) # uses mi_debug_fail_os_commit_after (MI_DEBUG>0 only) endif() diff --git a/src/page.c b/src/page.c index 57a5c3eb1..ad0a0660d 100644 --- a/src/page.c +++ b/src/page.c @@ -252,6 +252,107 @@ bool _mi_page_is_valid(mi_page_t* page) { #endif +/* ----------------------------------------------------------- + Free list corruption. + + A free block carries its `next` link in its own first word, so a write into a block after + it was freed, or a stale free that linked a still-live object, leaves a link that can point + anywhere. The allocation path pops a link without looking at it; the cold paths that read a + whole list (`mi_page_thread_collect_to_local`, the forced collect in `mi_page_free_collect_ex`, + and the idle sweep's `mi_page_purge_holes_walk`) are the ones that reach such a link first + in practice, and without a check they fault on it (oven-sh/bun BUN-40BH and its siblings). + So they check every link: a valid link is NULL or the start of a formed block of this page, + a list never holds more blocks than the page has free, and (in the sweep, which counts the + blocks) no block is on it twice. (`MI_ENCODE_FREELIST` builds already reject a link that + leaves the page in `mi_block_next`; release builds do not encode.) The checks cost a division + per block walked. Of the three walks only the thread-free collect is reached from allocation + (`_mi_malloc_generic`), and it already chases every link it collects; popping `page->free` + in `_mi_page_malloc` is not touched. +----------------------------------------------------------- */ + +#define MI_PAGE_MAX_CAPACITY (1 << 16) // `page->capacity` is a uint16_t (`mi_page_sweep_state_fits` checks that) + +// Is `block` the start of a formed block of `page`? On success `*idx` is its block index. +static inline bool mi_page_block_index_of(const mi_page_t* page, const mi_block_t* block, size_t* idx) { + const uintptr_t offset = (uintptr_t)block - (uintptr_t)mi_page_start(page); // wraps around when `block` lies before the page + const size_t i = (size_t)(offset / page->block_size); + *idx = i; + return (i < page->capacity && (uintptr_t)(i * page->block_size) == offset); +} + +// The most blocks the free lists of this page can hold between them: the blocks not in use. +// (A block whose memory is discarded is free as well but on no list, so this is an upper bound.) +static inline size_t mi_page_max_free_listed(const mi_page_t* page) { + return (page->used <= page->capacity ? (size_t)(page->capacity - page->used) : 0); +} + +// Cut the list rooted at `*list` so that it ends at `last` (NULL empties it). What is cut off +// is the block holding a bad link (possibly a live object, or a block something still writes +// to), a block that is on the list for the second time, or the blocks past the page's free +// count -- and everything linked behind it. Nothing cut off is handed out again: it is counted +// as used from now on, which also keeps the page from being given back to the arena while such +// a block is linked into it (for a double free this simply restores the count; anything else +// is leaked). `listed` is the number of blocks the page's free lists hold after the cut, if the +// caller knows it (the sweep does: it has just collected the other lists into this one), and +// then `used` becomes exact; otherwise (`MI_LISTED_UNKNOWN`) `used` goes up by one for the +// block holding the bad link, and what hung behind it stays uncounted, which only makes the +// page look emptier than it is. The error is reported before the cut, so an error handler that +// aborts sees the list as it was: `bad` is the link or block that failed the check, `holder` +// the block whose link it was (NULL for the head of the list). +#define MI_LISTED_UNKNOWN SIZE_MAX + +static void mi_page_free_list_cut(mi_page_t* page, mi_block_t** list, mi_block_t* last, size_t listed, + const char* what, const char* problem, const mi_block_t* bad, const mi_block_t* holder) { + if (holder == NULL) { + _mi_error_message(EFAULT, "corrupted %s list in page %p (block size %zu): %s %p\n", + what, page, page->block_size, problem, bad); + } + else { + _mi_error_message(EFAULT, "corrupted %s list in page %p (block size %zu): %s %p in block %p\n", + what, page, page->block_size, problem, bad, holder); + } + if (last == NULL) { *list = NULL; } + else { mi_block_set_next(page, last, NULL); } + if (listed == MI_LISTED_UNKNOWN) { + if (page->used < page->capacity) { page->used++; } + } + else { + const size_t off_list = listed + _mi_page_purged_count(page); // free, but on the list or discarded + page->used = (off_list < page->capacity ? (uint32_t)(page->capacity - off_list) : 0); + } +} + +// Walk the list rooted at `*list`, checking it. Returns its last block, or NULL when the list is +// (or, after a cut, has become) empty. +static mi_block_t* mi_page_free_list_checked_tail(mi_page_t* page, mi_block_t** list, const char* what) { + mi_block_t* block = *list; + if (block == NULL) return NULL; + size_t idx; + if mi_unlikely(!mi_page_block_index_of(page, block, &idx)) { + mi_page_free_list_cut(page, list, NULL, MI_LISTED_UNKNOWN, what, "invalid head", block, NULL); + return NULL; + } + const size_t max_count = mi_page_max_free_listed(page); + mi_block_t* prev = NULL; + size_t count = 1; + for (;;) { + if mi_unlikely(count > max_count) { + mi_page_free_list_cut(page, list, prev, MI_LISTED_UNKNOWN, what, "more blocks than the page has free, at block", block, NULL); + return prev; + } + mi_block_t* const next = mi_block_next(page, block); + if (next == NULL) return block; + if mi_unlikely(!mi_page_block_index_of(page, next, &idx)) { + mi_page_free_list_cut(page, list, prev, MI_LISTED_UNKNOWN, what, "invalid link", next, block); + return prev; + } + prev = block; + block = next; + count++; + } +} + + /* ----------------------------------------------------------- Page collect the `local_free` and `thread_free` lists ----------------------------------------------------------- */ @@ -265,7 +366,17 @@ static void mi_page_thread_collect_to_local(mi_page_t* page, mi_block_t* head) size_t count = 1; mi_block_t* last = head; mi_block_t* next; + size_t idx; + if mi_unlikely(!mi_page_block_index_of(page, head, &idx)) { + _mi_error_message(EFAULT, "corrupted thread-free list in page %p (block size %zu): invalid head %p\n", page, page->block_size, head); + return; // the thread-free items cannot be freed + } while ((next = mi_block_next(page, last)) != NULL && count <= max_count) { + // a link that is not a block of this page is followed by no one (see "Free list corruption" above) + if mi_unlikely(!mi_page_block_index_of(page, next, &idx)) { + _mi_error_message(EFAULT, "corrupted thread-free list in page %p (block size %zu): invalid link %p in block %p\n", page, page->block_size, next, last); + return; // the thread-free items cannot be freed + } count++; last = next; } @@ -686,8 +797,10 @@ void _mi_page_unpurge_unformed_upto(mi_page_t* page, uintptr_t end) { // Walk the free list of a page and discard every OS page in it that holds no live block. -// Returns false if any discard failed: those blocks went straight back on the free list and the -// page must be swept again, so the caller must not record it as swept. +// Returns false if any discard failed, or if the free list turned out to be corrupted: in both +// cases the page must be swept again, so the caller must not record it as swept. (A failed +// discard puts its blocks straight back on the free list; a corrupted list is cut at the +// corruption, see "Free list corruption" above, and is walked afresh by the next sweep.) static bool mi_page_purge_holes_walk(mi_page_t* page, mi_tld_t* tld) { if (page->free == NULL) return true; // nothing to take off the free list @@ -697,14 +810,29 @@ static bool mi_page_purge_holes_walk(mi_page_t* page, mi_tld_t* tld) { if (nbits > MI_PAGE_PURGE_BITS) return true; bool complete = true; - // 1. count, per OS page, the blocks on the free list that overlap it + // 1. count, per OS page, the blocks on the free list that overlap it. The list is checked on + // the way (see "Free list corruption"), including that no block is on it twice: a block + // listed twice (a double free) would be counted twice, and an OS page holding it and one + // live block would then look entirely free and be discarded from under the live block. + // (`listed` also bounds the walk: a list of distinct blocks of this page ends within + // `capacity` links.) After a cut the counts are not usable; the page is consistent again + // and the next sweep walks it afresh. uint16_t nfree[MI_PAGE_PURGE_BITS]; _mi_memzero(nfree, nbits * sizeof(uint16_t)); + uint64_t listed[MI_PAGE_MAX_CAPACITY / 64]; // by block index: seen on the list already? + _mi_memzero(listed, _mi_divide_up(page->capacity, 64) * sizeof(uint64_t)); size_t nvisited = 0; - for (mi_block_t* b = page->free; b != NULL; b = mi_block_next(page, b)) { - const size_t idx = mi_page_block_index(page, b); + size_t idx; + if mi_unlikely(!mi_page_block_index_of(page, page->free, &idx)) { + mi_page_free_list_cut(page, &page->free, NULL, 0, "free", "invalid head", page->free, NULL); + return false; + } + mi_block_t* prev = NULL; + mi_block_t* b = page->free; + for (;;) { mi_assert_internal(idx < page->capacity); mi_assert_internal(!mi_page_block_index_is_purged(page, idx)); // it is on the free list, so not purged + listed[idx / 64] |= ((uint64_t)1 << (idx % 64)); nvisited++; size_t kfirst, klast; mi_page_block_os_pages(page, idx, &kfirst, &klast); @@ -712,6 +840,22 @@ static bool mi_page_purge_holes_walk(mi_page_t* page, mi_tld_t* tld) { mi_assert_internal(nfree[k] < UINT16_MAX); nfree[k]++; } + mi_block_t* const next = mi_block_next(page, b); + if (next == NULL) break; + if mi_unlikely(!mi_page_block_index_of(page, next, &idx)) { + // `b` holds the bad link, so it goes as well: `prev` becomes the last block, `nvisited - 1` remain + mi_page_free_list_cut(page, &page->free, prev, nvisited - 1, "free", "invalid link", next, b); + tld->holes_sweep_visited += nvisited; + return false; + } + if mi_unlikely((listed[idx / 64] & ((uint64_t)1 << (idx % 64))) != 0) { + // `next` is on the list for the second time: its first occurrence stays, the list ends at `b` + mi_page_free_list_cut(page, &page->free, b, nvisited, "free", "block listed twice:", next, b); + tld->holes_sweep_visited += nvisited; + return false; + } + prev = b; + b = next; } tld->holes_sweep_visited += nvisited; // folded into the process-wide counter at the end of the pass @@ -738,10 +882,10 @@ static bool mi_page_purge_holes_walk(mi_page_t* page, mi_tld_t* tld) { // 3. rebuild the free list without the blocks that are about to lose memory. This must // happen *before* the discard: it walks `next` pointers that live in the very memory - // we are about to discard. + // we are about to discard. (The list was checked in step 1 and nothing has touched it since.) mi_block_t* keep = NULL; size_t ndropped = 0; - mi_block_t* b = page->free; + b = page->free; while (b != NULL) { mi_block_t* const next = mi_block_next(page, b); if (mi_page_block_overlaps(page, mi_page_block_index(page, b), todo)) { @@ -893,8 +1037,6 @@ void _mi_page_unpurge_all(mi_page_t* page) { one OS page, so nothing is double counted even for a block straddling a boundary. ----------------------------------------------------------- */ -#define MI_HOLES_MAX_CAP (1 << 16) // `page->capacity` is a uint16_t - static void mi_holes_mark_free_list(const mi_page_t* page, mi_block_t* b, uint64_t* set) { const size_t cap = page->capacity; for (size_t n = 0; b != NULL && n <= cap; n++) { // `n` bounds a corrupt or cyclic list @@ -953,7 +1095,7 @@ void _mi_page_holes_report_page(const mi_page_t* page, mi_holes_report_t* rep) { if (page == NULL || rep == NULL) return; const size_t bs = page->block_size; const size_t cap = page->capacity; - if (bs == 0 || cap > MI_HOLES_MAX_CAP) return; + if (bs == 0 || cap > MI_PAGE_MAX_CAPACITY) return; mi_holes_bin_t* const r = &rep->bin[_mi_bin(bs)]; r->pages++; if (bs > r->block_size) { r->block_size = bs; } @@ -963,7 +1105,7 @@ void _mi_page_holes_report_page(const mi_page_t* page, mi_holes_report_t* rep) { rep->unformed_discarded_bytes += _mi_page_unformed_purged_bytes(page); if (cap == 0) return; - uint64_t freelisted[MI_HOLES_MAX_CAP / 64]; + uint64_t freelisted[MI_PAGE_MAX_CAPACITY / 64]; const size_t nwords = _mi_divide_up(cap, 64); _mi_memzero(freelisted, nwords * sizeof(uint64_t)); mi_holes_mark_free_list(page, page->free, freelisted); @@ -1203,16 +1345,16 @@ static void mi_page_free_collect_ex(mi_page_t* page, bool force, bool allow_unpu page->free_is_zero = false; } else if (force) { - // append -- only on shutdown (force) as this is a linear operation - mi_block_t* tail = page->local_free; - mi_block_t* next; - while ((next = mi_block_next(page, tail)) != NULL) { - tail = next; + // append -- only on shutdown and in the idle sweep (force) as this is a linear operation; + // a corrupted `local_free` is cut at the corruption (see "Free list corruption") and what + // is left of it is appended + mi_block_t* const tail = mi_page_free_list_checked_tail(page, &page->local_free, "local free"); + if (tail != NULL) { + mi_block_set_next(page, tail, page->free); + page->free = page->local_free; + page->free_is_zero = false; } - mi_block_set_next(page, tail, page->free); - page->free = page->local_free; page->local_free = NULL; - page->free_is_zero = false; } } diff --git a/src/scavenger.c b/src/scavenger.c index 93c71b3bf..78a4d3321 100644 --- a/src/scavenger.c +++ b/src/scavenger.c @@ -322,8 +322,21 @@ void _mi_scavenger_start(void) { // unblocked will have process-directed signals dispatched to it and silently // discarded, starving signalfd/kqueue consumers. sigfillset on glibc/musl // already excludes the libc-internal realtime signals used for setxid/cancel. + // + // Except the signals a fault on this thread itself raises: a blocked SIGSEGV/SIGBUS + // is not queued, the kernel resets it to its default action and kills the process on + // the spot, so the host's crash handler never runs and a corrupted free list that the + // sweep trips over (see `mi_page_purge_holes_walk`) ends the process without a report. + // These are thread-directed by nature, so leaving them unblocked starves no one. sigset_t all, old; sigfillset(&all); + sigdelset(&all, SIGSEGV); + sigdelset(&all, SIGBUS); + sigdelset(&all, SIGILL); + sigdelset(&all, SIGFPE); + sigdelset(&all, SIGTRAP); + sigdelset(&all, SIGABRT); + sigdelset(&all, SIGSYS); pthread_sigmask(SIG_SETMASK, &all, &old); if (pthread_create(&_mi_scavenger_thread, NULL, &mi_scavenger_thread_main, NULL) != 0) { mi_atomic_store_release(&_mi_scavenger_running, (uintptr_t)0); diff --git a/test/test-freelist-corruption.c b/test/test-freelist-corruption.c new file mode 100644 index 000000000..d8e847d4c --- /dev/null +++ b/test/test-freelist-corruption.c @@ -0,0 +1,388 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2025, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +// The walkers that read a whole free list (the idle sweep, the forced collect and the +// thread-free collect; see the "Free list corruption" section in `src/page.c`) must survive a +// corrupted list: report it once through the error handler, cut the list, leave the page's +// accounting consistent, and never hand out the block that held the bad link. +// +// Each test corrupts a list the way the crashes in the field look (oven-sh/bun BUN-40BH and +// its siblings): the first words of a block were overwritten after it was freed. The bad link +// points 8 bytes into the block itself, so that a build that encodes its free lists (whose +// `mi_block_next` already rejects a link into another page) and a release build take the same +// path; the word at that address is 0xA0D, the value seen most often in the field. On a build +// without the checks these tests dereference 0xA0D and fault (and `free-twice` loops forever). + +#include +#include +#include +#include +#include + +#include "mimalloc.h" +#include "mimalloc/internal.h" // _mi_ptr_page, mi_block_next, mi_block_set_next, mi_page_thread_free, _mi_page_purged_count + +#include "testhelper.h" + +// --------------------------------------------------------------------------- +// The error handler and the output hook. A corruption must be reported as an +// EFAULT, and the walker under test must report it exactly once: that is +// counted on the message text, because a debug build also walks the lists in +// its assertions (`_mi_page_is_valid`) and reports a bad link from there first. +// --------------------------------------------------------------------------- + +static int error_count = 0; +static int error_last = 0; +static const char* report_needle = NULL; // the message the walker under test is expected to print ... +static int report_count = 0; // ... and how often it was printed + +static void on_error(int err, void* arg) { + MI_UNUSED(arg); + error_count++; + error_last = err; +} + +static void on_output(const char* msg, void* arg) { + MI_UNUSED(arg); + fputs(msg, stderr); + if (report_needle != NULL && strstr(msg, report_needle) != NULL) { report_count++; } +} + +static void expect_report(const char* needle) { + report_needle = needle; + report_count = 0; + error_count = 0; + error_last = 0; +} + +static bool reported_once(const char* what) { + if (report_count != 1 || error_count < 1 || error_last != EFAULT) { + fprintf(stderr, "\n %s: expected one \"%s\" report and an EFAULT, got %d report(s) and %d error(s), the last one %d\n", + what, report_needle, report_count, error_count, error_last); + return false; + } + return true; +} + +#define REPORT_FREE "corrupted free list in page" +#define REPORT_LOCAL_FREE "corrupted local free list in page" +#define REPORT_THREAD_FREE "corrupted thread-free list in page" + +// --------------------------------------------------------------------------- +// a page we control: NBLOCKS blocks in one page of a fresh heap; the odd ones +// are freed and collected onto `page->free`, the even ones stay live and hold +// a pattern. With every other block live no OS page is ever discardable, so +// the freed blocks stay on the list. +// --------------------------------------------------------------------------- + +#define NBLOCKS (24) +#define BSIZE (1000) + +typedef struct victim_s { + mi_heap_t* heap; + mi_page_t* page; + void* block[NBLOCKS]; + bool live[NBLOCKS]; +} victim_t; + +static uint8_t pattern_byte(size_t id, size_t off) { + return (uint8_t)((id * 131u) ^ (off * 7u) ^ (off >> 8)); +} + +static bool pattern_intact(const victim_t* v) { + for (size_t i = 0; i < NBLOCKS; i++) { + if (!v->live[i]) continue; + const uint8_t* const b = (const uint8_t*)v->block[i]; + for (size_t off = 0; off < BSIZE; off++) { + if (b[off] != pattern_byte(i, off)) { + fprintf(stderr, "\n live block %zu was damaged at offset %zu\n", i, off); + return false; + } + } + } + return true; +} + +static bool victim_setup(victim_t* v) { + memset(v, 0, sizeof(*v)); + v->heap = mi_heap_new(); + if (v->heap == NULL) { fprintf(stderr, "\n mi_heap_new failed\n"); return false; } + for (size_t i = 0; i < NBLOCKS; i++) { + uint8_t* const b = (uint8_t*)mi_heap_malloc(v->heap, BSIZE); + if (b == NULL) { fprintf(stderr, "\n mi_heap_malloc failed\n"); return false; } + for (size_t off = 0; off < BSIZE; off++) { b[off] = pattern_byte(i, off); } + v->block[i] = b; + v->live[i] = true; + } + v->page = _mi_ptr_page(v->block[0]); + for (size_t i = 1; i < NBLOCKS; i++) { + if (_mi_ptr_page(v->block[i]) != v->page) { fprintf(stderr, "\n the blocks do not share one page\n"); return false; } + } + for (size_t i = 1; i < NBLOCKS; i += 2) { + mi_free(v->block[i]); + v->live[i] = false; + } + error_count = 0; + mi_on_thread_idle(); // collects the frees onto `page->free` + if (error_count != 0) { fprintf(stderr, "\n the sweep of an intact page reported an error\n"); return false; } + if (v->page->free == NULL || mi_block_next(v->page, v->page->free) == NULL) { + fprintf(stderr, "\n expected at least two blocks on page->free\n"); + return false; + } + return true; +} + +static void victim_done(victim_t* v) { + if (v->heap != NULL) { mi_heap_destroy(v->heap); } + v->heap = NULL; +} + +// Overwrite the first words of a freed block the way the field crashes look: its link points +// 8 bytes into the block itself, and the word there is 0xA0D. Returns the bad link. +static mi_block_t* scribble(const mi_page_t* page, void* block) { + uintptr_t* const inside = (uintptr_t*)((uint8_t*)block + sizeof(mi_block_t)); + *inside = (uintptr_t)0xA0D; + mi_block_set_next(page, (mi_block_t*)block, (mi_block_t*)inside); + return (mi_block_t*)inside; +} + +// Walks are bounded so that a failing test cannot hang on a list that is still broken. +static size_t list_count(const mi_page_t* page, mi_block_t* head) { + size_t n = 0; + for (mi_block_t* b = head; b != NULL && n <= (size_t)page->capacity + 1; b = mi_block_next(page, b)) { n++; } + return n; +} + +static size_t list_occurrences(const mi_page_t* page, mi_block_t* head, const void* block) { + size_t n = 0, found = 0; + for (mi_block_t* b = head; b != NULL && n <= (size_t)page->capacity + 1; b = mi_block_next(page, b)) { + n++; + if ((const void*)b == block) { found++; } + } + return found; +} + +static bool on_no_list(const victim_t* v, const void* block) { + if (list_occurrences(v->page, v->page->free, block) != 0 || + list_occurrences(v->page, v->page->local_free, block) != 0 || + list_occurrences(v->page, mi_page_thread_free(v->page), block) != 0) { + fprintf(stderr, "\n the block that held the bad link is still on a list\n"); + return false; + } + return true; +} + +// The conservation of blocks that a cut must leave intact: `used` (which includes the blocks +// on the thread-free list) + the blocks on `free` and `local_free` + the discarded blocks. +static bool page_accounts(const victim_t* v, const char* when) { + const mi_page_t* const page = v->page; + const size_t listed = list_count(page, page->free) + list_count(page, page->local_free); + const size_t discarded = _mi_page_purged_count(page); + if ((size_t)page->used + listed + discarded != (size_t)page->capacity) { + fprintf(stderr, "\n %s: used %u + listed %zu + discarded %zu != capacity %u\n", + when, (unsigned)page->used, listed, discarded, (unsigned)page->capacity); + return false; + } + return true; +} + +typedef void (pass_fun_t)(victim_t* v); +static void pass_sweep(victim_t* v) { MI_UNUSED(v); mi_on_thread_idle(); } +static void pass_collect(victim_t* v) { mi_heap_collect(v->heap, true); } + +// After a cut the page must be an ordinary page again: another pass reports nothing, the +// accounting holds, the live blocks are intact, and `culprit` (the block that held the bad +// link; NULL when the head itself was bad) is never handed out again. +static bool page_recovered(victim_t* v, const void* culprit, pass_fun_t* pass) { + error_count = 0; + pass(v); + if (error_count != 0) { fprintf(stderr, "\n the second pass reported %d error(s)\n", error_count); return false; } + if (!page_accounts(v, "after the second pass")) return false; + void* got[4 * NBLOCKS]; + bool handed_out = false; + for (size_t i = 0; i < 4 * NBLOCKS; i++) { + got[i] = mi_heap_malloc(v->heap, BSIZE); + if (culprit != NULL && got[i] == culprit) { handed_out = true; } + } + for (size_t i = 0; i < 4 * NBLOCKS; i++) { mi_free(got[i]); } + if (handed_out) { fprintf(stderr, "\n the block that held the bad link was handed out again\n"); return false; } + return pattern_intact(v); +} + +// --------------------------------------------------------------------------- +// 1. a bad link in the middle of `page->free` (the idle sweep's walk): the +// list is cut in front of the block holding it +// --------------------------------------------------------------------------- + +static bool test_free_link(void) { + victim_t v; + bool ok_ = victim_setup(&v); + if (ok_) { + mi_block_t* const first = v.page->free; + mi_block_t* const culprit = mi_block_next(v.page, first); // the second block: something stays in front of the cut + scribble(v.page, culprit); + expect_report(REPORT_FREE); + mi_on_thread_idle(); + ok_ = reported_once("free-link") + && (v.page->free == first && mi_block_next(v.page, first) == NULL) + && on_no_list(&v, culprit) + && page_accounts(&v, "after the cut") + && page_recovered(&v, culprit, &pass_sweep); + } + victim_done(&v); + return ok_; +} + +// --------------------------------------------------------------------------- +// 2. the head of `page->free` itself is bad: the whole list is dropped +// --------------------------------------------------------------------------- + +static bool test_free_head(void) { + victim_t v; + bool ok_ = victim_setup(&v); + if (ok_) { + v.page->free = scribble(v.page, v.page->free); + expect_report(REPORT_FREE); + mi_on_thread_idle(); + ok_ = reported_once("free-head") + && (v.page->free == NULL) + && page_accounts(&v, "after the cut") + && page_recovered(&v, NULL, &pass_sweep); + } + victim_done(&v); + return ok_; +} + +// --------------------------------------------------------------------------- +// 3. a block that links to itself, as after a double free: the sweep must +// neither loop nor count the block twice (an OS page is discarded when every +// block in it is free, and a block counted twice can make it look that way). +// The block is an ordinary free block afterwards, so it may be handed out. +// --------------------------------------------------------------------------- + +static bool test_free_twice(void) { + victim_t v; + bool ok_ = victim_setup(&v); + if (ok_) { + mi_block_t* const twice = mi_block_next(v.page, v.page->free); + mi_block_set_next(v.page, twice, twice); + expect_report(REPORT_FREE); + mi_on_thread_idle(); + ok_ = reported_once("free-twice") + && (list_occurrences(v.page, v.page->free, twice) == 1) // its first occurrence stays ... + && (mi_block_next(v.page, twice) == NULL) // ... as the end of the list + && page_accounts(&v, "after the cut"); + if (ok_) { + error_count = 0; + mi_on_thread_idle(); + ok_ = (error_count == 0) && page_accounts(&v, "after the second sweep") && pattern_intact(&v); + if (error_count != 0) { fprintf(stderr, "\n the second sweep reported %d error(s)\n", error_count); } + } + } + victim_done(&v); + return ok_; +} + +// --------------------------------------------------------------------------- +// 4. a bad link on `page->local_free`, which the sweep's forced collect +// appends to `free`: the part in front of the cut is still appended +// --------------------------------------------------------------------------- + +static bool test_local_free_link(void) { + victim_t v; + bool ok_ = victim_setup(&v); + if (ok_) { + // frees by the owning thread go onto `local_free`, most recent first: block 2 -> block 0 + mi_free(v.block[0]); v.live[0] = false; + mi_free(v.block[2]); v.live[2] = false; + ok_ = (v.page->local_free == (mi_block_t*)v.block[2] && mi_block_next(v.page, v.page->local_free) == (mi_block_t*)v.block[0]); + if (!ok_) { fprintf(stderr, "\n local_free is not [block 2, block 0]\n"); } + } + if (ok_) { + scribble(v.page, v.block[0]); // the last block of the list: the cut drops exactly this one + expect_report(REPORT_LOCAL_FREE); + mi_on_thread_idle(); + ok_ = reported_once("local-free-link") + && on_no_list(&v, v.block[0]) + && (list_occurrences(v.page, v.page->free, v.block[2]) == 1) + && (v.page->local_free == NULL) + && page_accounts(&v, "after the cut") + && page_recovered(&v, v.block[0], &pass_sweep); + } + victim_done(&v); + return ok_; +} + +// --------------------------------------------------------------------------- +// 5. a bad link on the thread-free list (a block freed by another thread): +// the collect drops the list it took off the page, and the blocks on it +// stay accounted as used +// --------------------------------------------------------------------------- + +static void* freed_by_other_thread = NULL; + +static bool free_on_other_thread(void) { + mi_free(freed_by_other_thread); + return true; +} + +static bool test_thread_free_link(void) { + victim_t v; + bool ok_ = victim_setup(&v); + if (ok_) { + freed_by_other_thread = v.block[0]; + v.live[0] = false; + ok_ = mi_run_on_thread(&free_on_other_thread) && (mi_page_thread_free(v.page) == (mi_block_t*)v.block[0]); + if (!ok_) { fprintf(stderr, "\n the block freed on the other thread is not the head of the thread-free list\n"); } + } + if (ok_) { + const uint32_t used_before = v.page->used; // a block on the thread-free list still counts as used + scribble(v.page, v.block[0]); + expect_report(REPORT_THREAD_FREE); + mi_heap_collect(v.heap, true); + ok_ = reported_once("thread-free-link") + && (mi_page_thread_free(v.page) == NULL) + && on_no_list(&v, v.block[0]) + && (v.page->used == used_before) + && page_accounts(&v, "after the drop") + && page_recovered(&v, v.block[0], &pass_collect); + if (ok_ == false && v.page->used != used_before) { fprintf(stderr, "\n used changed from %u to %u\n", (unsigned)used_before, (unsigned)v.page->used); } + } + victim_done(&v); + return ok_; +} + +// --------------------------------------------------------------------------- +// main +// --------------------------------------------------------------------------- + +// An optional argument selects a single test, which is how each one is checked to fault (or +// loop) on a build without the checks: the first one to run would otherwise take the rest with it. +static bool selected(int argc, char** argv, const char* name) { + return (argc < 2 || strcmp(argv[1], name) == 0); +} + +int main(int argc, char** argv) { + mi_version(); + mi_register_error(&on_error, NULL); + mi_register_output(&on_output, NULL); + mi_option_set(mi_option_show_errors, 1); // the messages are part of what is tested: they have to format without faulting + mi_option_set(mi_option_purge_holes_full_every, 1); // walk every page on every sweep: a scribbled link changes nothing the skip check looks at + + if (mi_option_is_enabled(mi_option_purge_holes)) { + if (selected(argc, argv, "free-link")) { CHECK("free-link", test_free_link()); } + if (selected(argc, argv, "free-head")) { CHECK("free-head", test_free_head()); } + if (selected(argc, argv, "free-twice")) { CHECK("free-twice", test_free_twice()); } + if (selected(argc, argv, "local-free-link")) { CHECK("local-free-link", test_local_free_link()); } + } + else { + fprintf(stderr, "purge_holes is off: the sweep tests are skipped\n"); + } + if (selected(argc, argv, "thread-free-link")) { CHECK("thread-free-link", test_thread_free_link()); } + + return print_test_summary(); +}