From 8e59fb912e17db427d71df360dee078c798194b9 Mon Sep 17 00:00:00 2001 From: "Nelson, Erik" Date: Wed, 30 Sep 2026 00:34:02 +0000 Subject: [PATCH 1/5] Keep the bounds test's unchecked reads inside an array it owns; past a one-byte array they hit an unmapped page on Windows --- test/test_cache.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/test_cache.py b/test/test_cache.py index d6f22b8..1ba38a6 100644 --- a/test/test_cache.py +++ b/test/test_cache.py @@ -288,10 +288,15 @@ def test_a_read_only_install_warns_naming_numba_cache_dir_and_setting_it_caches( path.chmod(0o755) +# The bitmap is the first byte of an array that holds every byte the calls +# reach, so without bounds checking they read past the bitmap and stay inside +# memory the array owns. Behind a one-byte array of its own they read whatever +# followed it on the heap, and where that was an unmapped page the child died +# of an access violation, as it did once on Windows. CHECK_BOUNDS = ( "import numpy as np\n" "from numbarrow.core.is_null import is_null, unpack_booleans\n" - "bitmap = np.zeros(1, dtype=np.uint8)\n" + "bitmap = np.zeros(100000 // 8 + 1, dtype=np.uint8)[:1]\n" "outcomes = []\n" "for call in (lambda: is_null(100000, bitmap), lambda: unpack_booleans(0, 100000, bitmap)):\n" " try:\n" From e82646c42e374028c70556ba5934c54f2f529910 Mon Sep 17 00:00:00 2001 From: "Nelson, Erik" Date: Wed, 30 Sep 2026 02:44:03 +0000 Subject: [PATCH 2/5] Check the cache option on each of is_null.py's three decorators in the options test; it named three and exercised two --- .github/scripts/mutation_guard_check.py | 8 ++++++++ test/test_cache.py | 13 ++++++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/.github/scripts/mutation_guard_check.py b/.github/scripts/mutation_guard_check.py index 0380e81..87514c2 100755 --- a/.github/scripts/mutation_guard_check.py +++ b/.github/scripts/mutation_guard_check.py @@ -217,6 +217,14 @@ ' Optional(Array(uint8, 1, "C", readonly=True))))', "@jit_with_options(None)", ), + ( + "is_null_struct's decorator stops reading the options", + "numbarrow/core/is_null.py", + '@jit_with_options(boolean(int64, Optional(Array(uint8, 1, "C", readonly=True)),\n' + ' Optional(Array(uint8, 1, "C", readonly=True))))', + '@__import__("numba").njit(boolean(int64, Optional(Array(uint8, 1, "C", readonly=True)),\n' + ' Optional(Array(uint8, 1, "C", readonly=True))), cache=True)', + ), ( "viewers stop getting a cache name of their own", "numbarrow/utils/utils.py", diff --git a/test/test_cache.py b/test/test_cache.py index 1ba38a6..6cf99e1 100644 --- a/test/test_cache.py +++ b/test/test_cache.py @@ -311,8 +311,19 @@ def test_a_read_only_install_warns_naming_numba_cache_dir_and_setting_it_caches( def test_jit_options_reach_the_is_null_decorators(tmp_path): # The options test imported only the viewers, so hard-coding the options # on is_null.py's three decorators kept the suite green, and the documented - # boundscheck contract had no test at all. + # boundscheck contract had no test at all. Bounds checking shows on the two + # functions that index a bitmap; is_null_struct indexes nothing and calls + # is_null, whose own flags decide, so on its decorator the option that + # shows is the cache: off, nothing reaches the cache directory from any of + # the three, and on, each of the three leaves an index there. checked = _run(CHECK_BOUNDS, _env(tmp_path / "checked", {"cache": False, "boundscheck": True}), tmp_path) assert checked.returncode == 0 and checked.stdout.split() == ["IndexError", "IndexError"], checked.stderr unchecked = _run(CHECK_BOUNDS, _env(tmp_path / "unchecked", {"cache": False}), tmp_path) assert unchecked.returncode == 0 and unchecked.stdout.split() == ["returned", "returned"], unchecked.stderr + assert _index_files(tmp_path / "checked") == [] and _index_files(tmp_path / "unchecked") == [] + cached = _run(CHECK_BOUNDS, _env(tmp_path / "cached", {"cache": True}), tmp_path) + assert cached.returncode == 0 and cached.stdout.split() == ["returned", "returned"], cached.stderr + indexes = _index_files(tmp_path / "cached") + assert len(indexes) == 3, indexes + for function in ("is_null-", "unpack_booleans", "is_null_struct"): + assert any(function in name for name in indexes), (function, indexes) From 5055a56284d57866b4089dd4edb4ac27ce4a2c19 Mon Sep 17 00:00:00 2001 From: "Nelson, Erik" Date: Wed, 30 Sep 2026 12:42:52 +0000 Subject: [PATCH 3/5] Give the catalogue one entry per decorator of is_null.py, each hard-coded uncached; the one entry it had was killed by four older tests without the new legs --- .github/scripts/mutation_guard_check.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/scripts/mutation_guard_check.py b/.github/scripts/mutation_guard_check.py index 87514c2..4a07bd6 100755 --- a/.github/scripts/mutation_guard_check.py +++ b/.github/scripts/mutation_guard_check.py @@ -217,13 +217,26 @@ ' Optional(Array(uint8, 1, "C", readonly=True))))', "@jit_with_options(None)", ), + ( + "is_null's decorator stops reading the options", + "numbarrow/core/is_null.py", + '@jit_with_options(boolean(int64, Array(uint8, 1, "C", readonly=True)))', + '@__import__("numba").njit(boolean(int64, Array(uint8, 1, "C", readonly=True)), cache=False)', + ), + ( + "unpack_booleans' decorator stops reading the options", + "numbarrow/core/is_null.py", + '@jit_with_options(Array(bool_, 1, "C")(int64, int64, Array(uint8, 1, "C", readonly=True)))', + '@__import__("numba").njit(Array(bool_, 1, "C")(int64, int64, Array(uint8, 1, "C", readonly=True)), ' + 'cache=False)', + ), ( "is_null_struct's decorator stops reading the options", "numbarrow/core/is_null.py", '@jit_with_options(boolean(int64, Optional(Array(uint8, 1, "C", readonly=True)),\n' ' Optional(Array(uint8, 1, "C", readonly=True))))', '@__import__("numba").njit(boolean(int64, Optional(Array(uint8, 1, "C", readonly=True)),\n' - ' Optional(Array(uint8, 1, "C", readonly=True))), cache=True)', + ' Optional(Array(uint8, 1, "C", readonly=True))), cache=False)', ), ( "viewers stop getting a cache name of their own", From f8ce213cb53105fba9d7ff031097e7f7cbe0af0f Mon Sep 17 00:00:00 2001 From: "Nelson, Erik" Date: Wed, 30 Sep 2026 13:43:59 +0000 Subject: [PATCH 4/5] Add a catalogue entry that only the options test's cache legs catch; the three decorator entries were killed by older tests as well --- .github/scripts/mutation_guard_check.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/scripts/mutation_guard_check.py b/.github/scripts/mutation_guard_check.py index 4a07bd6..8790052 100755 --- a/.github/scripts/mutation_guard_check.py +++ b/.github/scripts/mutation_guard_check.py @@ -721,6 +721,13 @@ ' cache_failed = isinstance(error, OSError) or "no locator available" in str(error)', ' cache_failed = True', ), + ( + 'the cache option stops reaching unpack_booleans while every other option does', + 'numbarrow/core/configurations.py', + ' return njit(signature, **jit_options)(func)', + ' return njit(signature, **{**jit_options, "cache": jit_options.get("cache")' + ' and func.__name__ != "unpack_booleans"})(func)', + ), ( 'the repeated-name check stops looking inside a map', 'numbarrow/core/mapinarrow_factory.py', From 52b9fa822aca46897fc9aa716cfabd4502748d1a Mon Sep 17 00:00:00 2001 From: "Nelson, Erik" Date: Wed, 30 Sep 2026 14:19:42 +0000 Subject: [PATCH 5/5] Read an option that shows nowhere back from the three dispatchers in the options test; forwarding only cache and boundscheck kept the suite green --- .github/scripts/mutation_guard_check.py | 7 +++++++ test/test_cache.py | 12 +++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/.github/scripts/mutation_guard_check.py b/.github/scripts/mutation_guard_check.py index 8790052..f8daaa9 100755 --- a/.github/scripts/mutation_guard_check.py +++ b/.github/scripts/mutation_guard_check.py @@ -728,6 +728,13 @@ ' return njit(signature, **{**jit_options, "cache": jit_options.get("cache")' ' and func.__name__ != "unpack_booleans"})(func)', ), + ( + 'the decorators stop forwarding any option but cache and boundscheck', + 'numbarrow/core/configurations.py', + ' return njit(signature, **jit_options)(func)', + ' return njit(signature, **{key: value for key, value in jit_options.items()' + ' if key in ("cache", "boundscheck")})(func)', + ), ( 'the repeated-name check stops looking inside a map', 'numbarrow/core/mapinarrow_factory.py', diff --git a/test/test_cache.py b/test/test_cache.py index 6cf99e1..d90bea8 100644 --- a/test/test_cache.py +++ b/test/test_cache.py @@ -307,6 +307,11 @@ def test_a_read_only_install_warns_naming_numba_cache_dir_and_setting_it_caches( "print(' '.join(outcomes))\n" ) +SHOW_NOGIL = ( + "from numbarrow.core.is_null import is_null, unpack_booleans, is_null_struct\n" + "print(' '.join(str(f.targetoptions.get('nogil')) for f in (is_null, unpack_booleans, is_null_struct)))\n" +) + def test_jit_options_reach_the_is_null_decorators(tmp_path): # The options test imported only the viewers, so hard-coding the options @@ -315,7 +320,10 @@ def test_jit_options_reach_the_is_null_decorators(tmp_path): # functions that index a bitmap; is_null_struct indexes nothing and calls # is_null, whose own flags decide, so on its decorator the option that # shows is the cache: off, nothing reaches the cache directory from any of - # the three, and on, each of the three leaves an index there. + # the three, and on, each of the three leaves an index there. An option + # that shows on none of them, nogil, is read back from the three + # dispatchers, so a decorator forwarding only the two that show is caught + # as well. checked = _run(CHECK_BOUNDS, _env(tmp_path / "checked", {"cache": False, "boundscheck": True}), tmp_path) assert checked.returncode == 0 and checked.stdout.split() == ["IndexError", "IndexError"], checked.stderr unchecked = _run(CHECK_BOUNDS, _env(tmp_path / "unchecked", {"cache": False}), tmp_path) @@ -327,3 +335,5 @@ def test_jit_options_reach_the_is_null_decorators(tmp_path): assert len(indexes) == 3, indexes for function in ("is_null-", "unpack_booleans", "is_null_struct"): assert any(function in name for name in indexes), (function, indexes) + forwarded = _run(SHOW_NOGIL, _env(tmp_path / "nogil", {"cache": False, "nogil": True}), tmp_path) + assert forwarded.returncode == 0 and forwarded.stdout.split() == ["True", "True", "True"], forwarded.stderr