diff --git a/.github/scripts/mutation_guard_check.py b/.github/scripts/mutation_guard_check.py index 0380e81..f8daaa9 100755 --- a/.github/scripts/mutation_guard_check.py +++ b/.github/scripts/mutation_guard_check.py @@ -217,6 +217,27 @@ ' Optional(Array(uint8, 1, "C", readonly=True))))', "@jit_with_options(None)", ), + ( + "is_null's decorator stops reading the options", + "numbarrow/core/is_null.py", + '@jit_with_options(boolean(int64, Array(uint8, 1, "C", readonly=True)))', + '@__import__("numba").njit(boolean(int64, Array(uint8, 1, "C", readonly=True)), cache=False)', + ), + ( + "unpack_booleans' decorator stops reading the options", + "numbarrow/core/is_null.py", + '@jit_with_options(Array(bool_, 1, "C")(int64, int64, Array(uint8, 1, "C", readonly=True)))', + '@__import__("numba").njit(Array(bool_, 1, "C")(int64, int64, Array(uint8, 1, "C", readonly=True)), ' + 'cache=False)', + ), + ( + "is_null_struct's decorator stops reading the options", + "numbarrow/core/is_null.py", + '@jit_with_options(boolean(int64, Optional(Array(uint8, 1, "C", readonly=True)),\n' + ' Optional(Array(uint8, 1, "C", readonly=True))))', + '@__import__("numba").njit(boolean(int64, Optional(Array(uint8, 1, "C", readonly=True)),\n' + ' Optional(Array(uint8, 1, "C", readonly=True))), cache=False)', + ), ( "viewers stop getting a cache name of their own", "numbarrow/utils/utils.py", @@ -700,6 +721,20 @@ ' cache_failed = isinstance(error, OSError) or "no locator available" in str(error)', ' cache_failed = True', ), + ( + 'the cache option stops reaching unpack_booleans while every other option does', + 'numbarrow/core/configurations.py', + ' return njit(signature, **jit_options)(func)', + ' return njit(signature, **{**jit_options, "cache": jit_options.get("cache")' + ' and func.__name__ != "unpack_booleans"})(func)', + ), + ( + 'the decorators stop forwarding any option but cache and boundscheck', + 'numbarrow/core/configurations.py', + ' return njit(signature, **jit_options)(func)', + ' return njit(signature, **{key: value for key, value in jit_options.items()' + ' if key in ("cache", "boundscheck")})(func)', + ), ( 'the repeated-name check stops looking inside a map', 'numbarrow/core/mapinarrow_factory.py', diff --git a/test/test_cache.py b/test/test_cache.py index d6f22b8..d90bea8 100644 --- a/test/test_cache.py +++ b/test/test_cache.py @@ -288,10 +288,15 @@ def test_a_read_only_install_warns_naming_numba_cache_dir_and_setting_it_caches( path.chmod(0o755) +# The bitmap is the first byte of an array that holds every byte the calls +# reach, so without bounds checking they read past the bitmap and stay inside +# memory the array owns. Behind a one-byte array of its own they read whatever +# followed it on the heap, and where that was an unmapped page the child died +# of an access violation, as it did once on Windows. CHECK_BOUNDS = ( "import numpy as np\n" "from numbarrow.core.is_null import is_null, unpack_booleans\n" - "bitmap = np.zeros(1, dtype=np.uint8)\n" + "bitmap = np.zeros(100000 // 8 + 1, dtype=np.uint8)[:1]\n" "outcomes = []\n" "for call in (lambda: is_null(100000, bitmap), lambda: unpack_booleans(0, 100000, bitmap)):\n" " try:\n" @@ -302,12 +307,33 @@ def test_a_read_only_install_warns_naming_numba_cache_dir_and_setting_it_caches( "print(' '.join(outcomes))\n" ) +SHOW_NOGIL = ( + "from numbarrow.core.is_null import is_null, unpack_booleans, is_null_struct\n" + "print(' '.join(str(f.targetoptions.get('nogil')) for f in (is_null, unpack_booleans, is_null_struct)))\n" +) + def test_jit_options_reach_the_is_null_decorators(tmp_path): # The options test imported only the viewers, so hard-coding the options # on is_null.py's three decorators kept the suite green, and the documented - # boundscheck contract had no test at all. + # boundscheck contract had no test at all. Bounds checking shows on the two + # functions that index a bitmap; is_null_struct indexes nothing and calls + # is_null, whose own flags decide, so on its decorator the option that + # shows is the cache: off, nothing reaches the cache directory from any of + # the three, and on, each of the three leaves an index there. An option + # that shows on none of them, nogil, is read back from the three + # dispatchers, so a decorator forwarding only the two that show is caught + # as well. checked = _run(CHECK_BOUNDS, _env(tmp_path / "checked", {"cache": False, "boundscheck": True}), tmp_path) assert checked.returncode == 0 and checked.stdout.split() == ["IndexError", "IndexError"], checked.stderr unchecked = _run(CHECK_BOUNDS, _env(tmp_path / "unchecked", {"cache": False}), tmp_path) assert unchecked.returncode == 0 and unchecked.stdout.split() == ["returned", "returned"], unchecked.stderr + assert _index_files(tmp_path / "checked") == [] and _index_files(tmp_path / "unchecked") == [] + cached = _run(CHECK_BOUNDS, _env(tmp_path / "cached", {"cache": True}), tmp_path) + assert cached.returncode == 0 and cached.stdout.split() == ["returned", "returned"], cached.stderr + indexes = _index_files(tmp_path / "cached") + assert len(indexes) == 3, indexes + for function in ("is_null-", "unpack_booleans", "is_null_struct"): + assert any(function in name for name in indexes), (function, indexes) + forwarded = _run(SHOW_NOGIL, _env(tmp_path / "nogil", {"cache": False, "nogil": True}), tmp_path) + assert forwarded.returncode == 0 and forwarded.stdout.split() == ["True", "True", "True"], forwarded.stderr