Sync upstream/main: merge PR 13 (the bounds test reads memory it owns, the options test covers all three decorators)

.github/scripts/mutation_guard_check.py

@@ -217,6 +217,27 @@
         '                          Optional(Array(uint8, 1, "C", readonly=True))))',
         "@jit_with_options(None)",
     ),
+    (
+        "is_null's decorator stops reading the options",
+        "numbarrow/core/is_null.py",
+        '@jit_with_options(boolean(int64, Array(uint8, 1, "C", readonly=True)))',
+        '@__import__("numba").njit(boolean(int64, Array(uint8, 1, "C", readonly=True)), cache=False)',
+    ),
+    (
+        "unpack_booleans' decorator stops reading the options",
+        "numbarrow/core/is_null.py",
+        '@jit_with_options(Array(bool_, 1, "C")(int64, int64, Array(uint8, 1, "C", readonly=True)))',
+        '@__import__("numba").njit(Array(bool_, 1, "C")(int64, int64, Array(uint8, 1, "C", readonly=True)), '
+        'cache=False)',
+    ),
+    (
+        "is_null_struct's decorator stops reading the options",
+        "numbarrow/core/is_null.py",
+        '@jit_with_options(boolean(int64, Optional(Array(uint8, 1, "C", readonly=True)),\n'
+        '                          Optional(Array(uint8, 1, "C", readonly=True))))',
+        '@__import__("numba").njit(boolean(int64, Optional(Array(uint8, 1, "C", readonly=True)),\n'
+        '                          Optional(Array(uint8, 1, "C", readonly=True))), cache=False)',
+    ),
     (
         "viewers stop getting a cache name of their own",
         "numbarrow/utils/utils.py",
@@ -700,6 +721,20 @@
         '            cache_failed = isinstance(error, OSError) or "no locator available" in str(error)',
         '            cache_failed = True',
     ),
+    (
+        'the cache option stops reaching unpack_booleans while every other option does',
+        'numbarrow/core/configurations.py',
+        '            return njit(signature, **jit_options)(func)',
+        '            return njit(signature, **{**jit_options, "cache": jit_options.get("cache")'
+        ' and func.__name__ != "unpack_booleans"})(func)',
+    ),
+    (
+        'the decorators stop forwarding any option but cache and boundscheck',
+        'numbarrow/core/configurations.py',
+        '            return njit(signature, **jit_options)(func)',
+        '            return njit(signature, **{key: value for key, value in jit_options.items()'
+        ' if key in ("cache", "boundscheck")})(func)',
+    ),
     (
         'the repeated-name check stops looking inside a map',
         'numbarrow/core/mapinarrow_factory.py',

test/test_cache.py

@@ -288,10 +288,15 @@ def test_a_read_only_install_warns_naming_numba_cache_dir_and_setting_it_caches(
             path.chmod(0o755)
 
 
+# The bitmap is the first byte of an array that holds every byte the calls
+# reach, so without bounds checking they read past the bitmap and stay inside
+# memory the array owns. Behind a one-byte array of its own they read whatever
+# followed it on the heap, and where that was an unmapped page the child died
+# of an access violation, as it did once on Windows.
 CHECK_BOUNDS = (
     "import numpy as np\n"
     "from numbarrow.core.is_null import is_null, unpack_booleans\n"
-    "bitmap = np.zeros(1, dtype=np.uint8)\n"
+    "bitmap = np.zeros(100000 // 8 + 1, dtype=np.uint8)[:1]\n"
     "outcomes = []\n"
     "for call in (lambda: is_null(100000, bitmap), lambda: unpack_booleans(0, 100000, bitmap)):\n"
     "    try:\n"
@@ -302,12 +307,33 @@ def test_a_read_only_install_warns_naming_numba_cache_dir_and_setting_it_caches(
     "print(' '.join(outcomes))\n"
 )
 
+SHOW_NOGIL = (
+    "from numbarrow.core.is_null import is_null, unpack_booleans, is_null_struct\n"
+    "print(' '.join(str(f.targetoptions.get('nogil')) for f in (is_null, unpack_booleans, is_null_struct)))\n"
+)
+
 
 def test_jit_options_reach_the_is_null_decorators(tmp_path):
     # The options test imported only the viewers, so hard-coding the options
     # on is_null.py's three decorators kept the suite green, and the documented
-    # boundscheck contract had no test at all.
+    # boundscheck contract had no test at all. Bounds checking shows on the two
+    # functions that index a bitmap; is_null_struct indexes nothing and calls
+    # is_null, whose own flags decide, so on its decorator the option that
+    # shows is the cache: off, nothing reaches the cache directory from any of
+    # the three, and on, each of the three leaves an index there. An option
+    # that shows on none of them, nogil, is read back from the three
+    # dispatchers, so a decorator forwarding only the two that show is caught
+    # as well.
     checked = _run(CHECK_BOUNDS, _env(tmp_path / "checked", {"cache": False, "boundscheck": True}), tmp_path)
     assert checked.returncode == 0 and checked.stdout.split() == ["IndexError", "IndexError"], checked.stderr
     unchecked = _run(CHECK_BOUNDS, _env(tmp_path / "unchecked", {"cache": False}), tmp_path)
     assert unchecked.returncode == 0 and unchecked.stdout.split() == ["returned", "returned"], unchecked.stderr
+    assert _index_files(tmp_path / "checked") == [] and _index_files(tmp_path / "unchecked") == []
+    cached = _run(CHECK_BOUNDS, _env(tmp_path / "cached", {"cache": True}), tmp_path)
+    assert cached.returncode == 0 and cached.stdout.split() == ["returned", "returned"], cached.stderr
+    indexes = _index_files(tmp_path / "cached")
+    assert len(indexes) == 3, indexes
+    for function in ("is_null-", "unpack_booleans", "is_null_struct"):
+        assert any(function in name for name in indexes), (function, indexes)
+    forwarded = _run(SHOW_NOGIL, _env(tmp_path / "nogil", {"cache": False, "nogil": True}), tmp_path)
+    assert forwarded.returncode == 0 and forwarded.stdout.split() == ["True", "True", "True"], forwarded.stderr