Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 35 additions & 0 deletions .github/scripts/mutation_guard_check.py
Original file line number Diff line number Diff line change
Expand Up @@ -217,6 +217,27 @@
' Optional(Array(uint8, 1, "C", readonly=True))))',
"@jit_with_options(None)",
),
(
"is_null's decorator stops reading the options",
"numbarrow/core/is_null.py",
'@jit_with_options(boolean(int64, Array(uint8, 1, "C", readonly=True)))',
'@__import__("numba").njit(boolean(int64, Array(uint8, 1, "C", readonly=True)), cache=False)',
),
(
"unpack_booleans' decorator stops reading the options",
"numbarrow/core/is_null.py",
'@jit_with_options(Array(bool_, 1, "C")(int64, int64, Array(uint8, 1, "C", readonly=True)))',
'@__import__("numba").njit(Array(bool_, 1, "C")(int64, int64, Array(uint8, 1, "C", readonly=True)), '
'cache=False)',
),
(
"is_null_struct's decorator stops reading the options",
"numbarrow/core/is_null.py",
'@jit_with_options(boolean(int64, Optional(Array(uint8, 1, "C", readonly=True)),\n'
' Optional(Array(uint8, 1, "C", readonly=True))))',
'@__import__("numba").njit(boolean(int64, Optional(Array(uint8, 1, "C", readonly=True)),\n'
' Optional(Array(uint8, 1, "C", readonly=True))), cache=False)',
),
(
"viewers stop getting a cache name of their own",
"numbarrow/utils/utils.py",
Expand Down Expand Up @@ -700,6 +721,20 @@
' cache_failed = isinstance(error, OSError) or "no locator available" in str(error)',
' cache_failed = True',
),
(
'the cache option stops reaching unpack_booleans while every other option does',
'numbarrow/core/configurations.py',
' return njit(signature, **jit_options)(func)',
' return njit(signature, **{**jit_options, "cache": jit_options.get("cache")'
' and func.__name__ != "unpack_booleans"})(func)',
),
(
'the decorators stop forwarding any option but cache and boundscheck',
'numbarrow/core/configurations.py',
' return njit(signature, **jit_options)(func)',
' return njit(signature, **{key: value for key, value in jit_options.items()'
' if key in ("cache", "boundscheck")})(func)',
),
(
'the repeated-name check stops looking inside a map',
'numbarrow/core/mapinarrow_factory.py',
Expand Down
30 changes: 28 additions & 2 deletions test/test_cache.py
Original file line number Diff line number Diff line change
Expand Up @@ -288,10 +288,15 @@ def test_a_read_only_install_warns_naming_numba_cache_dir_and_setting_it_caches(
path.chmod(0o755)


# The bitmap is the first byte of an array that holds every byte the calls
# reach, so without bounds checking they read past the bitmap and stay inside
# memory the array owns. Behind a one-byte array of its own they read whatever
# followed it on the heap, and where that was an unmapped page the child died
# of an access violation, as it did once on Windows.
CHECK_BOUNDS = (
"import numpy as np\n"
"from numbarrow.core.is_null import is_null, unpack_booleans\n"
"bitmap = np.zeros(1, dtype=np.uint8)\n"
"bitmap = np.zeros(100000 // 8 + 1, dtype=np.uint8)[:1]\n"
"outcomes = []\n"
"for call in (lambda: is_null(100000, bitmap), lambda: unpack_booleans(0, 100000, bitmap)):\n"
" try:\n"
Expand All @@ -302,12 +307,33 @@ def test_a_read_only_install_warns_naming_numba_cache_dir_and_setting_it_caches(
"print(' '.join(outcomes))\n"
)

SHOW_NOGIL = (
"from numbarrow.core.is_null import is_null, unpack_booleans, is_null_struct\n"
"print(' '.join(str(f.targetoptions.get('nogil')) for f in (is_null, unpack_booleans, is_null_struct)))\n"
)


def test_jit_options_reach_the_is_null_decorators(tmp_path):
# The options test imported only the viewers, so hard-coding the options
# on is_null.py's three decorators kept the suite green, and the documented
# boundscheck contract had no test at all.
# boundscheck contract had no test at all. Bounds checking shows on the two
# functions that index a bitmap; is_null_struct indexes nothing and calls
# is_null, whose own flags decide, so on its decorator the option that
# shows is the cache: off, nothing reaches the cache directory from any of
# the three, and on, each of the three leaves an index there. An option
# that shows on none of them, nogil, is read back from the three
# dispatchers, so a decorator forwarding only the two that show is caught
# as well.
checked = _run(CHECK_BOUNDS, _env(tmp_path / "checked", {"cache": False, "boundscheck": True}), tmp_path)
assert checked.returncode == 0 and checked.stdout.split() == ["IndexError", "IndexError"], checked.stderr
unchecked = _run(CHECK_BOUNDS, _env(tmp_path / "unchecked", {"cache": False}), tmp_path)
assert unchecked.returncode == 0 and unchecked.stdout.split() == ["returned", "returned"], unchecked.stderr
assert _index_files(tmp_path / "checked") == [] and _index_files(tmp_path / "unchecked") == []
cached = _run(CHECK_BOUNDS, _env(tmp_path / "cached", {"cache": True}), tmp_path)
assert cached.returncode == 0 and cached.stdout.split() == ["returned", "returned"], cached.stderr
indexes = _index_files(tmp_path / "cached")
assert len(indexes) == 3, indexes
for function in ("is_null-", "unpack_booleans", "is_null_struct"):
assert any(function in name for name in indexes), (function, indexes)
forwarded = _run(SHOW_NOGIL, _env(tmp_path / "nogil", {"cache": False, "nogil": True}), tmp_path)
assert forwarded.returncode == 0 and forwarded.stdout.split() == ["True", "True", "True"], forwarded.stderr
Loading