diff --git a/docs/conf.py b/docs/conf.py index 0e94e2da..0059b5e6 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -14,8 +14,8 @@ project = "numbox" -copyright = "2025, Mikhail Goykhman" -author = "Mikhail Goykhman" +copyright = "2026, NumbOx GitHub Repository Contributors" +author = "NumbOx GitHub Repository Contributors" # -- General configuration --------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration diff --git a/docs/numbox.core.proxy.rst b/docs/numbox.core.proxy.rst index 325688fd..ad355e90 100644 --- a/docs/numbox.core.proxy.rst +++ b/docs/numbox.core.proxy.rst @@ -56,8 +56,9 @@ with the rest. From numba 0.61 onward ``.as_func`` inherits the mixed-container limit of any derive value: a tuple holding it alongside a plain ``CompileResultWAP`` no longer unifies, failing with a message-less ``AssertionError`` from -``numba.core.utils.unified_function_type``. See :doc:`numbox.core.work` for that -limit and for why making the two types compare equal is not available as a fix. +``numba.core.utils.unified_function_type`` (fixed in numba==0.68.0). +See :doc:`numbox.core.work` for that limit and for why making the two types +compare equal is not available as a fix. Referencing ``.as_func`` as a constant carries a caching caveat that passing it as a function-type argument does not; read `Cache invalidation`_ below before diff --git a/docs/numbox.core.work.rst b/docs/numbox.core.work.rst index 81b0d16e..787475b3 100644 --- a/docs/numbox.core.work.rst +++ b/docs/numbox.core.work.rst @@ -591,20 +591,21 @@ These limits are worth knowing: - A container that mixes a `cres` derive with a differently typed function value: a tuple holding a `cres` alongside a plain ``CompileResultWAP``, a signature-declared njit dispatcher or a ``cfunc``. numba unifies the element types before any of numbox's - conversions apply, and ``numba.core.utils.unified_function_type`` requires every - function type it meets to equal the first through a bare ``assert``, so the failure - arrives as an ``AssertionError`` carrying no message. A lazily compiled ``@njit`` - dispatcher in the mix is the one subcase that reads differently: unification accepts - it, and numbox's guard then rejects it at the unboxing boundary with a ``TypeError`` - naming the offending type. + conversions apply, and ``numba.core.utils.unified_function_type``, for numba<0.68.0, + requires every function type it meets to equal the first through a bare ``assert``, + so the failure arrives as an ``AssertionError`` carrying no message (fixed in numba==0.68.0). + A lazily compiled ``@njit`` dispatcher in the mix is the one subcase that reads + differently: unification accepts it, and numbox's guard then rejects it at the unboxing + boundary with a ``TypeError`` naming the offending type. From numba 0.61 onward a ``@proxy`` binding's ``.as_func`` is a `DeriveFunctionType` value as well (see :doc:`numbox.core.proxy`), so it meets this limit identically: a - tuple mixing it with a plain ``CompileResultWAP`` fails on the same assertion, where - before it unified and returned a value. + tuple mixing it with a plain ``CompileResultWAP`` fails on the same assertion, for numba<0.68, + where before it unified and returned a value. Homogeneous containers are unaffected, including a tuple of two `cres` derives, or a `cres` derive alongside a `@proxy` binding's `.as_func` of the same signature. This is not specific to numbox: two plain ``CompileResultWAP`` values of different - signatures but the same argument count fail identically with numbox uninvolved. + signatures but the same argument count fail identically with numbox uninvolved, + for numba<0.68. What numbox changes is how easily the case is reached, since `DeriveFunctionType` is a distinct type from ``FunctionType`` and numba compares function types by class. Making the two compare equal is not available as a fix. numba interns types in a cache keyed by a diff --git a/numbox/core/configurations.py b/numbox/core/configurations.py index 8c9f5dfc..375dac44 100644 --- a/numbox/core/configurations.py +++ b/numbox/core/configurations.py @@ -18,6 +18,10 @@ from numba.core.types import FunctionType, void +numba_version = int(version("numba").split(".")[1]) +assert numba_version >= 60, numba_version + + def get_jit_options(): """ E.g., export NUMBOX_JIT_OPTIONS='{"cache": false}' @@ -87,17 +91,22 @@ def check_cache_location(py_file, longest_file_name=0): def is_a_cache_error(error): - """Whether ``error`` is numba's for a cache it cannot set up: no locator, or a location it cannot use. + """Whether ``error`` is numba's for a cache it cannot set up: no locator, a location it cannot use, or + missing source files(s) on numba>=0.68. numba itself passes over a location on any ``OSError`` from making its directory or writing a file there, permission denied, a read-only file system, a path into a file, a component too long, a full disk, so any ``OSError`` counts. So does the ``ValueError`` numba raises for an archive under a directory whose name holds ``.zip``: its ``.zip`` locator takes the file by that substring and then finds no ``.zip`` in it. + numba>=0.68 started hashing the source contents of the archive into the archive's cache stamp, therefore + missing .py raise KeyError. """ if isinstance(error, OSError): return True if isinstance(error, ValueError): return "No zip file found" in str(error) + if isinstance(error, KeyError): + return numba_version >= 68 and "There is no item named" in str(error) return isinstance(error, RuntimeError) and "no locator available" in str(error) @@ -237,6 +246,14 @@ def cache_remedy(py_file, failure, silence, package="numbox"): files on disk: a package built on numbox that puts the question for its own files with ``check_cache_location`` and ``is_a_cache_error`` passes its own name, as it passes ``check_cache_location`` a ``longest_file_name`` for its own functions, ``LONGEST_CACHE_FILE_NAME`` being numbox's. + + On numba==0.68.0, ``KeyError`` caused by the Windows ZIP path separator + bug https://github.com/numba/numba/issues/10889 can be worked around by + disabling caching. The bug is scheduled to be fixed in 0.68.1. + + On numba>=0.68, ``KeyError`` caused by missing .py source files in the ZIP + archive (when only compiled `.pyc` files are present) can be avoided by either + disabling caching or including all source files in the archive. """ if os.path.exists(py_file): if isinstance(failure, OSError) and failure.errno == errno.ENAMETOOLONG: @@ -249,18 +266,37 @@ def cache_remedy(py_file, failure, silence, package="numbox"): # numba itself passes a location it cannot make or write over, for a # source on disk, so the error here is the no-locator one. return f"Set NUMBA_CACHE_DIR to a writable directory, or {silence}" + if isinstance(failure, KeyError): + if "\\" in str(failure) and os.name == "nt": + return ( + "Numba 0.68's ZIP cache locator uses Windows path separators for ZIP members, " + "see https://github.com/numba/numba/issues/10889" + ) + return f"For numba>=0.68, put all source .py file(s) in .zip, or {silence}" if isinstance(failure, OSError) or getattr(sys, "frozen", False): # The .zip's error names the location numba picked, which is under # the user's cache directory; the frozen application's names nothing. location = getattr(failure, "filename", None) or AppDirs(appname="numba", appauthor=False).user_cache_dir + if ( + isinstance(failure, OSError) + and failure.errno == errno.EISDIR and location and location.endswith(".zip") + and os.path.isdir(location) and py_file.startswith(location + os.sep) + ): + return ( + f"the path {location} is a directory whose name ends in .zip, " + "which numba mistakes for the source archive; rename that " + "directory, or use a path without a .zip-named directory, " + f"or {silence}" + ) if isinstance(failure, FileNotFoundError) and py_file.startswith(failure.filename + os.sep): # The stamp numba reads at decoration, of the archive the code # names: .pyc members compiled to name an archive since moved. A # location that cannot be made, under a dangling link, is ENOENT # too, and is not under the code's file. + suggestion = "" if numba_version >= 68 else "compile the archive's .pyc members to name its path now, or " return ( f"the module's code names {failure.filename}, which is not there: its .pyc was compiled to name " - "that path, so compile the archive's .pyc members to name its path now, or ship its source files; " + f"that path, {suggestion}ship its source files; " f"or {silence}" ) if isinstance(failure, OSError) and failure.errno == errno.ENAMETOOLONG: @@ -316,7 +352,7 @@ def uncached_where_no_cache_can_be_written(options): for py_file in _module_files(): try: check_cache_location(py_file, LONGEST_CACHE_FILE_NAME) - except (RuntimeError, OSError, ValueError) as error: + except (RuntimeError, OSError, KeyError, ValueError) as error: if not is_a_cache_error(error): raise failure = error @@ -356,10 +392,6 @@ def _strict_cache_mode(): MAX_STR_LENGTH = 2 ** 31 - 1 - -numba_version = int(version("numba").split(".")[1]) -assert numba_version >= 60, numba_version - #: numba's `FunctionModel`, resolved once. Looking a data model up is type machinery and #: compiles nothing, which is what lets the layout be read here: `numbox.utils.lowlevel` and #: `numbox.utils.derive_wap` both need it on paths that must stay free of compilation side diff --git a/numbox/core/variable/variable.py b/numbox/core/variable/variable.py index a9c05bc1..e763ba89 100644 --- a/numbox/core/variable/variable.py +++ b/numbox/core/variable/variable.py @@ -385,7 +385,7 @@ def _calculate(self, nodes: list[CompiledNode], values: Storage): raise RuntimeError(f"Uninitialized input {input_.qual_name()} for {node.variable}") args[i] = arg if self.debug: - print(f"Calculating {node}\nwith metadata\n{node.variable.metadata}", file=sys.stderr) + print(f"Calculating {node}\nwith metadata\n{node.variable.metadata}", file=sys.stderr, flush=True) result = node.variable.formula(*args) values.get(node.variable).value = result diff --git a/numbox/utils/preprocessing.py b/numbox/utils/preprocessing.py index fad4bbea..faf671e2 100644 --- a/numbox/utils/preprocessing.py +++ b/numbox/utils/preprocessing.py @@ -155,7 +155,7 @@ def _cached_at_or_uncached(py_file: str, jit_options: dict) -> dict: return jit_options try: check_cache_location(py_file, LONGEST_CACHE_FILE_NAME) - except (RuntimeError, OSError, ValueError) as error: + except (RuntimeError, OSError, KeyError, ValueError) as error: if not is_a_cache_error(error): raise failure = error diff --git a/pyproject.toml b/pyproject.toml index f9c75e3f..d06285b9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ include = ["numbox*"] [project] name = "numbox" dependencies = [ - "numba>=0.60.0,<0.68.0" + "numba>=0.60.0,<0.69.0" ] dynamic = ["version"] requires-python = ">=3.10" diff --git a/test/core/test_no_cache_location.py b/test/core/test_no_cache_location.py index 29b5c49b..86601a1f 100644 --- a/test/core/test_no_cache_location.py +++ b/test/core/test_no_cache_location.py @@ -13,6 +13,7 @@ import marshal import os import py_compile +import pytest import re import shutil import subprocess @@ -20,7 +21,6 @@ import zipfile from pathlib import Path -import pytest REPO = Path(__file__).resolve().parent.parent.parent @@ -130,6 +130,9 @@ def test_options_without_a_cache_key_are_asked_for_the_sites_that_cache_under_th assert run.stderr.count("compiles without a cache") == 1, run.stderr +windows_path_sep_bug = "Numba 0.68's ZIP cache locator uses Windows path separators for ZIP members" + + def test_a_zip_import_is_cached_by_numba_from_0_61(tmp_path): # The warning sends an archive's user to a .zip, which numba caches from # 0.61 on, in the user's cache directory whatever NUMBA_CACHE_DIR says. @@ -142,6 +145,8 @@ def test_a_zip_import_is_cached_by_numba_from_0_61(tmp_path): NUMBA_CACHE_DIR=str(tmp_path / "cache")) env.pop("NUMBOX_JIT_OPTIONS", None) run = _run(env, tmp_path) + if windows_path_sep_bug in run.stderr: + pytest.skip(windows_path_sep_bug) assert run.returncode == 0 and str(archive) in run.stdout, run.stderr cached = tuple(int(part) for part in numba.__version__.split(".")[:2]) >= (0, 61) assert ("compiles without a cache" not in run.stderr) == cached, run.stderr @@ -297,6 +302,8 @@ def test_a_pyc_in_a_zip_asks_by_the_file_it_was_compiled_from(tmp_path, tree): env.pop("NUMBOX_JIT_OPTIONS", None) run = subprocess.run([sys.executable, "-W", "always", "-c", IMPORT_LIBM], capture_output=True, text=True, env=env, cwd=str(tmp_path)) + if windows_path_sep_bug in run.stderr: + pytest.skip(windows_path_sep_bug) assert run.returncode == 0 and str(archive) in run.stdout, run.stderr assert run.stderr.count("compiles without a cache") == 1, run.stderr assert "source is not a file on disk" in run.stderr and "holding its source files" in run.stderr, run.stderr @@ -331,6 +338,8 @@ def test_a_stale_pyc_beside_its_source_in_a_zip_is_passed_over_as_zipimport_pass env.pop("NUMBOX_JIT_OPTIONS", None) run = subprocess.run([sys.executable, "-W", "always", "-c", IMPORT_LIBM], capture_output=True, text=True, env=env, cwd=str(tmp_path)) + if windows_path_sep_bug in run.stderr: + pytest.skip(windows_path_sep_bug) assert run.returncode == 0 and run.stdout.strip().endswith("libm.py"), run.stderr if _zip_is_cached(): assert "compiles without a cache" not in run.stderr, run.stderr @@ -357,6 +366,8 @@ def test_a_stale_pyc_whose_source_in_the_zip_does_not_compile_is_passed_over(tmp NUMBA_CACHE_DIR=str(tmp_path / "cache")) env.pop("NUMBOX_JIT_OPTIONS", None) run = _run(env, tmp_path) + if windows_path_sep_bug in run.stderr: + pytest.skip(windows_path_sep_bug) assert run.returncode == 0 and str(archive) in run.stdout, run.stderr if _zip_is_cached(): assert "compiles without a cache" not in run.stderr, run.stderr @@ -382,6 +393,8 @@ def test_a_stray_pyc_in_a_zip_that_nothing_imports_is_passed_over(tmp_path, dama NUMBA_CACHE_DIR=str(tmp_path / "cache")) env.pop("NUMBOX_JIT_OPTIONS", None) run = _run(env, tmp_path) + if windows_path_sep_bug in run.stderr: + pytest.skip(windows_path_sep_bug) assert run.returncode == 0 and str(archive) in run.stdout, run.stderr if _zip_is_cached(): assert "compiles without a cache" not in run.stderr, run.stderr @@ -480,6 +493,8 @@ def test_a_moved_zip_whose_pyc_members_name_its_old_path_compiles_uncached_and_i env.pop("NUMBOX_JIT_OPTIONS", None) run = subprocess.run([sys.executable, "-W", "always", "-c", IMPORT_LIBM], capture_output=True, text=True, env=env, cwd=str(tmp_path)) + if windows_path_sep_bug in run.stderr: + pytest.skip(windows_path_sep_bug) assert run.returncode == 0 and str(moved) in run.stdout, run.stderr assert run.stderr.count("compiles without a cache") == 1, run.stderr if _zip_is_cached(): @@ -557,6 +572,10 @@ def test_a_zip_import_whose_location_for_one_directory_stopped_being_writable_ta env.pop("NUMBOX_JIT_OPTIONS", None) warm = _run(env, tmp_path) assert warm.returncode == 0, warm.stderr + if "use a path without a .zip-named directory" in warm.stderr: + pytest.skip("numba's caching doesn't distinguish directory with .zip in its name from an archive from 0.68 on") + if "put all source .py file(s) in .zip" in warm.stderr: + pytest.skip("numba's caching demands .py sources in a .zip from 0.68 on") if not _zip_is_cached(): pytest.skip("numba caches a .zip from 0.61 on") # Under XDG_CACHE_HOME on Linux, under Library/Caches on macOS. diff --git a/test/core/test_proxy.py b/test/core/test_proxy.py index 0bb0723d..f0c27813 100644 --- a/test/core/test_proxy.py +++ b/test/core/test_proxy.py @@ -18,6 +18,7 @@ from numbox.core.bindings.errno import errno_get from numbox.core.bindings.libc import getenv, memcpy from numbox.core.bindings.call import _call_lib_func +from numbox.core.configurations import numba_version from numbox.core.proxy.proxy import proxy, proxy_if_available, make_proxy_name from numbox.utils.derive_wap import DeriveFunctionType, DeriveWAP, jit_addr_supported from numbox.utils.lowlevel import array_data_p, get_unicode_data_p @@ -602,10 +603,11 @@ def call_through_declared_plain(f, x): @pytest.mark.skipif(not jit_addr_supported(), reason=_JIT_ADDR_REASON) -def test_proxy_as_func_mixed_with_a_numba_native_wrapper_fails_to_unify(): - """Characterization of a narrowing: a heterogeneous tuple of function values is refused. +def test_proxy_as_func_mixed_with_a_numba_native_wrapper_fails_to_unify_for_numba_lt_68_and_works_for_numba_gte_68(): + """Characterization of a narrowing: a heterogeneous tuple of function values is refused + for numba<0.68. - numba unifies a tuple's element types in ``unified_function_type`` with a bare + numba<0.68.0 unifies a tuple's element types in ``unified_function_type`` with a bare class-identity comparison, which runs before any of numbox's conversions get a say, so a ``DeriveFunctionType`` element beside a plain ``FunctionType`` one trips a message-less ``AssertionError`` out of ``numba/core/utils.py``. The same tuple returned a value while @@ -615,7 +617,7 @@ def test_proxy_as_func_mixed_with_a_numba_native_wrapper_fails_to_unify(): Why the two types cannot simply be made to compare equal is worked through in ``test/utils/test_derive_wap.py::test_the_derive_type_stays_distinct_from_the_plain_function_type``. - numba's assert carries no message, so the frame it was raised from is checked instead of any + numba<0.68.0's assert carries no message, so the frame it was raised from is checked instead of any text. Matching on ``AssertionError`` alone would be satisfied by an unrelated one raised anywhere in the same call. """ @@ -629,11 +631,14 @@ def native(x): def use_pair(pair, x): return pair[0](x) + pair[1](x) - with pytest.raises(AssertionError) as raised: - use_pair((aux_raises.as_func, foreign), -3.0) - assert raised.traceback[-1].name == "unified_function_type", ( - f"the refusal moved out of numba's function-type unification: {raised.traceback[-1].name}" - ) + if numba_version >= 68: + assert use_pair((aux_raises.as_func, foreign), -3.0) == 5 + else: + with pytest.raises(AssertionError) as raised: + use_pair((aux_raises.as_func, foreign), -3.0) + assert raised.traceback[-1].name == "unified_function_type", ( + f"the refusal moved out of numba's function-type unification: {raised.traceback[-1].name}" + ) if __name__ == "__main__":