diff --git a/docs/numbox.core.configurations.rst b/docs/numbox.core.configurations.rst index f6292475..6e587e10 100644 --- a/docs/numbox.core.configurations.rst +++ b/docs/numbox.core.configurations.rst @@ -41,7 +41,11 @@ user's cache directory, the archive's directories are listed and the question pu ``.pyc`` in the archive that zipimport would run, which it takes before the ``.py`` beside it unless it is stale against it or of another interpreter, and whose code keeps the file it was compiled from, is asked by that file, since that is what numba looks up for it, on disk or gone, and one zipimport would pass over is -passed over; any other archive has no location at all. A directory of the package reached through a symlink +passed over, and a ``.pyc`` whose code names a member the archive does not hold, one alone with no ``.py`` beside +it or one compiled to name another path in the archive, ``compileall -d``, beside its ``.py`` or not, is asked for +itself whichever member stands for its directory, since numba 0.68 and later read the member's source from the +archive by the name its code gives and find none for it; any other archive has no location at all. A directory of +the package reached through a symlink is walked like the rest, wherever the link points. The check makes the cache directories it asks about, as numba would at the first decoration in each; with caching beside the sources that is an empty ``__pycache__`` per directory of the package, a linked one included. numba's own writability check makes a temporary file, @@ -51,20 +55,184 @@ the check here makes a file named as long as the longest numba writes for the pa which a test holds every function of the package under), so that location turns caching off with the warning instead. -- For a source file on disk the remedy is ``NUMBA_CACHE_DIR`` pointed at a writable directory; where its - location is too long for the file system, a shorter ``NUMBA_CACHE_DIR`` or none, since numba's location - under the variable is a directory named for the source's directory, by its name and a hash of its path, and - without the variable numba caches beside the source; else the package installed at a shorter path, where the - location beside the source is the long one. -- For a ``.zip``, or a frozen application, it is the user's cache directory made writable: a ``.zip`` is the - one archive numba caches, from 0.61 on, and it caches it there, taking the directory without checking that - it can be written; a frozen application (``sys.frozen``) is cached there too, its sources not being on disk. +- For a source file on disk that numba cannot read, which its IPython locator reads for the function's source and + every locator reads for the stamp from numba 0.68, the remedy is to make it readable. For one it can, the remedy + is ``NUMBA_CACHE_DIR`` pointed at a writable directory; where it is + set and numba passed it over, the warning names it and asks for a writable directory at a short path, since + numba passes over one too deep to make its directory in as it does an unwritable one; and where + ``NUMBA_CACHE_LOCATOR_CLASSES``, from numba 0.62, leaves the user-provided locator out, it is the locations + the list names made writable, numba never reading the variable. Where the + location is too long for the file system, the remedy is for the location the error names: numba takes a + directory under ``NUMBA_CACHE_DIR`` where that is set, else the ``__pycache__`` beside the source, else a + directory under the user's cache directory, each of the two under a cache directory named for the source's + directory, by its name and a hash of its path; the path the error names is matched whole against each location + a locator numba reaches gives it, in the order numba tries them, since the three can nest and + ``NUMBA_CACHE_DIR`` set to the user's cache directory makes one path of two, and the error is read as that + locator's, the ones numba tries before it having passed the file over, as numba's own errors mean and an error a + caller passes on naming a location numba did not take need not; numba reaches no locator listed + after its ``.zip`` one for a path with ``.zip`` in it, which that locator takes without trying its location, so + a location one of those would give is none of numba's. So a shorter ``NUMBA_CACHE_DIR`` for the first; the + package installed at a shorter path for the second; the user's cache directory at a shorter path, through + ``XDG_CACHE_HOME`` or ``HOME``, ``HOME`` alone on macOS and nothing on Windows, where numba asks the system, + for the third; and for either of the last two ``NUMBA_CACHE_DIR`` set to a short path, where numba tries it + before the locator that took the location, or, where it is set and numba passed it over, unwritable or too + deep, the warning names it and the remedy is a writable directory at a short path. The directory numba names after + the source's, under ``NUMBA_CACHE_DIR`` or the user's cache directory, is that directory's name and forty-one + bytes, and one longer than a name can be on the file system holding the location, which the system is asked for, + 255 where it cannot say, in bytes, or in the UTF-16 units Windows and macOS count a name in, can be made at no + path, and is refused whatever the errno, EINVAL on Windows: an error naming such a location, numba's no-locator + error + for a file on disk where every locator it tried that names its directory so found the name too long for its file + system, the variable's unset by that limit, and a frozen application's are told the name and its length, and the + file in a directory of a shorter name, with, for a file on disk, the ``__pycache__`` beside the source made + writable where the in-tree locator is listed, or that locator listed where it is not, which caches under no such + name. A cache file numba could not make or open in the location it took, whose name, given after the source's name + and the function's qualified name, is longer than a name can be on that file system, is told the name and its + length, which no shorter path cures, and a shorter name for the function or its file, the package's to give, + and the check's error for a name as long as the package's functions get, which that file system takes no name + as long as, is told the length, with + ``NUMBA_CACHE_DIR`` pointed at a directory on a file system that takes a name that long offered where a file + system in use takes the name, 255 being their limit, less the 29 of the longest name numba writes for the + function, a data file's with three digits for its number, numba numbering them from one without bound, through + the temporary name it writes each file through, + and numba tries the variable before the locator that took + the file, as it is offered at a short path for a path too long, and nothing offered for a ``.zip`` member, which + the variable does nothing for. + numba hashes the source's directory path as UTF-8 for that name and raises for bytes that are not, at + decoration, where a locator that names its directory so reaches the file, so for such a path the remedy names no + location under ``NUMBA_CACHE_DIR`` or the user's cache directory, and a ``.zip`` under such a path, whose + archive's path numba's ``.zip`` locator hashes the same way for its directory under the user's cache directory, + is told the archive under a UTF-8 path, a frozen application under one, whose path numba's user-wide locator + hashes the same way, the application under a UTF-8 path, and a file on disk under one is told the + ``__pycache__`` beside it too, + which numba's in-tree locator hashes no path for: that locator to list before the first that hashes, where the + list leaves it out or puts it after one, ``NUMBA_CACHE_DIR`` to unset where its locator alone hashes before the + in-tree one, or that directory to make writable where the in-tree locator came first and passed it over, and + with any of these where it cannot be written. + A location that is none of + numba's is named as the error names it, with the variable, where numba tries it before any other locator, named + as set or asked for at a short path, and nothing said of numba passing it over, which that location cannot show. + The name an error carries is read as a string, bytes or a path object alike, and a name of another kind, an integer + file descriptor's, as none. An error can name no file, or carry an + empty string or empty bytes as its name. A caller can build such an error, and numba's first save raises one where a + write fails, as on a full disk. So does inspect where, for numba's IPython locator, it reads no source in a cell file + on disk, unreadable or, before Python 3.13, empty, with no errno either, no call of the file system's having raised + it, and that error, + for a file on disk that a locator of IPython's family numba reaches takes, is told as inspect's, that locator having + tried no location: the file to make readable, with the source in it; for a cell, whose source numba reads from + linecache, where IPython holds it while it runs the cell, as that, numba caching a cell only while IPython holds + its source; and for a ``.zip`` member whose archive holds no source, a ``.pyc`` alone, as the archive's, its source + files to ship. Every other OSError that ``check_cache_location`` + raises names a file. An error with no name, or an empty + one, is told the locations numba could have taken, in numba's order and each once. For a path too long, the remedy is + to put whichever of them is too long at a shorter path. For a file refused for another reason, such as a full disk, + the remedy gives the reason and asks for room or a writable directory there. A path object is never empty: pathlib + reads an empty string as the working directory, which the remedy names as a location that is none of numba's. + ``NUMBA_CACHE_LOCATOR_CLASSES`` decides that order, each entry a class of numba's caching module or, by its + dotted path, a subclass of one, which takes what its parent takes, caches where it does and is told as it is, or a + class of none of numba's families, with a ``from_function`` and a place of its own, which takes what its + ``from_function``, asked with a probe as numba asks it, takes, where it caches being its own to say: an error + naming a location that no locator of + numba's gives the file is told that location as the error names it, which is then that class's, and one naming no + file is told the location numba took as the class's own, with no path to name, among numba's locators' places where + those are reached; an + entry numba cannot resolve, which it refuses before any location is tried, or raises importlib's error for, a + relative module path's TypeError, and one that names no class, or a class with no ``from_function``, which it + fails on at a function it decorates where it reaches the entry, are no locators here, and numba's own error for + either is raised as it was. + numba's IPython locator caches a cell file it takes in ``numba_cache`` under IPython's cache directory, with no + directory per file, and reads the function's source when it takes the file, as inspect reads it, a file on disk + itself, a cell from linecache and a ``.zip`` member through the module's loader, which the check's probe has, a + module of its own whose loader reads the member from the archive as zipimport's does, the member's lines then in + linecache as any reader leaves them; that location too long is told as + IPython's cache directory at a shorter path, for a cell file on disk and for a ``.zip`` member in an ipykernel + directory alike, where that locator is listed before the ``.zip`` one, which takes a file without trying its + location, so that none listed after it is reached, with no ``NUMBA_CACHE_DIR`` offered for a source not on + disk; a cell, which no other locator of numba's takes, gets numba's no-locator error where that locator passes its + location over, and is told that location, to make writable or put at a shorter path, or IPython's cache directory + to make a home for where IPython raised for it, with a class of none of numba's families listed with it named as + having passed the cell over too; the remedy asks IPython for that directory where a locator of that family is + listed + and takes the file, and leaves IPython alone for every other file, a plain ``.zip`` member among them, since + IPython warns when asked under a home it cannot write and leaves a temporary directory behind, a new one on + every call where it can write neither its cache directory nor its own, in which case numba makes ``numba_cache`` + in one and saves in another, the check raises the error numba's first save would, ``FileNotFoundError`` naming + the directory, an error of another errno naming a ``numba_cache`` being a caller's, told as the location it + names, and the warning asks for + ``IPYTHONDIR``, or ``XDG_CACHE_HOME`` on a POSIX system, where IPython 8 and later read it, at a writable path; + numba + imports IPython for the location when it makes it and catches only OSError there, so where IPython is not + importable it raises ImportError at decoration for a file that locator takes, no cache error, and an error a + caller passes on naming that location is told it as none of numba's where another locator numba reaches has a + place for the file, and the list where none has, as numba's no-locator error is told it; and where IPython raises + OSError for its directory numba passes the locator over, as it does a location it cannot make, and the warning, + asked the same, has no location of IPython's to match the error against. numba's IPython locator takes a file on + disk + only in an ipykernel directory and its ``.zip`` locator only a path with ``.zip`` in it, so either ahead of + the rest changes nothing for any other file; the ``.zip`` locator ahead of them all takes such a path first and + caches it under the user's cache directory where a part of the path ends in ``.zip``, an archive or a + directory, or, where none does, finds no archive + in it, and the warning then asks for a locator for a file on disk listed before it, the user-provided one with + ``NUMBA_CACHE_DIR`` set, which alone it takes nothing without; after some of them, it finds no archive once + those have passed the file over, numba trying none after it, and the warning asks for one of their locations + made writable or such a locator listed before it. A list with no locator that takes the file is told so, for an + error a caller passes on under it as for numba's no-locator one, and so is one whose only locator for the file is + the user-provided one with ``NUMBA_CACHE_DIR`` unset, which takes nothing without it: the variable to set, as + numba's no-locator error is told there. Where the location numba took refuses a file for another reason, a + full disk or permissions changed since numba's own check, the warning names the location and the reason and + asks for room or a writable directory there, with ``NUMBA_CACHE_DIR`` as the alternative where numba tries it + before that location, or set to another directory where the location is the variable's own; numba passes over + a location it cannot make or write in, so only its no-locator error means the variable was passed over. An + error naming a location that is none of numba's is told that location as the error names it, and one naming + no file is told the locations numba could have taken. +- For a ``.zip``, or a frozen application, it is the user's cache directory made writable, or room made there, + with the reason the error gives, a full disk or permissions: a ``.zip`` is the one archive numba caches, from + 0.61 on, and it caches it there, taking the directory without checking that it can be written; the error names + that location, or a file numba's first save writes in it, and the location is told with the reason; an error + naming a location that is none of numba's, under the list and in its order, is told that location as the error + names it, as for a source on disk, with nothing of what moves the user's cache directory and nothing of where + numba caches the file, which that location is none of; a frozen + application (``sys.frozen``) is cached there too, its sources not being on disk, and its error is numba's + no-locator one, which gives no reason, numba having passed the location over on its error, unwritable or too deep + alike, so that directory is named as one numba could not use, to be made writable or put at a shorter path, or, + where the name numba gives the directory after the source's is too long for the file system, told so, as is a + cache file's name in that location, or in IPython's ``numba_cache`` for a member in an ipykernel directory, too + long for the file system, which no shorter path cures. + A frozen application's path with ``.zip`` in it and no part ending in it gets the ``.zip`` locator's error instead, + where the list puts that locator before the user-wide one or once the user-wide one passed its location over, and + the warning asks for the user-wide locator listed before it, or for that location made writable or put at a + shorter path. numba reads ``NUMBA_CACHE_DIR`` only for a source file on disk, so the variable changes nothing for either. + An error that names no file, a caller's, or numba's first save's on a full disk, is told the locations numba + could have taken, as for a source on disk, in its order and each once, with the reason, or, for a path too long, + to put whichever is too long at a shorter path; for a member in an ipykernel directory where a locator of + IPython's family is reached, the warning says nothing of the user's cache directory as where numba caches the + file, and IPython's ``numba_cache`` is among them where IPython gives its directory. A ``.zip`` whose cache directory holds every entry but can no longer be written falls back too, where numba alone would have loaded the entries: the writability check is the rule numba applies to every other - placement. + placement. numba caches a ``.zip`` through its ``.zip`` locator alone, so a ``NUMBA_CACHE_LOCATOR_CLASSES`` + that leaves that locator out gives numba no locator for a source in a ``.zip``, and the warning asks for it to + be listed; so for a notebook cell and numba's IPython locator, and for a frozen application and numba's + user-wide locator, which alone takes one; any of them is told + for an error a caller passes on under such a list as for numba's no-locator one, since numba took no location + under it, and a source neither in a ``.zip`` nor a frozen application's, which no locator takes, is told the + source files on disk or a ``.zip`` holding them for any error. - For an ``.egg``, ``.whl`` or ``.pyz``, a ``.pyc``-only install or a ``.pyc`` in a ``.zip``, it is the source - files on disk or a ``.zip`` holding them. + files on disk or a ``.zip`` holding them, each with the locator it needs listed where + ``NUMBA_CACHE_LOCATOR_CLASSES`` leaves it out: one of the three locators for a file on disk for the first, the + ``.zip`` locator for the second, and ``NUMBA_CACHE_DIR`` set where the user-provided locator is the one listed for + a file on disk, which takes nothing without it. +- From numba 0.68 the archive's stamp hashes each member's source, read from the archive by the name the + module's code gives under the archive's path: a member shipped as ``.pyc`` alone is told the source files, on + disk or in a ``.zip``, naming the member; on Windows under numba 0.68.0 every member is refused, its name + built with the platform's separator (`numba issue 10889 `_, + fixed for 0.68.1), and the warning asks for the install on disk, or a release carrying the fix; and a directory + whose name + ends in ``.zip`` above the + archive, which numba takes for the archive by the first such part of the path and from 0.68 opens for the + source, is told as the directory it is, to put the archive under a path with none such above it; a source + file on disk under such a directory, which that locator takes once the locators before it passed the file + over, is told to make one of their locations writable or to put the file elsewhere. - ``NUMBOX_JIT_OPTIONS='{"cache": false}'`` turns caching off and silences the warning in every case, the package's options being what it sets; the anchors' warning under a caller's own options, below, is silenced by those. diff --git a/docs/numbox.core.proxy.rst b/docs/numbox.core.proxy.rst index ad355e90..36aec306 100644 --- a/docs/numbox.core.proxy.rst +++ b/docs/numbox.core.proxy.rst @@ -53,10 +53,10 @@ inferred-signature ``@njit`` argument keeps the numbox type and propagates, from numba 0.61 onward; on 0.60 there is no numbox type to keep and it discards along with the rest. -From numba 0.61 onward ``.as_func`` inherits the mixed-container limit of any -derive value: a tuple holding it alongside a plain ``CompileResultWAP`` no longer -unifies, failing with a message-less ``AssertionError`` from -``numba.core.utils.unified_function_type`` (fixed in numba==0.68.0). +From numba 0.61 to 0.67 ``.as_func`` inherits the mixed-container limit of any +derive value: a tuple holding it alongside a plain ``CompileResultWAP`` does not +unify, failing with a message-less ``AssertionError`` from +``numba.core.utils.unified_function_type``; numba 0.68.0 unifies it, as 0.60 did. See :doc:`numbox.core.work` for that limit and for why making the two types compare equal is not available as a fix. diff --git a/docs/numbox.utils.rst b/docs/numbox.utils.rst index a944c4bf..aa8f2b14 100644 --- a/docs/numbox.utils.rst +++ b/docs/numbox.utils.rst @@ -141,11 +141,16 @@ opcode encodes an *index* into ``co_consts`` rather than the value itself, so two methods differing only in a numeric literal (``return self.x + 1`` vs ``return self.x + 1000``) produce identical ``co_code``. With a shared anchor file both would resolve to the same -``cache_subpath`` (numba's -``_CacheLocator.get_suitable_cache_subpath`` derives the cache subdir -from a hash of ``co_filename``) and the second to compile would -silently load the first's binary. Per-content anchors segregate the -two by ``cache_subpath`` so the collision never arises. +cache files: numba names them after the anchor file's stem and the +function's qualified name (``CacheImpl.get_filename_base``), in one +directory for them all, the ``__pycache__`` beside the anchors where +numba's in-tree locator takes them or, under ``NUMBA_CACHE_DIR`` or +the user's cache directory, a subdirectory derived from a hash of the +anchors' *directory* (``_CacheLocator.get_suitable_cache_subpath``), +so the second to compile would silently load the first's binary. +Per-content anchors segregate the two by the anchor's stem, which +numba puts in the cache files' names, so the collision never arises; +every anchor shares that one directory. Structural body changes -- different operators, additional statements, renamed variables -- produce different ``co_code`` and @@ -178,16 +183,29 @@ reports a component too long as a syntax error, and the warning then offers the directory, quoting the error); with the names bounded, as below, the directory is the only part that can make it so, ``NUMBA_CACHE_DIR`` or the user's cache directory, and ``NUMBA_CACHE_DIR`` at a shorter -path moves the anchor out of either. The anchor's check writes names -shorter than the longest numba writes for a struct, which carry its name -and its methods', so a user cache directory within about 230 bytes of -the path limit, 4096 on Linux, passes it and overflows at numba's first -save instead, where numba's own error names the length; the remedy is -the same. A ``NUMBA_CACHE_DIR`` that deep is too deep for the package's -own files first, whose location under it appends their directory's path -and whose check reserves the length of numba's files, so the package -answers, with its remedy, and the generated code compiles under its -answer. +path moves the anchor out of either. A cache directory whose path is +not UTF-8 is another: numba hashes the anchor's directory path as UTF-8 +to name its cache directory, and the warning asks for ``NUMBA_CACHE_DIR`` +at a UTF-8 path. The anchor's check writes a name as long as the +longest numba writes for the functions it anchors, read off the +generated code, the functions decorated themselves or defined within a +decorated one, an overload's constructor among them: the stem, the +function's qualified name, its line, Python's version, three digits +for the data file's number, which numba counts from one without +bound, and the 21 of the temporary name numba writes each +file through, so a user cache directory +or a ``NUMBA_CACHE_DIR`` within that of the path limit, 4096 on Linux, +fails it and the generated code compiles without a cache after the +warning; the check wrote a short name there, which numba's own check +passes too, and numba's first save died through the struct's making +with no warning at all, and a bound for any struct, 251, then turned +caching off on a file system whose name limit is under it for a struct +whose names were a quarter of that. The package's own check answers +first only nearer the limit, within about 175 bytes of it: the package's +location under ``NUMBA_CACHE_DIR`` is a directory named for its own, by +its name and a hash of its path, and its check reserves the length of +numba's files for the package's functions, so it fails only where those +no longer fit; the generated code then compiles under its answer. The anchor's name carries the struct's or the function's, and numba names its cache files after the anchor and the qualified name of the diff --git a/numbox/core/configurations.py b/numbox/core/configurations.py index 375dac44..775500fe 100644 --- a/numbox/core/configurations.py +++ b/numbox/core/configurations.py @@ -1,10 +1,17 @@ +import ast +import contextlib import errno import importlib.machinery import inspect +import itertools import os +import pathlib import json +import linecache +import re import sys import tempfile +import types import warnings import zipfile import zipimport @@ -54,55 +61,186 @@ def _cache_probe(): # this bound. LONGEST_CACHE_FILE_NAME = 128 +# The digits reserved for the number numba gives a function's data file, one per signature, counted from one +# without bound: three, a thousandth signature of one function being beyond what the package and its generated +# code make; a name an error carries with a wider number is read by its own width. Two digits, a ninety-ninth +# signature's, left the hundredth's name a byte longer than the reserve. +_OVERLOAD_DIGITS = 3 + +# The probe's module, one per check, named apart from every other's, stands in +# sys.modules while numba picks the locator. +_probes = itertools.count() + + +class _MemberSource: + """The loader of the check's probe's module, which inspect reads a ``.zip`` member's source through, as zipimport + for the member's own module: the member's bytes decoded as zipimport decodes them, as UTF-8, or None where no + archive above the file holds the member, or it is not UTF-8, which zipimport raises for and linecache lets + through: inspect's error tells such a member, and the remedy reads why.""" + + def __init__(self, py_file): + self._py_file = py_file + + def get_source(self, name): + found = _archive_above(self._py_file) + if found is None: + return None + zip_path, member = found + try: + with zipfile.ZipFile(zip_path) as archive: + return archive.read(member).decode() + except (KeyError, OSError, zipfile.BadZipFile, UnicodeDecodeError): + return None + + +@contextlib.contextmanager +def _probing(py_file): + """A probe, a function never compiled whose code names ``py_file``, in ``sys.modules`` as a function of a module + of its own for the block, which asks a locator about the file as numba asks it at decoration. + + ``py_file``, a string, bytes or a path object, is read as a string, the name the probe's code carries, as compile + makes it of bytes. numba's IPython locator reads the function's source when it takes the file, as inspect reads + it: a file on disk itself, a cell from linecache, where IPython holds it while it runs the cell, and a ``.zip`` + member through the function's module's loader, zipimport's, which gives the member's source from the archive, or + None for a member compiled to ``.pyc`` alone; the probe's module's loader reads the member as zipimport's does, + and inspect looks the function's module up in ``sys.modules``. The lines linecache holds from then on are the + member's own, as any reader leaves them; for a member the archive holds no source for linecache keeps the probe's + loader to ask again, and the block takes it out at its end, a loader of the probe's own staying for no file. + """ + file = os.fsdecode(py_file) + # The probe's own two lines, put in linecache under the member's name + # instead while numba picked the locator, were what a reader of the + # member meanwhile, in another thread, read for it. + module = types.ModuleType(f"_numbox_cache_probe_{next(_probes)}") + module.__file__ = file + module.__loader__ = _MemberSource(file) + exec(compile("def _cache_probe():\n pass\n", file, "exec"), module.__dict__) # nosec B102 - fixed source + held = file in linecache.cache + sys.modules[module.__name__] = module + try: + yield module._cache_probe + finally: + sys.modules.pop(module.__name__, None) + if not held and len(linecache.cache.get(file, ())) == 1: + linecache.cache.pop(file, None) + + +def _takes(cls, py_file): + """Whether ``cls``, a locator class of none of numba's families, takes ``py_file``: its ``from_function``, asked + with a probe as numba asks it at decoration, gives a locator for a file it takes and None for one it passes over. + A class whose ``from_function`` raises takes the file, the error being its own, as numba raises it at decoration; + the remedy took such a class for taking any file, and read the file as its own where it had passed it over and + numba had gone on to the locators listed after it.""" + with _probing(py_file) as probe: + try: + return cls.from_function(probe, os.fsdecode(py_file)) is not None + except Exception: + return True + def check_cache_location(py_file, longest_file_name=0): - """Raise as numba would where a function whose source is ``py_file`` cannot be cached; else return. + """Raise as numba would where a function whose source is ``py_file``, a string, bytes or a path object, read as a + string, cannot be cached; else return. The question is put the way numba puts it: the cache set-up that decoration runs, which picks the location for the file or raises ``RuntimeError`` with no locator, then the source's stamp, which decoration reads - and which stats the archive for a ``.zip``, then the writability check, which decoration runs for every + and which stats the archive for a ``.zip``, reading the member's source from it too from numba 0.68, as it then + reads a source on disk for its hash, then the + writability check, which decoration runs for every location but a ``.zip``'s and the first save runs for all, raising ``OSError``. The probe is compiled with - ``py_file`` as its file, which is all a locator reads of it. Nothing is compiled, and nothing is written - but the cache directory itself. An ``OSError`` from the check names the location numba picked. + ``py_file`` as its file, which is all a locator reads of it but for numba's IPython locator, which reads the + function's source too, as inspect reads it: a file on disk itself, a cell from linecache, where IPython holds it + while it runs the cell, and a ``.zip`` member through the function's module's loader, zipimport's, which gives the + member's source from the archive; the probe is a function of a module of its own, whose loader reads the member + as zipimport's does, in ``sys.modules`` while numba picks the locator, and the lines linecache holds from then on + are the member's own, as any reader leaves them; for a member the archive holds no source for linecache keeps the + probe's loader to ask again, and the check takes it out, a loader of its own staying for no file. A cell linecache + does not hold, or a member compiled to ``.pyc`` alone, raises inspect's error, as numba does at decoration. + Nothing is + compiled, and nothing is written but the cache directory itself and, asked for it under an IPython that can write + neither its cache directory nor its own, the temporary directories IPython makes, one per call, three for the + check, two of them numba's own making the directory. An ``OSError`` from the check names the + location numba picked, but for three: inspect's, which names no file, for a cell file on disk that numba's + IPython locator takes, where inspect reads the source from the file itself, before that locator tries its + location, and finds none there, in a file it cannot read or, before Python 3.13, an empty one, which linecache + gives a blank line from then on; the stamp's, which names the archive a ``.zip`` member's code names where that + has since moved or cannot be read, read once the locator is picked, as numba's own does at decoration; and, from + numba 0.68, where + the stamp hashes the source, the stamp's naming the source itself, which cannot be read, or the directory named + ``.zip`` above a file on disk, which numba's ``.zip`` locator takes for the archive and opens. A location that + is not there once made, numba's IPython locator asking IPython for its directory + at every save and IPython giving a new temporary one on every call where it can write neither its cache + directory nor its own, raises ``FileNotFoundError`` naming the last directory IPython gave the check, which is + not there, as numba's first save names the one IPython gave it, a temporary directory in either case and a + different one. numba's writability check makes a temporary file whose name is short, or none at all on Linux, and the files it saves have names of up to a hundred bytes and more, so a location within their length of the path limit passes the check and the first save dies. Given ``longest_file_name``, a file with a name that long is made and removed in the location too, and the ``OSError`` is the location's: the package asks with - ``LONGEST_CACHE_FILE_NAME``, the bound on numba's names for its own files. + ``LONGEST_CACHE_FILE_NAME``, the bound on numba's names for its own files. Where that name is longer than the + file system holding the location takes, the error carries the length as ``longest_file_name``, no shorter + path curing it, and the remedy asks for a file system that takes it. """ - namespace = {} - exec(compile("def _cache_probe():\n pass\n", os.fspath(py_file), "exec"), namespace) # nosec B102 - fixed source - probe = namespace["_cache_probe"] - locator = CompileResultCacheImpl(probe).locator + # What a locator is asked about: a probe, in a module of its own. + with _probing(py_file) as probe: + locator = CompileResultCacheImpl(probe).locator # numba reads the source's stamp at decoration too, the archive's for a # .zip, which is not there where the code names an archive since moved. locator.get_source_stamp() try: locator.ensure_cache_path() - if longest_file_name: - # tempfile's name is the prefix and eight characters of its own. - with tempfile.NamedTemporaryFile(dir=locator.get_cache_path(), prefix="x" * (longest_file_name - 8)): - pass except OSError as error: # The error names what failed, a temporary file's name among the # possibilities; the location it is in is what a reader can act on. raise OSError(error.errno, error.strerror, locator.get_cache_path()) from error + # The location is asked for once more, as numba asks at every save, and + # its IPython locator asks IPython, which gives a new temporary directory + # on every call where it can write neither its cache directory nor its + # own: numba makes numba_cache in one and saves in another, dying there + # past its own check; the check passed the file the same way, and asked + # for the location at every step, a temporary directory each, where numba + # asks twice making it and the check once more. + location = locator.get_cache_path() + if not os.path.isdir(location): + raise FileNotFoundError(errno.ENOENT, "No such file or directory", location) + if longest_file_name: + try: + # tempfile's name is the prefix and eight characters of its own. + with tempfile.NamedTemporaryFile(dir=location, prefix="x" * (longest_file_name - 8)): + pass + except OSError as error: + raised = OSError(error.errno, error.strerror, location) + if error.errno == errno.ENAMETOOLONG and _name_limit(location) < longest_file_name: + # The name alone is too long for the file system, not the + # path: the error carries the length, for the remedy, which + # read it as the location's and offered a shorter path. + raised.longest_file_name = longest_file_name + raise raised from error def is_a_cache_error(error): - """Whether ``error`` is numba's for a cache it cannot set up: no locator, a location it cannot use, or - missing source files(s) on numba>=0.68. + """Whether ``error`` is numba's for a cache it cannot set up: no locator, a location it cannot use, or, from + numba 0.68, a ``.zip`` member whose source it cannot read from the archive. numba itself passes over a location on any ``OSError`` from making its directory or writing a file there, permission denied, a read-only file system, a path into a file, a component too long, a full disk, so any ``OSError`` counts. So does the ``ValueError`` numba raises for an archive under a directory whose name - holds ``.zip``: its ``.zip`` locator takes the file by that substring and then finds no ``.zip`` in it. - numba>=0.68 started hashing the source contents of the archive into the archive's cache stamp, therefore - missing .py raise KeyError. + holds ``.zip``: its ``.zip`` locator takes the file by that substring and then finds no ``.zip`` in it. And + so does the ``KeyError`` numba 0.68 and later raise reading a ``.zip`` member's source from the archive, + which they hash into the archive's stamp: zipfile's, naming the member, for a name the archive does not + hold, a module shipped as ``.pyc`` alone, or, on Windows under numba 0.68.0, any member, the name built with + the platform's separator (numba issue 10889, fixed in 0.68.1). And so does the ``UnicodeEncodeError`` numba + raises hashing the source's directory path as UTF-8 for the name of its directory under ``NUMBA_CACHE_DIR`` or + the user's cache directory, for a path holding bytes that are not, decoded with surrogates: a locator that + names its directory so raises it at decoration where it is reached, the ones before it having passed the file + over, and no location takes the file; a package installed under such a directory, its ``__pycache__`` + unwritable, died at import on it where, with caching off, it imports. """ if isinstance(error, OSError): return True + if isinstance(error, UnicodeEncodeError): + return error.reason == "surrogates not allowed" if isinstance(error, ValueError): return "No zip file found" in str(error) if isinstance(error, KeyError): @@ -166,15 +304,20 @@ def _sourceless_compiled_from(pyc_path): def _archived_module_files(own): - """One module per directory of numbox inside the ``.zip`` that holds ``own``, this module as imported, and the - file each ``.pyc`` in it that zipimport would run was compiled from; nothing for any other archive. + """One module per directory of numbox inside the ``.zip`` that holds ``own``, this module as imported, the + file each ``.pyc`` in it that zipimport would run was compiled from, and each ``.pyc`` whose code names a member + the archive does not hold, alone with no ``.py`` beside it or compiled to name another path in the archive, for + itself; nothing for any other archive. numba caches a ``.zip`` per directory of it, each in a location of its own under the user's cache directory, so the directories answer separately there too. A ``.pyc`` run from the archive keeps the file it was compiled from as its code's file, and that is what numba looks up for its functions, the archive's own path for the - module is not; so such a member asks by that file, there or gone. The archive is the first part of the path - named ``.zip`` that is one, a directory so named above it being no archive. Any other archive has no location - at all, and the probe's own file has already asked. + module is not; so such a member asks by that file, there or gone. numba 0.68 and later read the member's source + from the archive by the name its code gives, and a ``.pyc`` alone has none there, nor has one compiled to name + another path in the archive, ``compileall -d``, beside its ``.py`` or not, so either asks for itself whichever + member stands for its directory. The archive is the first part of the path named ``.zip`` that is one, a + directory so named above it being no archive. Any other archive has no location at all, and the probe's own + file has already asked. """ parts = own.split(os.sep) for depth, part in enumerate(parts): @@ -188,6 +331,7 @@ def _archived_module_files(own): names = sorted(name for name in archive.namelist() if name.startswith(package + "/") and name.endswith((".py", ".pyc")) and "/__pycache__/" not in name) + held = set(names) seen = set() for name in names: directory, _, file = name.rpartition("/") @@ -198,6 +342,21 @@ def _archived_module_files(own): if not member.startswith(zip_path + os.sep): yield member continue + named = member[len(zip_path) + 1:].replace(os.sep, "/") + if named not in held: + # The member the code names is not in the archive: a .pyc + # alone names the .py it was compiled from, gone, and one + # compiled to name another path in the archive, compileall's + # -d, names a member that was never there, beside its .py or + # not. numba 0.68 and later read the member's source from the + # archive by that name, and find none, so it is asked for + # itself whichever member stands for its directory; a listing + # that let a .py member stand for the directory never asked, + # and the module died at decoration, as it did for a .pyc + # beside its .py that a listing took for naming the archive. + seen.add(directory) + yield member + continue else: member = os.path.join(zip_path, *name.split("/")) if directory not in seen: @@ -212,8 +371,9 @@ def _compiled_from(zip_path, directory, stem): magic number, nor one stale against that ``.py``, which it passes over for the source, nor one it cannot unmarshal, and the module's import fails with it; so its answer, its code, is read. The file that names is the archive's own path for the module where zipimport compiled the ``.py``, or where the ``.pyc`` was - compiled to name the archive, and stands for the directory as a ``.py`` member does; else it is the file - elsewhere the ``.pyc`` was compiled from, there or gone, and asks for itself. No code at all, for whatever + compiled to name a member the archive holds, and stands for the directory as a ``.py`` member does; else it + is a file the archive does not hold, elsewhere or a path in it the ``.pyc`` was compiled to name, there or + gone, and asks for itself. No code at all, for whatever stops zipimport, a source it cannot compile or a member it cannot decompress among the reasons, leaves nothing to ask. """ @@ -226,96 +386,1588 @@ def _compiled_from(zip_path, directory, stem): return None +def _moved_through(): + """What moves numba's user cache directory on this platform, as a clause for a remedy; nothing on Windows. + + numba's ``appdirs`` asks the system for it on Windows, which no variable changes, puts it under ``HOME`` on + macOS, and takes ``XDG_CACHE_HOME``, else a directory under ``HOME``, elsewhere. + """ + if sys.platform == "win32": + return "" + if sys.platform == "darwin": + return ", through HOME" + return ", through XDG_CACHE_HOME or HOME" + + +def _measured(name): + """``name``'s length as the file system counts it, in bytes, or in the UTF-16 units Windows and macOS count a name + in, and the unit's name. A surrogate, which a name decoded with surrogateescape carries for a byte that is not + UTF-8, and which Windows takes in a name, is one unit; the count raised on it, and the remedy died on a name + numba's error carried instead of answering.""" + if os.name == "nt" or sys.platform == "darwin": + return len(name.encode("utf-16-le", "surrogatepass")) // 2, "characters" + return len(os.fsencode(name)), "bytes" + + +def _name_limit(location): + """The longest name a file can have where ``location`` is: 255 where the system cannot say, or says 0, as a file + system with no limit of its own reports it, or for none. + + In bytes, which the file systems elsewhere count a name in, or in the UTF-16 units Windows and macOS count one in. + + Asked of the nearest directory above ``location`` that exists, since the location itself may not be there yet. + """ + if location is None or not hasattr(os, "statvfs"): + return 255 + path = os.path.abspath(location) + while not os.path.exists(path) and os.path.dirname(path) != path: + path = os.path.dirname(path) + try: + return os.statvfs(path).f_namemax or 255 + except OSError: + return 255 + + +def _class_named(module, name): + """The class of this name in ``module``, as numba spells its locators from 0.62, or, under 0.60 and 0.61, with the + underscore it put before them until then; None where the module has no such class. The underscore is tried under + those two alone, which read no ``NUMBA_CACHE_LOCATOR_CLASSES``, so that a listed name numba refuses from 0.62, + ``CacheLocator`` for its base class, is left out as numba leaves it.""" + cls = getattr(module, name, None) + if cls is None and numba_version < 62: + cls = getattr(module, "_" + name, None) + return cls + + +def _numba_locator(name): + """numba's cache locator class of this name, or None where this numba has none: the ``.zip`` locator arrived in + 0.61.""" + from numba.core import caching + return _class_named(caching, name) + + +def _locators(listed): + """numba's cache locator classes in the order it tries them. + + ``listed`` is ``NUMBA_CACHE_LOCATOR_CLASSES`` as numba 0.62 and later read it, each entry a class of numba's + caching module by its name or its dotted path, resolved as numba resolves it, with the underscore numba put before + its locators' names until 0.62 allowed in either; an entry numba could not resolve + it has refused already, before any location was tried, or raised importlib's error for, a relative module path's + TypeError, and one that names no class, or a class with no ``from_function``, it fails on, for want of + ``from_function``, at a function it decorates where it reaches the entry, so neither is a locator here, and + numba's own error for either is raised as it was. Empty, the order is numba's own. + """ + from numba.core import caching + if not listed: + return list(caching.CacheImpl._locator_classes) + classes = [] + for entry in listed.split(","): + entry = entry.strip() + if "." in entry: + module_path, class_name = entry.rsplit(".", 1) + try: + cls = _class_named(importlib.import_module(module_path), class_name) + except Exception: + # importlib's errors, a relative path's TypeError, and + # whatever the module raises on import, which numba raises + # as its own at decoration and the check as its own before + # any remedy: the remedy, reached for an error of the + # cache's, raised the module's for the entry. + cls = None + else: + cls = _numba_locator(entry) + if isinstance(cls, type) and callable(getattr(cls, "from_function", None)): + classes.append(cls) + return classes + + +def _existing_above(path): + """The first path at or above ``path`` that is there: the directory a directory on the way to ``path`` is made + under, which refused where the location is not there, or a file in the way, which no directory can be made under; + the root at the last. ``path`` is read absolute: a relative one with no ancestor on the way walked up to nothing, + and the remedy said a file with no name was in the way.""" + path = os.path.abspath(path) + while not os.path.exists(path): + parent = os.path.dirname(path) + if parent == path: + break + path = parent + return path + + +def _ipython_numba_cache(): + """numba's cache location for a cell file its IPython locator takes: ``numba_cache`` under IPython's cache + directory, with no directory per file; None where IPython is not importable, which numba's locator raises + ImportError for at decoration, no cache error, so the location is then none of numba's, and None where IPython + raises for its directory: OSError, which numba's locator passes the file over on, as it does a location it cannot + make, so that numba took no location of IPython's, or anything else, which numba's locator would raise at + decoration and which the remedy, asked the same for the words of a warning, takes as no location given.""" + try: + try: + from IPython.paths import get_ipython_cache_dir + except ImportError: + from IPython.utils.path import get_ipython_cache_dir + return os.path.join(get_ipython_cache_dir(), "numba_cache") + except Exception: + return None + + +# What inspect says where it reads no source: a cell linecache does not hold, a member the archive holds no source +# for, a function whose line the lines it read do not reach, and a function of no module. +_INSPECTS_ERRORS = ("could not get source code", "source code not available", "lineno is out of bounds") + + +def _is_inspects(failure): + """Whether ``failure`` is inspect's, for a source it could not read: an ``OSError`` with no errno and no name and + one of inspect's own messages; a caller's error with no errno and no name is not, though it looks the same.""" + return (isinstance(failure, OSError) and failure.errno is None and not getattr(failure, "filename", None) + and str(failure) in _INSPECTS_ERRORS) + + +def _ipython_cache_is_temporary(first): + """Whether IPython gives a new temporary directory as its cache directory on every call, which it does, warning + so, where it can write neither its cache directory nor its own; numba's IPython locator asks for the directory at + every save, so numba makes ``numba_cache`` in one and saves in another. ``first`` is one answer already given, + and one more is asked for; each is a directory IPython leaves behind. An IPython that gives a directory once and + raises the next time gives no temporary directory: the second answer, none, counted as a new one.""" + if first is None: + return False + second = _ipython_numba_cache() + return second is not None and second != first + + +def _taken_by_ipython(py_file): + """Whether numba's IPython locator takes ``py_file`` by its name alone, on disk or not: a cell, ````, or + a file in an ipykernel directory. It reads the function's source when it does.""" + return py_file.startswith("=0.68, ``KeyError`` caused by missing .py source files in the ZIP - archive (when only compiled `.pyc` files are present) can be avoided by either - disabling caching or including all source files in the archive. + From numba 0.68 the archive's stamp hashes the member's source, read from the archive by the name the + module's code gives under the archive's path, and the ``KeyError`` zipfile raises for a name the archive does + not hold is told the source files, on disk or in a ``.zip``, naming the member, a module shipped as ``.pyc`` + alone; on Windows under numba 0.68.0 every member gets that error, its name built with the platform's + separator (numba issue 10889, fixed for 0.68.1), and the remedy is the install on disk, or a release carrying + the fix, where the archive holds the member's source under the name it reads, else the source files as for any + member. The + first part of the path ending in ``.zip`` is what numba takes for the archive, a directory so named above it + included, and from 0.68 + opens to read the source: the ``IsADirectoryError`` is told the directory, to put the archive under a path + with none such above it; a file on disk under such a directory, which that locator takes once the ones before it + passed the file over, is told to make one of their locations writable or to put the file elsewhere. A file in an + ipykernel directory that numba's IPython locator takes, a cell file on disk + or a ``.zip`` member, where IPython gives a new temporary directory as its cache directory on every call, able + to write neither that directory nor its own, is told so for an error naming a file, as the one numba's first save + raises there is ``FileNotFoundError`` naming the directory it could not save in, with ``IPYTHONDIR`` or + ``XDG_CACHE_HOME`` at a writable path as the remedy, and an error of another errno naming a ``numba_cache`` is a + caller's, told as the location it names: numba makes ``numba_cache`` in one such directory and saves in another; + an error naming no + file is told the locations numba could have taken, with its reason, as above. """ - if os.path.exists(py_file): - if isinstance(failure, OSError) and failure.errno == errno.ENAMETOOLONG: - # numba's own check passes the location, its temporary file - # fitting where its cache files would not; the error names it. + from numba import config + from numba.core.caching import _CacheLocator + # check_cache_location takes any path-like, bytes too, and so does this: + # the str the remedy reads, which os.fspath would leave bytes as bytes. + py_file = os.fsdecode(py_file) + # The error names the file refused as the call that raised it was given + # it, bytes for one given bytes, or as a caller built it, a path object + # too, and the remedy reads the name as str, as it reads the source's. An + # empty str or bytes is no name; a path object built from an empty string + # names the working directory, ".", as pathlib reads it; a name of another + # kind, an integer file descriptor's as os.stat(fd) raises it, is none. + filename = getattr(failure, "filename", None) + filename = os.fsdecode(filename) if filename and isinstance(filename, (str, bytes, os.PathLike)) else None + listed = getattr(config, "CACHE_LOCATOR_CLASSES", "") + user_provided = _numba_locator("UserProvidedCacheLocator") + in_tree = _numba_locator("InTreeCacheLocator") + user_wide = _numba_locator("UserWideCacheLocator") + for_ipython = _numba_locator("IPythonCacheLocator") + for_a_zip = _numba_locator("ZipCacheLocator") + + def one_of(cls, base): + # Whether ``cls`` is ``base`` or a subclass of it: a subclass, listed + # by its dotted path, takes what its parent takes and caches where it + # does, as numba's InTreeCacheLocatorFsAgnostic does, so each of + # numba's locators stands for its family. + return base is not None and issubclass(cls, base) + + def of_numbas(cls): + # Whether ``cls`` is of one of numba's locator families, whose places + # the remedy knows; a class of none of them, with a from_function and + # a place of its own, caches where it alone can say. + return any(one_of(cls, base) for base in (user_provided, in_tree, user_wide, for_ipython, for_a_zip)) + + def could_have_taken(locations): + # Which location numba took cannot be told from an error that names + # no file, so the ones it could have taken, the values of + # ``locations`` in numba's order, are listed each once: on disk, + # NUMBA_CACHE_DIR set to the user's cache directory makes one path of + # two. A class of none of numba's families has a place the remedy + # cannot name, listed as the class's own. + known = list(dict.fromkeys( + location if location else f"{cls.__name__}'s own" + for cls, location in locations.items() if location or not of_numbas(cls))) + if len(known) > 1: + return f"the location numba took, one of {', '.join(known[:-1])} or {known[-1]}" + return f"the location numba took, {known[0]}" + + def listed_without(base): + # Whether NUMBA_CACHE_LOCATOR_CLASSES is set and names no class of + # ``base``'s family. + return base is not None and bool(listed) and not any(one_of(cls, base) for cls in _locators(listed)) + + try: + subpath = _CacheLocator.get_suitable_cache_subpath(py_file) + except UnicodeEncodeError: + # numba hashes the source's directory path as UTF-8 for the name, + # and raises for bytes that are not, at decoration, where a locator + # that names its directory so reaches the file; asked for a file + # none did, the remedy has no such location to name. + subpath = None + + def too_long_a_name(location): + # numba names the directory it caches the file in, under + # NUMBA_CACHE_DIR or the user's cache directory, after the source's + # directory, by its name and a hash of its path, and a name longer + # than the file system holding ``location`` takes can be made at no + # path, so no shorter one cures it: the clause that says so, or None + # where the name fits, or numba has none to give. + if subpath is None: + return None + limit = _name_limit(location) + length, counted_in = _measured(subpath) + if length <= limit: + return None + return ( + f"numba names the directory it caches this file in after the source's directory, {subpath}, {length} " + f"{counted_in}, which is longer than a name can be on that file system ({limit})" + ) + + a_shorter_name = f"a shorter name for the function or its file, which is {package}'s to give" + + def a_name_limit_below(location): + # The check's error for the file of the longest name the package's + # functions get, which the file system holding ``location`` takes no + # name as long as: the error carries the length, and the clause + # says so, or None for any other error. The remedy read the error as + # the location's and offered a shorter path, which cures nothing of + # a name. + longest = getattr(failure, "longest_file_name", 0) + if not longest or location is None: + return None + limit = _name_limit(location) + if limit >= longest: + return None + length, counted_in = _measured("x" * longest) + return ( + f"numba's cache files for {package}'s functions have names of up to {length} {counted_in}, longer than " + f"a name can be on that file system ({limit})" + ) + + def too_long_a_file_name(location): + # numba names the files it caches a function in after the source's + # name and the function's qualified name, and a name longer than the + # file system holding ``location`` takes can be made at no path, so + # no shorter one cures it, a shorter name alone: the clause that says + # so for the file the error names in ``location``, and whether a file + # system in use takes the name at all, 255 being the limit of ext4, + # NTFS and APFS alike, in the units each counts, for the longest name + # numba writes for the function: the index, ".nbi", and a data file + # per signature, "..nbc", numbered from one without bound, each + # through a temporary name 21 longer, ".tmp." and sixteen hex digits, + # the data file's with the digits reserved for its number, or the + # number the error names where that is wider; or + # None where the error names no such file, or its name fits. The + # remedy asked for the package, the user's cache directory, IPython's + # or the variable at a shorter path, which move nothing of the name, + # then offered a file system that takes the name where none in use + # does, never naming the shorter name, then one that takes the name + # where numba's save through the temporary name failed on it, and + # then one where the index fit and the data file did not. + if not filename or location is None: + return None + named = os.path.abspath(filename) + if os.path.dirname(named) != os.path.abspath(location): + return None + name = os.path.basename(named) + length, counted_in = _measured(name) + limit = _name_limit(location) + if length <= limit: + return None + clause = ( + f"numba names the cache files for a function after the source's name and the function's qualified " + f"name, {name}, {length} {counted_in}, which is longer than a name can be on that file system ({limit})" + ) + stripped = re.sub(r"\.tmp\.[0-9a-f]{16}$", "", name) + numbered = re.search(r"\.(\d+)\.nbc$", stripped) + digits = max(_OVERLOAD_DIGITS, len(numbered.group(1))) if numbered else _OVERLOAD_DIGITS + base = re.sub(r"\.nbi$|\.\d+\.nbc$", "", stripped) + return clause, _measured(base)[0] + len(".") + digits + len(".nbc") + 21 <= 255 + + a_file_on_disk = os.path.exists(py_file) + + def may_take(cls): + # What a locator takes by the file's name alone, before any location + # is tried: numba's IPython locator a cell, or a file in an ipykernel + # directory, on disk or not; its .zip locator a path with .zip in it, + # which it caches where a part ends in .zip, an archive or a directory, + # and raises for where none does; its user-wide one a frozen + # application's file besides any on disk; the rest any file on disk; + # and a class of none of numba's families, with a from_function and + # a place of its own, what its from_function takes, asked with a + # probe as numba asks it, where it caches being its own to say: the + # remedy took such a class for taking any file, and told a file it + # passed over as its own, where numba went on to the next listed. + if one_of(cls, for_ipython): + return _taken_by_ipython(py_file) + if one_of(cls, for_a_zip): + return ".zip" in py_file + if one_of(cls, user_wide): + return a_file_on_disk or bool(getattr(sys, "frozen", False)) + if of_numbas(cls): + return a_file_on_disk + return _takes(cls, py_file) + + def names_a_path_above(): + # Whether the error names a path above the file, the archive a + # member's code names or a directory on the way, both read absolute: + # a relative file against a name given absolute matched nothing, and + # a member's moved archive was told as a location refusing a file. + # The working directory is none: a path object built from an empty + # string names it, which the remedy tells as a location that is none + # of numba's, and it stood above a file under it as a moved archive. + above = os.path.abspath(filename) if filename else "" + return bool(above) and above != os.getcwd() and os.path.abspath(py_file).startswith(above + os.sep) + + def names_another_location(): + # Whether the error names a location of one of numba's locators for a + # file on disk other than IPython's, or a file in it: NUMBA_CACHE_DIR's, + # the __pycache__ beside the source, or the user's cache directory; + # one that is none of them, a temporary directory IPython gave, is + # IPython's locator's. + if not filename: + return False + named = os.path.abspath(filename) + if named == os.path.abspath(py_file): + # The source itself, which numba reads: refused, it is told to be + # made readable, whatever IPython gives for its directory. + return True + others = [os.path.abspath(config.CACHE_DIR)] if config.CACHE_DIR else [] + others.append(os.path.abspath(os.path.join(os.path.dirname(py_file), "__pycache__"))) + others.append(os.path.abspath(AppDirs(appname="numba", appauthor=False).user_cache_dir)) + return any(named == other or named.startswith(other + os.sep) for other in others) + + asked = [] + + def ipython_numba_cache(): + # IPython's cache directory, asked for once: under a home it cannot + # write IPython makes a temporary directory per call and warns. + if not asked: + asked.append(_ipython_numba_cache()) + return asked[0] + + def names_a_numba_cache(): + # Whether the error names IPython's location for numba, a numba_cache + # directory, or a file in one, as not there: the error numba's first + # save raises under an IPython giving a temporary directory per call + # is FileNotFoundError, naming the numba_cache it could not save in, + # a directory made in another, and so is the check's; a caller's + # naming another location shows nothing of that locator, and was told + # IPYTHONDIR for a cell on disk the in-tree locator took, and for a + # member under such an IPython on any errno, a refused directory of + # its own among them. + return ( + bool(filename) and failure.errno == errno.ENOENT + and "numba_cache" in os.path.abspath(filename).split(os.sep)) + + if isinstance(failure, OSError) and names_a_numba_cache() and not names_a_path_above() and ( + _ipython_locator_reads(py_file)) and ( + not a_file_on_disk or not names_another_location() + ) and _ipython_cache_is_temporary(ipython_numba_cache()): + # numba's IPython locator, reached first for a member in an ipykernel + # directory and for a cell file on disk there once the locators before + # it passed the file over, asks IPython for its cache directory at + # every save, and IPython, able to write neither that directory nor + # its own, gives a new temporary directory on every call: numba makes + # numba_cache in one and saves in another, dying there. The remedy + # told the one the error names, a temporary directory, as refusing a + # file, and for a file on disk offered NUMBA_CACHE_DIR beside it; an + # error naming no file for a file on disk shows nothing of which + # locator took it, and is told the locations numba could have taken. + # inspect's error, with no errno and no name, came before the locator + # tried any location, and is told as inspect's below, whatever + # IPython gives: the remedy blamed the directory for a cell whose + # source linecache does not hold, and for a member shipped as .pyc. + # An error naming no file is told the locations numba could have + # taken, with its reason, as the docstring says, whatever IPython + # gives: the one numba's first save raises here names the directory + # it could not save in, and a caller's with a reason and no name, a + # full disk's, was told IPYTHONDIR with the reason dropped. An error + # naming a path above the file, the archive a member's code names + # where that has since moved, is the stamp's, and is told below; the + # temporary directory's error names a numba_cache under it. + # IPython reads XDG_CACHE_HOME for its cache directory on every POSIX + # system from IPython 8, macOS included, which IPython 7 left out; + # the remedy said Linux, and steered a macOS reader off the variable. + return ( + "numba caches this file in numba_cache under IPython's cache directory, and IPython, able to write neither " + "that directory nor its own, gives a new temporary directory on every call, so numba makes numba_cache " + "in one and saves in another: set IPYTHONDIR, or XDG_CACHE_HOME on a POSIX system, to a writable " + "directory; or " + f"{silence}" + ) + + if a_file_on_disk: + def reads_the_variable(cls): + return one_of(cls, user_provided) + + def caches_beside_the_source(cls): + return one_of(cls, in_tree) + + def caches_under_the_user_cache_dir(cls): + return one_of(cls, user_wide) or one_of(cls, for_a_zip) + + order = [cls for cls in _locators(listed) if may_take(cls)] + # numba's .zip locator takes a path with .zip in it without trying + # its location, and a class of none of numba's families in the order + # took the file, asked, so none listed after either is reached: the + # remedy read the locators after such a class as reached, and told an + # error naming IPython's location as numba caching the file there. + zip_at = next((at for at, cls in enumerate(order) if one_of(cls, for_a_zip)), None) + stops_at = next((at for at, cls in enumerate(order) if one_of(cls, for_a_zip) or not of_numbas(cls)), None) + reached = order if stops_at is None else order[:stops_at + 1] + # The .zip locator listed after such a class is not reached, and the + # errors it raises are a caller's under the list: the remedy told them + # as the .zip locator's, the locators before it having passed the + # file over, where one of them took it. + zip_at = zip_at if zip_at == stops_at else None + variable_at = next((at for at, cls in enumerate(order) if reads_the_variable(cls)), None) + cache_dir_read = variable_at is not None + + def instead(taken, asks="a short path", shared=None, made="a writable directory at a short path"): + # NUMBA_CACHE_DIR as the alternative to the location numba took, + # offered where numba tries the variable before that location's + # locator: set, it was passed over, unwritable or too deep, and + # ``made`` is what it is asked to be made; unset, + # it would be taken, and ``asks`` is what it is asked to be. None + # is a location that is none of numba's, taken for the first in + # the order other than the variable's own; it shows nothing of + # what numba did with the variable, so a set one is named without + # the claim that numba passed it over. ``shared`` is the last + # locator numba reaches whose place the error names too, where + # the user-wide and .zip locators give the file one path under + # the user's cache directory: which of them took it the error + # cannot show, so the variable is offered where numba tries it + # before the last, without the claim; the remedy read the first + # as the one, and left the variable out where it sat between. + if subpath is None: + # numba names its directory under the variable by hashing + # the source's directory as UTF-8, and raises for one that is + # not: the remedy offered the variable, and the import died + # on it where the warning had stood. + return ( + ", and NUMBA_CACHE_DIR cures nothing, numba hashing the source's directory as UTF-8 for its " + "directory there, which this one is not" + ) + if taken in order: + position = order.index(taken) + passed_over = " and numba could not use" + else: + position = next((at for at, cls in enumerate(order) if not reads_the_variable(cls)), len(order)) + passed_over = "" + if shared is not None and shared is not taken and shared in order: + position = max(position, order.index(shared)) + passed_over = "" + if not cache_dir_read or variable_at >= position: + return "" + if config.CACHE_DIR: + return f", or NUMBA_CACHE_DIR, which is set to {config.CACHE_DIR}{passed_over}, made {made}" + return f", or NUMBA_CACHE_DIR set to {asks}" + + user_cache_dir = AppDirs(appname="numba", appauthor=False).user_cache_dir + + def beside_the_source(): + # The in-tree locator caches beside the source, in its __pycache__, + # under no directory named after the source's: the one of numba's + # locators for a file on disk that a name too long leaves, made + # writable where numba reaches it and it passed the file over, or + # listed where NUMBA_CACHE_LOCATOR_CLASSES leaves it out, or + # before the .zip locator where that one, listed first, took a + # path with .zip in it and left it unreached; the offer to make + # the __pycache__ writable went to a list numba never reached it + # under. + if any(caches_beside_the_source(cls) for cls in reached): + beside = os.path.abspath(os.path.join(os.path.dirname(py_file), "__pycache__")) + return ( + f", or make the __pycache__ beside the source, {beside}, writable, where InTreeCacheLocator " + "caches it under no such name" + ) + if listed: + # Listed after the locator that took the file, the .zip one + # or a class of none of numba's families, the in-tree locator + # goes before it; the remedy named the .zip locator for any. + before = "" + if stops_at is not None and any(caches_beside_the_source(cls) for cls in order): + stopper = order[stops_at] + before = f" before {'ZipCacheLocator' if one_of(stopper, for_a_zip) else stopper.__name__}" + return ( + f", or list InTreeCacheLocator{before}, which caches beside the source, in its __pycache__, under " + "no such name" + ) + return "" + + def location_of(cls): + # Where the class places the file: the .zip locator under the user's + # cache directory like the user-wide one. None for a class that is + # not numba's, whose place is its own, for the user-provided one + # without NUMBA_CACHE_DIR, which takes nothing without it, and for + # the IPython one without IPython, or where IPython raises OSError + # for its directory, which numba passes the locator over on. + if reads_the_variable(cls): + return os.path.abspath(os.path.join(config.CACHE_DIR, subpath)) if config.CACHE_DIR and subpath else None + if caches_beside_the_source(cls): + return os.path.abspath(os.path.join(os.path.dirname(py_file), "__pycache__")) + if caches_under_the_user_cache_dir(cls): + return os.path.abspath(os.path.join(user_cache_dir, subpath)) if subpath else None + if one_of(cls, for_ipython): + ipython_cache = ipython_numba_cache() + return os.path.abspath(ipython_cache) if ipython_cache else None + return None + + locations = {cls: location_of(cls) for cls in reached} + if _is_inspects(failure) and any(one_of(cls, for_ipython) for cls in reached): + # inspect's error, which no call of the file system's raised, so + # with no errno and no name: numba's IPython locator reads the + # function's source, from the file itself for one on disk, and + # inspect raises where it reads none there, a file it cannot read + # or, before Python 3.13, an empty one, before that locator tries + # its location, the + # ones numba tries before it having passed the file over; the + # remedy told the location numba took, which it had not, as + # refusing a file. + return ( + f"numba's IPython locator reads the function's source from its file before trying its location, and " + f"read none in {py_file} ({failure}): make the file readable, with the source in it; or {silence}" + ) + if isinstance(failure, OSError) and filename and os.path.abspath(filename) == os.path.abspath(py_file): + # numba reads the source itself: its IPython locator for the + # function's source, and from 0.68 every locator for the stamp, + # which hashes the file; a file that refuses to be read was told + # as the location numba took refusing a file, to make writable. + reason = failure.strerror or failure return ( - "the path is too long for the file system: a shorter NUMBA_CACHE_DIR, or none, where it is set, " - f"else {package} installed at a shorter path; or {silence}" + f"numba reads the function's source from its file, {py_file}, which cannot be read ({reason}): make " + f"it readable; or {silence}" + ) + # Under a list with no locator that takes the file, or none with a + # place for it, the user-provided one alone with NUMBA_CACHE_DIR unset, + # numba took no location, and the list is the remedy, below, for an + # error a caller passes on as for numba's no-locator one; a class of + # none of numba's families has a place of its own, which an error + # can name. + if isinstance(failure, OSError) and (any(locations.values()) or not all(of_numbas(cls) for cls in reached)): + # The error names the location numba took, or a file numba writes + # in it: numba's own check passed the location, its temporary file + # fitting where its cache files would not, or the package's named + # file was refused since, a full disk or permissions changed. The + # locator that took it decides the remedy, the first in numba's + # order whose place the error names: whole paths, since the places + # can nest, an install under either cache directory or + # NUMBA_CACHE_DIR above the user's; and in order, since + # NUMBA_CACHE_DIR set to the user's cache directory makes one path + # of two. + if filename: + named = os.path.abspath(filename) + named = {named, os.path.dirname(named)} + else: + # A caller can pass on an error that names nothing, and numba's + # first save raises one where a write fails, as on a full disk. + named = set() + taken = next((cls for cls in reached if locations[cls] and locations[cls] in named), None) + shared = next((cls for cls in reversed(reached) if locations[cls] and locations[cls] in named), None) + # A name too long is refused whatever the errno, EINVAL on Windows. + named_after_the_source = taken is not None and ( + reads_the_variable(taken) or caches_under_the_user_cache_dir(taken)) + too_long = too_long_a_name(locations[taken]) if named_after_the_source else None + if too_long: + return f"{too_long}: put the file in a directory of a shorter name{beside_the_source()}; or {silence}" + too_long_file = too_long_a_file_name(locations[taken]) if taken is not None else None + if too_long_file: + # The variable, where numba tries it before the locator that + # took the file and a file system in use takes the name, + # pointed at a directory on one, as it is offered at a short + # path for a path too long; the variable's own location took + # the file only set, so it is named. + clause, a_file_system_takes_it = too_long_file + offer = "" + if a_file_system_takes_it: + fits = "directory on a file system that takes a name that long" + if reads_the_variable(taken): + offer = f", or NUMBA_CACHE_DIR, which is set to {config.CACHE_DIR}, pointed at a {fits}" + else: + offer = instead(taken, asks=f"a {fits}", shared=shared, made=f"a writable {fits}") + return f"{clause}, and no shorter path cures it: {a_shorter_name}{offer}; or {silence}" + too_long_for_the_names = a_name_limit_below(locations[taken]) if taken is not None else None + if too_long_for_the_names: + fits = "directory on a file system that takes a name that long" + if reads_the_variable(taken): + offer = f", or NUMBA_CACHE_DIR, which is set to {config.CACHE_DIR}, pointed at a {fits}" + else: + offer = instead(taken, asks=f"a {fits}", shared=shared, made=f"a writable {fits}") + return f"{too_long_for_the_names}, so no shorter path cures it{offer}; or {silence}" + if failure.errno == errno.ENAMETOOLONG: + if taken is not None and reads_the_variable(taken): + # The variable's location, which numba's own check passed + # and the first save overflows: numba keeps taking the + # variable as it is, so the user's cache directory at a + # shorter path, offered where the variable is set to it, + # moved nothing numba reads. + cure = "a shorter NUMBA_CACHE_DIR" + elif taken is not None and caches_beside_the_source(taken): + cure = f"{package} installed at a shorter path{instead(taken)}" + elif taken is not None and caches_under_the_user_cache_dir(taken): + cure = ( + f"the user's cache directory, {user_cache_dir}, at a shorter path{_moved_through()}" + f"{instead(taken, shared=shared)}" + ) + elif taken is not None and one_of(taken, for_ipython): + cure = f"IPython's cache directory, {os.path.dirname(locations[taken])}, at a shorter path{instead(taken)}" + elif not filename: + cure = f"{could_have_taken(locations)}, at a shorter path{instead(None)}" + else: + cure = f"that location, {filename}, at a shorter path{instead(None)}" + return f"the path is too long for the file system: {cure}; or {silence}" + if (failure.errno == errno.EISDIR and zip_at is not None and filename and os.path.isdir(filename) + and filename.endswith(".zip") and names_a_path_above()): + # numba's .zip locator takes a file on disk for the ".zip" in + # its path once the locators before it passed the file over, + # takes the first part of the path ending in .zip for the + # archive, a directory so named included, and from 0.68 opens + # it to read the source, which a directory refuses; the remedy + # told the reader to make room in that directory, no cache + # location at all, with the remedy for a member alone knowing + # the case; under a list with no .zip locator the error is a + # caller's, and is told as the location it names. + if zip_at: + reached_how = "reached once the locators before it passed this file over" + cure = f"make one of those locators' locations writable{instead(None, 'a writable directory')}, or " + else: + # Listed first, the .zip locator takes the file before any + # locator for a file on disk; the remedy spoke of locators + # before it that were not there. + reached_how = "which NUMBA_CACHE_LOCATOR_CLASSES puts before every locator for a source file on disk" + cure = "list UserProvidedCacheLocator, with NUMBA_CACHE_DIR set, InTreeCacheLocator or " \ + "UserWideCacheLocator before it, or " + return ( + f"numba's .zip locator, {reached_how}, takes the first part of the path ending in .zip, {filename}, " + "for an archive, and it is a directory, which numba 0.68 and later open for the source: " + f"{cure}put the file under a path with no such directory; or {silence}" + ) + # numba took the location, so the variable was not passed over for + # it; set elsewhere it moves the cache off a disk that is full. A + # location that is none of numba's is named as the error names it, + # and an error that names none is told the ones numba could have + # taken. + if taken is not None and reads_the_variable(taken): + alternative = "" if subpath is None else ", or NUMBA_CACHE_DIR set to another writable directory" + else: + alternative = instead(taken, "a writable directory", shared=shared) + if taken: + opening = f"numba caches this file in {locations[taken]}," + elif filename: + opening = f"that location, {filename}," + else: + opening = f"{could_have_taken(locations)}," + # An error a caller builds from a message alone has no strerror. + reason = failure.strerror or failure + return ( + f"{opening} where no file can be written ({reason}): make room there, or make it " + f"writable{alternative}; or {silence}" ) # numba itself passes a location it cannot make or write over, for a - # source on disk, so the error here is the no-locator one. + # source on disk, so the error here is the no-locator one, every + # locator tried and passed over, or the .zip locator's for a path with + # .zip in it and no part ending in it, which numba raises where it + # reaches that locator: the locators before it passed the file over, + # and none after it was tried. + zip_raised = isinstance(failure, ValueError) and zip_at is not None + tried = order[:zip_at] if zip_raised else order + if not tried: + if zip_raised: + return ( + "numba's .zip locator, which NUMBA_CACHE_LOCATOR_CLASSES puts before every locator for a source " + 'file on disk, takes this file for the ".zip" in its path and finds no archive there: list ' + "UserProvidedCacheLocator, with NUMBA_CACHE_DIR set, InTreeCacheLocator or UserWideCacheLocator " + f"before it, or {silence}" + ) + return ( + f"numba looks only where NUMBA_CACHE_LOCATOR_CLASSES, {listed}, says, and none of those locators takes " + "a source file on disk: list UserProvidedCacheLocator, with NUMBA_CACHE_DIR set, InTreeCacheLocator " + f"or UserWideCacheLocator, or {silence}" + ) + named_after_the_source = [ + cls for cls in tried + if cls in locations and (reads_the_variable(cls) or caches_under_the_user_cache_dir(cls))] + if named_after_the_source and all(too_long_a_name(locations[cls]) for cls in named_after_the_source): + # Every locator numba tried, of those it reaches, a class listed + # after the .zip locator being unreached for a path that locator + # takes, that names its directory after the source's found the + # name too long for its file system, the + # variable's by that of the directory it is set to or, unset, by + # the limit nearly every file system has, and passed the file + # over for that, which no writable directory at a short path + # cures; the remedy asked for one, and, where the .zip locator + # then raised for a path with .zip in it and no part ending in + # it, asked for the variable without a word of the name. + too_long = too_long_a_name(locations[named_after_the_source[0]]) + # Under the user-provided locator alone, with the variable unset, + # numba took no location at all and never measured the name: the + # variable is what it takes nothing without, and the name is what + # would stop it once set; the remedy said the name alone. + variable = ( + ", with NUMBA_CACHE_DIR set, which UserProvidedCacheLocator takes nothing without" + if not config.CACHE_DIR and all(reads_the_variable(cls) for cls in tried) else "") + return ( + f"{too_long}: put the file in a directory of a shorter name{variable}{beside_the_source()}; or {silence}" + ) + hashing = [ + cls for cls in tried if (reads_the_variable(cls) and config.CACHE_DIR) or caches_under_the_user_cache_dir(cls)] + if subpath is None and (isinstance(failure, UnicodeEncodeError) or hashing): + # numba hashes the source's directory path as UTF-8 for its + # directory under NUMBA_CACHE_DIR or the user's cache directory, + # and the locator that names it so raises: the variable's, set, + # before the in-tree locator is reached, which caches beside the + # source and hashes no path, or the user-wide one once the in-tree + # locator passed an unwritable __pycache__ over. The remedy said + # the variable cures nothing and asked for a UTF-8 path alone, + # where the variable unset, the same install imports cached; + # under a list without the variable's locator it told the list, + # and its locations to make writable, which hashing refuses + # whatever they are. numba hashed where the error is its + # UnicodeEncodeError, or where a locator that names its + # directory so was tried: the variable's, set, or the user-wide + # one; the in-tree locator alone hashes nothing, and its + # no-locator error is the __pycache__ refused. + hashed = ( + "numba hashes the source's directory as UTF-8 for the directory it caches this file in under " + f"NUMBA_CACHE_DIR or the user's cache directory, and {os.path.dirname(py_file)} is not" + ) + + # The locators that hash: the variable's, set, the user-wide one, + # and the .zip one for a path with .zip in it. The in-tree one + # caches beside the source, in its __pycache__, hashing no path: + # listed after one that hashes, it is never reached, and the + # remedy asked for its __pycache__ to be made writable, which it + # was, or for it to be listed, after the one that hashes, where + # numba still raises; the cure is the in-tree locator before the + # first that hashes, the variable unset where its locator alone + # is that one, or the __pycache__ made writable where the + # in-tree locator came first and passed it over. + def hashes(cls): + return ((reads_the_variable(cls) and bool(config.CACHE_DIR)) or caches_under_the_user_cache_dir(cls) + or (one_of(cls, for_a_zip) and ".zip" in py_file)) + + beside = os.path.abspath(os.path.join(os.path.dirname(py_file), "__pycache__")) + in_tree_at = next((at for at, cls in enumerate(order) if caches_beside_the_source(cls)), None) + before_in_tree = [cls for cls in (order if in_tree_at is None else order[:in_tree_at]) if hashes(cls)] + # Whether the in-tree locator takes the file, asked as numba asks + # it: the __pycache__ it can write; the remedy hedged "where that + # directory can be written" and never asked for it, so the + # variable unset as told left numba raising the same error. + if in_tree is not None and _takes(in_tree, py_file): + caches_beside = f"numba caches beside the source, in its __pycache__, {beside}, hashing no path" + else: + caches_beside = ( + f"make the __pycache__ beside the source, {beside}, writable, and numba caches there, hashing no " + "path") + if in_tree_at is None: + where = f" before {before_in_tree[0].__name__}" if before_in_tree else "" + return ( + f"{hashed}: put the file under a UTF-8 path, or list InTreeCacheLocator{where}, and " + f"{caches_beside}; or {silence}" + ) + if before_in_tree and all(reads_the_variable(cls) for cls in before_in_tree): + return ( + f"{hashed}, and NUMBA_CACHE_DIR, which is set to {config.CACHE_DIR}, is hashed for first: unset " + f"it, and {caches_beside}; or put the file under a UTF-8 path; or {silence}" + ) + if before_in_tree: + return ( + f"{hashed}, and NUMBA_CACHE_LOCATOR_CLASSES puts {before_in_tree[0].__name__}, which hashes it, " + f"before InTreeCacheLocator: list InTreeCacheLocator before it, and {caches_beside}; or put the " + f"file under a UTF-8 path; or {silence}" + ) + return ( + f"{hashed}: put the file under a UTF-8 path, or make the __pycache__ beside the source, {beside}, " + f"writable, where InTreeCacheLocator caches it hashing no path; or {silence}" + ) + if not any(reads_the_variable(cls) for cls in tried): + if zip_raised: + return ( + 'numba\'s .zip locator takes this file for the ".zip" in its path and finds no archive there, after ' + f"every locator NUMBA_CACHE_LOCATOR_CLASSES, {listed}, puts before it passed the file over: make " + "one of those locations writable, or list UserProvidedCacheLocator, with NUMBA_CACHE_DIR set, " + f"InTreeCacheLocator or UserWideCacheLocator before it, or {silence}" + ) + return ( + f"numba looks only where NUMBA_CACHE_LOCATOR_CLASSES, {listed}, says: make one of those locations " + f"writable, or {silence}" + ) + if config.CACHE_DIR: + # Set and passed over: unwritable, or too deep for numba to make + # its directory there, which a writable one as deep would not cure. + return ( + f"NUMBA_CACHE_DIR is set to {config.CACHE_DIR}, which numba could not use: set it to a writable " + f"directory at a short path, or {silence}" + ) return f"Set NUMBA_CACHE_DIR to a writable directory, or {silence}" + + def the_source_files(): + # The source files, on disk or in a .zip, each with the locator it + # needs listed where NUMBA_CACHE_LOCATOR_CLASSES leaves it out. + on_disk = f"install {package} with its source files on disk, unpacked from any archive" + if listed: + # The install caches through a locator for a file on disk the list + # names: none, and it caches no more than the archive does; the + # user-provided one alone, and that takes nothing without + # NUMBA_CACHE_DIR. + for_a_file_on_disk = [ + cls for cls in _locators(listed) + if any(one_of(cls, base) for base in (user_provided, in_tree, user_wide)) + ] + if not for_a_file_on_disk: + on_disk += ( + ", and list UserProvidedCacheLocator, with NUMBA_CACHE_DIR set, InTreeCacheLocator or " + f"UserWideCacheLocator in NUMBA_CACHE_LOCATOR_CLASSES, {listed}, which has none of them" + ) + elif all(one_of(cls, user_provided) for cls in for_a_file_on_disk) and not config.CACHE_DIR: + on_disk += ( + ", with NUMBA_CACHE_DIR set, which UserProvidedCacheLocator, the one locator for a file on disk in " + f"NUMBA_CACHE_LOCATOR_CLASSES, {listed}, takes nothing without" + ) + a_zip = ( + "import it from a .zip holding its source files, which numba 0.61 and later cache in the user's cache " + "directory" + ) + found = _archive_above(py_file) + if found is not None and for_a_zip is not None and not found[0].endswith(".zip"): + # The file is in an archive already, APP.ZIP or an .egg, which + # numba's .zip locator never takes, looking for a part of the path + # ending in ".zip", lower-case: the remedy offered a .zip holding + # the source files to a reader importing from one, and never named + # the extension numba does not read; and it held the rename back + # where a directory above the archive had ".zip" in its name, the + # locator taking the file for that and finding no archive. + # numba takes the first part of the path ending in .zip for the + # archive, so a directory so named above it is in the way, and the + # rename alone left numba opening the directory: the remedy + # offered the rename under one. + archive_parts = pathlib.Path(found[0]).parts + in_the_way = next( + (str(pathlib.Path(*archive_parts[:at + 1])) for at, part in enumerate(archive_parts[:-1]) + if part.endswith(".zip")), None) + moved = ( + f", and put it under a path with no part ending in .zip above it, {in_the_way} being one, which that " + "locator takes for the archive first" if in_the_way else "") + a_zip = ( + f"rename the archive, {found[0]}, to end in .zip, lower-case, which numba's .zip locator looks for in " + f"the path, with its source files in it{moved}, so that numba 0.61 and later cache it in the user's " + "cache directory" + ) + if listed_without(for_a_zip): + # numba's .zip locator alone takes a .zip, so a list without it + # would leave numba no locator for the .zip offered either. + a_zip += f" through ZipCacheLocator, once NUMBA_CACHE_LOCATOR_CLASSES, {listed}, lists it" + return on_disk, a_zip + if isinstance(failure, KeyError): - if "\\" in str(failure) and os.name == "nt": + # numba 0.68 and later read a .zip member's source from the archive + # for the stamp, by the name the module's code gives under the + # archive's path, and zipfile raises for a name the archive does not + # hold: a module shipped as .pyc alone, or, on Windows under 0.68.0, + # any member, the name built with the platform's separator (numba + # issue 10889, fixed in 0.68.1). The remedy told the archive's user + # to add source files that were there; and one that read a backslash + # as the separator's doing alone told a module shipped as .pyc alone + # that numba 0.68.1 reads the archive as it is, which holds no + # source for it under the name that release reads either. + member = _missing_member(failure) + held = member.replace("\\", "/") + on_disk, a_zip = the_source_files() + if os.name == "nt" and "\\" in member and _numbas_archive_holds(py_file, held): + # The install on disk is a cure now; a numba release with the fix, + # none out as of 0.68.0, reads the member as the archive names it, + # and is named by the fix, not as a release to install. return ( - "Numba 0.68's ZIP cache locator uses Windows path separators for ZIP members, " - "see https://github.com/numba/numba/issues/10889" + f"numba 0.68.0 reads a .zip member's source by a name built with Windows's separator, {member}, which " + "no archive holds (numba issue https://github.com/numba/numba/issues/10889, fixed for numba 0.68.1): " + f"to cache, {on_disk}, which no separator touches, or a numba release that carries the fix, which " + f"reads the member as the archive names it; or {silence}" ) - return f"For numba>=0.68, put all source .py file(s) in .zip, or {silence}" - if isinstance(failure, OSError) or getattr(sys, "frozen", False): - # The .zip's error names the location numba picked, which is under - # the user's cache directory; the frozen application's names nothing. - location = getattr(failure, "filename", None) or AppDirs(appname="numba", appauthor=False).user_cache_dir - if ( - isinstance(failure, OSError) - and failure.errno == errno.EISDIR and location and location.endswith(".zip") - and os.path.isdir(location) and py_file.startswith(location + os.sep) - ): + naming = _pyc_naming(py_file) + if naming is not None and naming[1]: + # The archive holds the module's source, beside a .pyc compiled + # to name another path in the archive, which zipimport runs and + # numba reads the source by: the remedy told the module as + # shipped as .pyc alone, and asked for source files that were + # there. + compiled, source = naming[0], naming[0][:-1] return ( - f"the path {location} is a directory whose name ends in .zip, " - "which numba mistakes for the source archive; rename that " - "directory, or use a path without a .zip-named directory, " - f"or {silence}" + f"numba 0.68 and later read a .zip member's source from the archive by the name the module's code " + f"gives, {held}, which {compiled} was compiled to name, and this one holds none so named, while it " + f"holds the module's source, {source}: to cache, compile the .pyc to name its path in the archive, " + f"or leave it out, so that Python compiles the source and numba reads it under that name; or {silence}" ) - if isinstance(failure, FileNotFoundError) and py_file.startswith(failure.filename + os.sep): + return ( + f"numba 0.68 and later read a .zip member's source from the archive, and this one holds none under " + f"{held}, the module shipped as .pyc alone: to cache, {on_disk}, or {a_zip}; or {silence}" + ) + # The locators numba reaches for a file not on disk, in its order: its + # .zip locator takes a file without trying its location, and a class of + # none of numba's families in the order took the file, asked, so none + # listed after either is reached; its IPython and user-wide ones pass a + # location they cannot make or write over, and the next is tried. + order = [cls for cls in _locators(listed) if may_take(cls)] + zip_at = next((at for at, cls in enumerate(order) if one_of(cls, for_a_zip)), None) + stops_at = next((at for at, cls in enumerate(order) if one_of(cls, for_a_zip) or not of_numbas(cls)), None) + reached = order if stops_at is None else order[:stops_at + 1] + # The .zip locator listed after such a class is not reached, and its + # error for a path with no part ending in .zip is a caller's under the list. + zip_at = zip_at if zip_at == stops_at else None + user_cache_dir = AppDirs(appname="numba", appauthor=False).user_cache_dir + in_the_user_cache = os.path.join(user_cache_dir, subpath) if subpath else None + + def location_of(cls): + # Where the class places the file: the IPython locator in + # numba_cache under IPython's cache directory, None without + # IPython, or where IPython raises OSError for the directory, which + # numba passes the locator over on; IPython is asked for the + # directory where a locator of that family is reached and for no + # other file, since asked for any .zip member it warned, with its + # own directory unwritable, on every import of the package, and + # left a temporary directory behind; the .zip and user-wide ones in + # a directory of their own under the user's cache directory; a class + # of none of numba's families in a place of its own, None here. + if one_of(cls, for_ipython): + ipython_cache = ipython_numba_cache() + return os.path.abspath(ipython_cache) if ipython_cache else None + if not of_numbas(cls) or in_the_user_cache is None: + return None + return os.path.abspath(in_the_user_cache) + + locations = {cls: location_of(cls) for cls in reached} + took_no_location = not any(locations.values()) and all(of_numbas(cls) for cls in reached) + if getattr(sys, "frozen", False) and listed_without(user_wide) and ( + took_no_location or not isinstance(failure, (OSError, ValueError))): + # numba's user-wide locator alone of numba's takes a frozen + # application, so a list without it leaves numba no locator of its + # own for one: its no-locator error, whatever classes of the reader's + # own the list holds, those having passed the file over, and any + # error a caller passes on where the list has no place for the file; + # the remedy asked for a writable directory numba never tried. + return ( + "numba caches a frozen application through UserWideCacheLocator alone, and " + f"NUMBA_CACHE_LOCATOR_CLASSES, {listed}, leaves it out: list it, or {silence}" + ) + # Under a list with no locator that takes the file, or none with a place + # for it, a class of none of numba's families having one of its own, + # numba took no location, and the list is the remedy, below, for an error + # a caller passes on as for numba's no-locator one, as for a source on + # disk. + parts = pathlib.Path(py_file).parts + in_a_zip = any(part.endswith(".zip") for part in parts) + # A directory named .zip above the file, which numba's .zip locator takes + # for an archive all the same, and caches nothing under; a cell under one, + # whose source linecache does not hold, was told the archive remedy, the + # source files to ship. + a_directory_named_zip = any( + os.path.isdir(os.path.join(*parts[:at + 1])) for at, part in enumerate(parts) if part.endswith(".zip")) + # A member of any archive, an .egg or .whl as a .zip, which zipimport runs + # and whose source inspect reads through its loader; the .zip is the one + # numba's .zip locator takes, and the only one it caches. + in_an_archive = (in_a_zip and not a_directory_named_zip) or _archive_above(py_file) is not None + # A caller's error with no errno and no name looked like inspect's and + # was told to make the file readable; inspect's are known by their words. + inspects = _is_inspects(failure) and any(one_of(cls, for_ipython) for cls in reached) + if inspects and not in_an_archive: + # inspect's error, with no errno and no name, for a cell whose source + # linecache does not hold: numba's IPython locator reads a cell's + # source from linecache, where IPython holds it while it runs the + # cell, and inspect raises before the locator tries its location; + # the remedy told the location numba took as refusing a file. For a + # member of an archive that holds no source for it, a .pyc alone, + # inspect's error gets the archive remedy below, the source files to + # ship; a member of an .egg got the cell's, the archive unknown by + # its name. + return ( + f"numba's IPython locator reads a cell's source from linecache, which holds none for {py_file} " + f"({failure}), and numba caches a cell only while IPython holds its source; or {silence}" + ) + not_utf8 = _member_not_utf8(py_file) if inspects else None + if not_utf8: + # inspect's error for a member of an archive that holds its source, + # not UTF-8: zipimport decodes a member's source as UTF-8 and raises + # for one that is not, and the probe's loader read it as zipimport + # does; the error came out of the check as its own, no cache error, + # and the import died where numba would die at decoration. + return ( + f"numba's IPython locator reads the member's source as UTF-8, which {py_file} is not ({not_utf8}): ship " + f"it as UTF-8; or {silence}" + ) + hashed_by = next((cls for cls in reached if (one_of(cls, user_wide) and getattr(sys, "frozen", False)) + or (one_of(cls, for_a_zip) and in_a_zip)), None) + if isinstance(failure, UnicodeEncodeError) and hashed_by is not None: + # numba's .zip locator names its directory under the user's cache + # directory after the source's directory, the archive's path in it, + # hashed as UTF-8, and raises for bytes that are not, at decoration + # and in the check alike, as the user-wide locator does for a frozen + # application's file; the remedy, taking the error for a caller's + # with no place for the file, offered the .zip the reader was + # importing from, or the source files on disk, and said nothing of + # the path. The first of the two numba reaches is the one that + # raised: the user-wide locator, before the .zip one in numba's + # order, for a frozen application under a directory named .zip, + # which the remedy told as an archive. + what, where = (("a .zip", "the archive") if one_of(hashed_by, for_a_zip) + else ("a frozen application", "the application")) + return ( + f"numba caches {what} in the user's cache directory, and NUMBA_CACHE_DIR has no effect here, because the " + "source is not a file on disk: numba hashes the source's directory as UTF-8 for the directory it caches " + f"this file in there, and {os.path.dirname(py_file)} is not: put {where} under a UTF-8 path, or " + f"{silence}" + ) + if not took_no_location and not inspects and (isinstance(failure, OSError) or getattr(sys, "frozen", False)): + if isinstance(failure, ValueError) and zip_at is not None: + # numba's .zip locator takes a frozen application's file for the + # ".zip" in its path, without trying its location, and raises + # where no part of the path ends in .zip, an archive or a + # directory: listed before the user-wide locator, which alone + # takes the file, it leaves that one unreached, and the remedy is + # to list it before the .zip one; after it, the user-wide one + # passed its location over, unwritable or too deep, and that + # location is the remedy, as for a source on disk. + if not any(one_of(cls, user_wide) for cls in order[:zip_at]): + return ( + f"numba's .zip locator, listed in NUMBA_CACHE_LOCATOR_CLASSES, {listed}, with no " + 'UserWideCacheLocator before it, takes this file for the ".zip" in its path and finds no archive ' + f"there: list UserWideCacheLocator, the one locator for a frozen application, before it, or {silence}" + ) + return ( + 'numba\'s .zip locator takes this file for the ".zip" in its path and finds no archive there, after ' + "UserWideCacheLocator, which numba tries before it, could not use the user's cache directory, " + f"{user_cache_dir}: make it writable, or put it at a shorter path{_moved_through()}, or {silence}" + ) + if filename: + named = os.path.abspath(filename) + named = {named, os.path.dirname(named)} + else: + named = set() + # The error names the location numba took, or a file numba writes in + # it, as the first save's does, and the locator that took it decides + # the remedy, the first reached whose place the error names; the + # frozen application's no-locator error names nothing. + taken = next((cls for cls in reached if locations[cls] and locations[cls] in named), None) + if taken is not None and one_of(taken, for_ipython): + # numba caches the member in numba_cache under IPython's cache + # directory: the remedy a cell file on disk gets there, with no + # NUMBA_CACHE_DIR to offer for a source not on disk. + ipython_cache = locations[taken] + too_long_file = too_long_a_file_name(ipython_cache) + if too_long_file: + return f"{too_long_file[0]}, and no shorter path cures it: {a_shorter_name}; or {silence}" + too_long_for_the_names = a_name_limit_below(ipython_cache) + if too_long_for_the_names: + return f"{too_long_for_the_names}, so no shorter path cures it; or {silence}" + if failure.errno == errno.ENAMETOOLONG: + return ( + f"the path is too long for the file system: IPython's cache directory, " + f"{os.path.dirname(ipython_cache)}, at a shorter path; or {silence}" + ) + reason = failure.strerror or failure + return ( + f"numba caches this file in {ipython_cache}, where no file can be written ({reason}): make room " + f"there, or make it writable; or {silence}" + ) + # The .zip's location, a directory of numba's own under the user's + # cache directory, is what a reader can act on, named by the error or + # the directory of the file it names; an error that names no file is + # told the locations numba could have taken, IPython's numba_cache + # among them where a locator of that family is reached; the frozen + # application's no-locator error names nothing, and the user's cache + # directory is told. + if taken is not None: + location = filename if os.path.abspath(filename) == locations[taken] else os.path.dirname(filename) + else: + location = filename or user_cache_dir + # Where numba caches the file: a .zip's or a frozen application's + # location is under the user's cache directory; a location that is + # none of numba's is not, and an error that names no file for a + # member a locator of IPython's family, or a class of none of numba's + # families, is reached for may be that one's, with IPython's + # directory given or not, so nothing is said of that directory for + # any of them. + in_the_user_cache_alone = taken is not None or ( + not filename and all(of_numbas(cls) and not one_of(cls, for_ipython) for cls in reached)) + where = ( + "numba caches a .zip, or a frozen application, in the user's cache directory, and " + if in_the_user_cache_alone else "") + opening = f"{where}NUMBA_CACHE_DIR has no effect here, because the source is not a file on disk: " + if isinstance(failure, FileNotFoundError) and names_a_path_above(): # The stamp numba reads at decoration, of the archive the code # names: .pyc members compiled to name an archive since moved. A # location that cannot be made, under a dangling link, is ENOENT # too, and is not under the code's file. suggestion = "" if numba_version >= 68 else "compile the archive's .pyc members to name its path now, or " return ( - f"the module's code names {failure.filename}, which is not there: its .pyc was compiled to name " - f"that path, {suggestion}ship its source files; " + f"the module's code names {filename}, which is not there: its .pyc was compiled to name " + f"that path, so {suggestion}ship its source files; " f"or {silence}" ) + if (isinstance(failure, OSError) and failure.errno == errno.EISDIR and filename and os.path.isdir(filename) + and filename.endswith(".zip") and names_a_path_above()): + # numba takes the first part of the path ending in .zip for the + # archive, a directory so named above it included, and from 0.68 + # opens it to read the member's source, which a directory + # refuses; the remedy told the reader to make room in it. + return ( + f"{opening}numba takes the first part of the path ending in .zip, {filename}, for the archive, and it " + "is a directory, which numba 0.68 and later open to read the member's source: put the archive under " + f"a path with no such directory above it, or {silence}" + ) + if isinstance(failure, OSError) and names_a_path_above(): + # The archive the member's code names, which numba reads for the + # stamp, and from 0.68 opens for the member's source, and cannot: + # the remedy told it as a location refusing a file, to be made + # writable, which reads nothing. + reason = failure.strerror or failure + return ( + f"{opening}numba reads the archive, {filename}, for the member's stamp, and from 0.68 for the " + f"member's source, and cannot ({reason}): make it readable, or {silence}" + ) + if isinstance(failure, OSError) and filename and taken is None: + # A location that is none of numba's, under the list and in its + # order, which an error a caller passes on can name: told as the + # error names it, as for a source on disk, with nothing of what + # moves the user's cache directory, which moves nothing of it, and + # nothing of where numba caches the file, which the location is + # none of: a member numba's IPython locator takes, with IPython + # not importable, numba caches nowhere. + if failure.errno == errno.ENAMETOOLONG: + return ( + f"{opening}the path is too long for the file system, so put that location, {filename}, at a " + f"shorter path, or {silence}" + ) + reason = failure.strerror or failure + return ( + f"{opening}no file can be written at that location, {filename} ({reason}), so make room there, or " + f"make it writable, or {silence}" + ) + too_long = too_long_a_name(locations[taken]) if isinstance(failure, OSError) and taken is not None else None + if too_long: + # The .zip's directory under the user's cache directory, named + # after the source's, which that directory at a shorter path + # cures nothing of, refused whatever the errno, EINVAL on + # Windows; the remedy asked for one. + return f"{opening}{too_long}, so put the file in a directory of a shorter name, or {silence}" + too_long_file = ( + too_long_a_file_name(locations[taken]) if isinstance(failure, OSError) and taken is not None else None) + if too_long_file: + return f"{opening}{too_long_file[0]}, and no shorter path cures it: {a_shorter_name}, or {silence}" + too_long_for_the_names = ( + a_name_limit_below(locations[taken]) if isinstance(failure, OSError) and taken is not None else None) + if too_long_for_the_names: + return f"{opening}{too_long_for_the_names}, so no shorter path cures it, or {silence}" if isinstance(failure, OSError) and failure.errno == errno.ENAMETOOLONG: + if taken is None: + # The error names no file, and which of the locations is too + # long cannot be told; the user's cache directory moves them + # all where no locator of IPython's family is reached. + moved_through = _moved_through() if in_the_user_cache_alone else "" + return ( + f"{opening}the path is too long for the file system, so put {could_have_taken(locations)}, at a " + f"shorter path{moved_through}, or {silence}" + ) return ( - "numba caches a .zip, or a frozen application, in the user's cache directory, and NUMBA_CACHE_DIR " - f"has no effect here, because the source is not a file on disk: the path is too long for the file " - f"system, so put that directory, {location}, at a shorter path, through XDG_CACHE_HOME or HOME, or " - f"{silence}" + f"{opening}the path is too long for the file system, so put that directory, {location}, at a shorter " + f"path{_moved_through()}, or {silence}" ) + if isinstance(failure, OSError): + # The location refused a file for another reason, a full disk or + # permissions, which the error gives, as the remedy for a source on + # disk gives it; the frozen application's no-locator error gives + # none, numba having passed the location over on it. + reason = failure.strerror or failure + if taken is None: + return ( + f"{opening}no file can be written in {could_have_taken(locations)} ({reason}), so make room " + f"there, or make it writable, or {silence}" + ) + # numba's .zip locator takes the file without making its + # location, and the first save makes it, or the check does, under + # the first directory above it that is there, which refused: the + # remedy asked for room in a directory that was not there, or for + # it to be made writable, and never named the one refusing. An + # error naming a file in the location found the location there. + made_under = "" + if os.path.abspath(filename) == locations[taken] and not os.path.isdir(location): + above = _existing_above(location) + if not os.path.isdir(above): + # A file in the way, which the remedy passed over for the + # directory above it, writable, and asked for room in. + return ( + f"{opening}no file can be written in that directory, {location}, which cannot be made, " + f"{above} being a file in its path ({reason}), so put that file elsewhere, or {silence}" + ) + made_under = f", which cannot be made under {above}" + return ( + f"{opening}no file can be written in that directory, {location}{made_under} ({reason}), so make room " + f"there, or make it writable, or {silence}" + ) + # numba's no-locator error for a frozen application gives no reason, + # numba having passed the user-wide location over on its error, an + # unwritable directory or one too deep to make alike, or one whose + # name, after the source's directory, is too long for the file + # system, which is told where it is, no shorter path curing it. + too_long = too_long_a_name(in_the_user_cache) + if too_long: + return f"{opening}{too_long}, so put the file in a directory of a shorter name, or {silence}" + return ( + f"{opening}numba could not use that directory, {location}: make it writable, or put it at a shorter " + f"path{_moved_through()}, or {silence}" + ) + of_ipythons = [cls for cls in reached if one_of(cls, for_ipython)] + if not isinstance(failure, (OSError, ValueError)) and of_ipythons and all( + one_of(cls, for_ipython) or not of_numbas(cls) for cls in reached): + # numba's no-locator error for a file its IPython locator alone of + # numba's takes, a cell or a .zip member in an ipykernel directory + # under a list with no .zip locator after it: the locator passed its + # location over, as numba's user-wide one does a frozen application's, + # unwritable or too deep to make, or IPython raised for its directory + # and gave none; a class of none of numba's families listed with it + # passed the file over too, numba having found no locator, and is + # named, where the remedy, taking it for one of numba's, answered + # with the archive remedy; asked, it is in the order no longer. + ipython_cache = locations[of_ipythons[0]] + passed_over = [cls.__name__ for cls in _locators(listed) if not of_numbas(cls)] + passed_over = f", and {' and '.join(passed_over)}, listed too, passed the file over" if passed_over else "" + if ipython_cache: + return ( + f"numba caches this file in {ipython_cache}, which it could not make or write in{passed_over}: make " + f"it writable, or put IPython's cache directory, {os.path.dirname(ipython_cache)}, at a shorter path; " + f"or {silence}" + ) + return ( + "numba caches this file in numba_cache under IPython's cache directory, which IPython raised for, under a " + f"home it cannot write{passed_over}: make that home writable; or {silence}" + ) + if _taken_by_ipython(py_file) and not in_a_zip and listed_without(for_ipython): + # numba's IPython locator alone takes a cell, or a file in an ipykernel + # directory that is not on disk, so a list without it leaves numba no + # locator for one; the archive remedy offered source files on disk and + # locators that take none of these. + return ( + f"numba caches a notebook cell through IPythonCacheLocator alone, and NUMBA_CACHE_LOCATOR_CLASSES, {listed}, " + f"leaves it out: list it; or {silence}" + ) + if in_a_zip and not a_directory_named_zip and listed_without(for_a_zip): + # numba's .zip locator alone takes a source in a .zip, so a list + # without it leaves numba no locator for one, and the archive remedy + # offered the .zip the reader was importing from; a source that is + # nowhere under a directory named .zip was told to list it too, where + # that locator takes the directory for an archive and caches nothing. return ( - "numba caches a .zip, or a frozen application, in the user's cache directory, and NUMBA_CACHE_DIR " - f"has no effect here, because the source is not a file on disk: make that directory, {location}, " - f"writable, or {silence}" + f"numba caches a .zip through ZipCacheLocator alone, and NUMBA_CACHE_LOCATOR_CLASSES, {listed}, leaves " + f"it out: list it; or {silence}" ) + on_disk, a_zip = the_source_files() return ( - "NUMBA_CACHE_DIR has no effect here, because the source is not a file on disk: to cache, " - f"install {package} with its source files on disk, unpacked from any archive, or import it from a " - f".zip holding its source files, which numba 0.61 and later cache in the user's cache directory; or " - f"{silence}" + f"NUMBA_CACHE_DIR has no effect here, because the source is not a file on disk: to cache, {on_disk}, or " + f"{a_zip}; or {silence}" ) @@ -331,11 +1983,17 @@ def uncached_where_no_cache_can_be_written(options): Every function numbox caches decorates under the one ``jit_options``, so the question is put here, once, for a source file in each directory of the package that holds a module, and answered for the package. Where numba can cache no function of one of those files the options come back with ``cache`` off and one warning - names the remedy: ``NUMBA_CACHE_DIR`` for a source file on disk, or a shorter one, or none, where the - location is too long for the file system; for a ``.zip``, the user's cache directory made writable, since - numba reads ``NUMBA_CACHE_DIR`` only for a source file on disk; for any other archive, + names the remedy: ``NUMBA_CACHE_DIR`` for a source file on disk, and where the location is too long for the + file system what shortens the one it is, as ``cache_remedy`` sets out; for a ``.zip``, the location numba took + under the user's cache directory made writable or given room, with the reason, or that directory at a shorter + path where the path is too long, and the ``.zip`` locator to list where ``NUMBA_CACHE_LOCATOR_CLASSES`` leaves it + out, numba reading ``NUMBA_CACHE_DIR`` only for a source file on disk; for any other archive, or a ``.pyc``-only install, or a ``.pyc`` in a ``.zip``, the source files on disk or a ``.zip`` holding them, - which numba 0.61 and later cache in the user's cache directory. ``NUMBOX_JIT_OPTIONS='{"cache": false}'`` + which numba 0.61 and later cache in the user's cache directory, and so for a ``.zip`` member the archive holds + no source for, a ``.pyc`` alone, from numba 0.68, which reads the member's source from the archive for the + stamp, the member named; on Windows under numba 0.68.0 every member is such, its name built with the + platform's separator (numba issue 10889), and the remedy is the install on disk, or a numba release carrying the + fix. ``NUMBOX_JIT_OPTIONS='{"cache": false}'`` turns caching off and silences the warning. An error that is not the cache's, as ``is_a_cache_error`` draws the line, is raised as it was. diff --git a/numbox/utils/preprocessing.py b/numbox/utils/preprocessing.py index faf671e2..b301ea7f 100644 --- a/numbox/utils/preprocessing.py +++ b/numbox/utils/preprocessing.py @@ -5,15 +5,18 @@ "Cache-anchor mechanism" section in ``docs/numbox.utils.rst`` for the rationale and references. """ +import ast import errno import hashlib import os +import sys import tempfile import time import warnings from pathlib import Path -from numbox.core.configurations import LONGEST_CACHE_FILE_NAME, cache_remedy, check_cache_location, is_a_cache_error +from numbox.core.configurations import (LONGEST_CACHE_FILE_NAME, _OVERLOAD_DIGITS, cache_remedy, check_cache_location, + is_a_cache_error) def _anchor_root(subdir: str = "numbox-structref") -> Path: @@ -26,6 +29,51 @@ def _anchor_root(subdir: str = "numbox-structref") -> Path: _STEM_MAX = 40 + +def _longest_cache_file_name(stem: str, code_txt: str) -> int: + """The longest name numba writes for a function of the code anchored at a file of this stem, in bytes. + + numba names a function's cache files after the file's stem and the function's qualified name, with the angle + brackets of ```` dropped, its line, Python's version and ABI flags, and, for a data file, the overload's + number, and writes each through a temporary name 21 bytes longer, ``.tmp.`` and sixteen hex digits: the longest + of the functions the code jits, decorated themselves, with ``njit`` or a decorator of the package's own that + jits, or defined within a decorated one, as an ``overload``'s constructor is, which numba compiles with the + overload's options and writes cache files for under the overload's name and ``locals``; nested ones and + methods included, with the digits ``_OVERLOAD_DIGITS`` reserves for the data file's number, three, numba + numbering them from one without bound. A plain function at the top or in a + class body, a struct's method under its own name, which calls the jitted thunk, gets no cache file, and + counted, a method named with two hundred characters put the longest over the file system's 255, and the struct + compiled uncached; counted by five decorators' names alone, the builder's derive function, jitted through the + package's own decorator, and the constructor overload's inner function were left out, and numba's save died + past the check. The anchor's check asks numba's location for a name this long, as the package's asks for its + own with ``LONGEST_CACHE_FILE_NAME``, so that a cache directory within that of the path limit answers here, + uncached after one warning, where numba's own check passed a name of eight and the first save raised through + the struct's making; a bound for any struct, 251, turned caching off on a file system whose name limit is + under it, eCryptfs's 143, for a struct whose names were 58. + """ + names = [] + + def walk(node, outer, jitted): + for child in ast.iter_child_nodes(node): + if isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef)): + qualname = f"{outer}{child.name}" + counts = jitted or bool(child.decorator_list) + if counts: + names.append((qualname, child.lineno)) + walk(child, f"{qualname}..", counts) + elif isinstance(child, ast.ClassDef): + walk(child, f"{outer}{child.name}.", jitted) + else: + walk(child, outer, jitted) + + walk(ast.parse(code_txt), "", False) + version = f"py{sys.version_info[0]}{sys.version_info[1]}{getattr(sys, 'abiflags', '')}" + return max( + (len(f"{stem}.{qualname.replace('<', '').replace('>', '')}-{lineno}.{version}.{'9' * _OVERLOAD_DIGITS}.nbc.tmp." + .encode("utf-8")) + 16 + for qualname, lineno in names), default=0) + + # The options generated code compiles under are the package's unless the caller # gave its own, as make_structref, compile_kernel and the builder take, and # compile_kernel's cache argument overrides them all; the variable reaches only @@ -93,7 +141,10 @@ def _anchor_or_error(path: Path, code_txt: str): the directory makes it so or the name numbox made from a struct's or a function's; numba itself passes a location over on any ``OSError``. Then numba is asked, the question the package puts for its own modules, and no locator, or a location it cannot use, is the cache's; the answer may be the user's cache directory - where the anchor's own cannot be written. + where the anchor's own cannot be written. So is numba's ``UnicodeEncodeError`` for the anchor's directory path, + which numba hashes as UTF-8 to name its cache directory and raises for where it is not. The question is put + with the longest name numba writes for the code's functions, read off the code, so that a location within that + of the path limit, which numba's own check passes with a name of eight, answers here. """ try: path.parent.mkdir(parents=True, exist_ok=True) @@ -101,7 +152,7 @@ def _anchor_or_error(path: Path, code_txt: str): except OSError as error: return error try: - check_cache_location(path) + check_cache_location(path, _longest_cache_file_name(path.stem, code_txt)) except (RuntimeError, OSError, ValueError) as error: if not is_a_cache_error(error): raise @@ -128,11 +179,30 @@ def _anchored_or_uncached(path: Path, code_txt: str, jit_options: dict) -> dict: error = _anchor_or_error(path, code_txt) if error is None: return jit_options - if isinstance(error, OSError) and error.errno == errno.ENAMETOOLONG: + if isinstance(error, OSError) and getattr(error, "longest_file_name", 0): + # The file system takes no name as long as numba's longest for the + # anchor's functions, which the check asked for: no shorter path cures + # it, and the warning offered one. + from numbox.core.configurations import _name_limit + remedy = ( + f"numba's cache files for the functions anchored here have names of up to {error.longest_file_name} " + f"bytes, longer than a name can be on that file system ({_name_limit(error.filename)}): NUMBA_CACHE_DIR " + f"at a directory on a file system that takes a name that long, or {_SILENCE}" + ) + elif isinstance(error, OSError) and error.errno == errno.ENAMETOOLONG: # The names numbox makes are bounded, so the long part is the # directory, NUMBA_CACHE_DIR or the user's cache directory; a short # NUMBA_CACHE_DIR moves the anchor out of either. remedy = f"The path is too long for the file system: NUMBA_CACHE_DIR at a shorter path, or {_SILENCE}" + elif isinstance(error, UnicodeEncodeError): + # numba hashes the anchor's directory path as UTF-8 to name its cache + # directory under NUMBA_CACHE_DIR or the user's cache directory, and + # raises for bytes that are not; the warning offered a writable + # directory, which the anchor's is. + remedy = ( + f"numba hashes the anchor's directory path as UTF-8 to name its cache directory, and {path.parent} is " + f"not: NUMBA_CACHE_DIR at a UTF-8 path, or {_SILENCE}" + ) else: remedy = f"Set NUMBA_CACHE_DIR to a writable directory, or {_SILENCE}" warnings.warn( diff --git a/test/core/test_no_cache_location.py b/test/core/test_no_cache_location.py index 86601a1f..1a2e5fa3 100644 --- a/test/core/test_no_cache_location.py +++ b/test/core/test_no_cache_location.py @@ -19,9 +19,12 @@ import subprocess import sys import zipfile +from importlib.metadata import version from pathlib import Path +from numbox.core.configurations import LONGEST_CACHE_FILE_NAME, numba_version + REPO = Path(__file__).resolve().parent.parent.parent # chmod takes write access from a directory neither on Windows nor from root. @@ -43,16 +46,24 @@ ) +ARCHIVE_REMEDY = ( + "NUMBA_CACHE_DIR has no effect here, because the source is not a file on disk: to cache, install numbox with its " + "source files on disk, unpacked from any archive, or import it from a .zip holding its source files, which numba " + "0.61 and later cache in the user's cache directory") + + def _archive(path, bytecode_naming_the_archive=()): """numbox's modules zipped into ``path``, which goes on PYTHONPATH as it is. - A directory named in ``bytecode_naming_the_archive`` goes in as ``.pyc`` alone, each compiled to name its - path inside the archive, as ``compileall -d`` and ``py_compile``'s ``dfile`` do. + A directory, or a file, named in ``bytecode_naming_the_archive`` goes in as ``.pyc`` alone, each compiled to + name its path inside the archive, as ``compileall -d`` and ``py_compile``'s ``dfile`` do. """ with zipfile.ZipFile(path, "w") as zipped: for source in sorted((REPO / "numbox").rglob("*.py")): member = str(source.relative_to(REPO)) - if source.parent.relative_to(REPO).as_posix() in bytecode_naming_the_archive: + as_bytecode = (source.parent.relative_to(REPO).as_posix() in bytecode_naming_the_archive + or source.relative_to(REPO).as_posix() in bytecode_naming_the_archive) + if as_bytecode: compiled = py_compile.compile(str(source), cfile=str(path.parent / (source.name + "c")), dfile=os.path.join(str(path), member), doraise=True) zipped.write(compiled, member + "c") @@ -130,7 +141,24 @@ def test_options_without_a_cache_key_are_asked_for_the_sites_that_cache_under_th assert run.stderr.count("compiles without a cache") == 1, run.stderr -windows_path_sep_bug = "Numba 0.68's ZIP cache locator uses Windows path separators for ZIP members" +windows_path_sep_bug = "numba 0.68.0 reads a .zip member's source by a name built with Windows's separator" + + +# numba 0.68.0 on Windows reads a .zip member's source by a name built with +# Windows's separator and raises KeyError before any location is tried, numba +# issue 10889, fixed for 0.68.1: a check of a member there meets the bug, not +# the case under test, where the fork's matrix runs Windows on 0.67.0 and +# upstream's on the latest numba. +on_windows_under_numba_0_68_0 = pytest.mark.skipif( + os.name == "nt" and version("numba") == "0.68.0", reason=windows_path_sep_bug) + + +def _skip_for_windows_separator_bug(stderr): + """Skip a test whose import met numba issue 10889, which numba 0.68.0 has on Windows alone: the remedy's text for + it in ``stderr`` on any other platform is a regression, which the skip hid wherever it keyed on the text alone.""" + if windows_path_sep_bug in stderr: + assert os.name == "nt", f"the Windows separator remedy on {os.name}: {stderr}" + pytest.skip(windows_path_sep_bug) def test_a_zip_import_is_cached_by_numba_from_0_61(tmp_path): @@ -145,8 +173,7 @@ def test_a_zip_import_is_cached_by_numba_from_0_61(tmp_path): NUMBA_CACHE_DIR=str(tmp_path / "cache")) env.pop("NUMBOX_JIT_OPTIONS", None) run = _run(env, tmp_path) - if windows_path_sep_bug in run.stderr: - pytest.skip(windows_path_sep_bug) + _skip_for_windows_separator_bug(run.stderr) assert run.returncode == 0 and str(archive) in run.stdout, run.stderr cached = tuple(int(part) for part in numba.__version__.split(".")[:2]) >= (0, 61) assert ("compiles without a cache" not in run.stderr) == cached, run.stderr @@ -302,8 +329,7 @@ def test_a_pyc_in_a_zip_asks_by_the_file_it_was_compiled_from(tmp_path, tree): env.pop("NUMBOX_JIT_OPTIONS", None) run = subprocess.run([sys.executable, "-W", "always", "-c", IMPORT_LIBM], capture_output=True, text=True, env=env, cwd=str(tmp_path)) - if windows_path_sep_bug in run.stderr: - pytest.skip(windows_path_sep_bug) + _skip_for_windows_separator_bug(run.stderr) assert run.returncode == 0 and str(archive) in run.stdout, run.stderr assert run.stderr.count("compiles without a cache") == 1, run.stderr assert "source is not a file on disk" in run.stderr and "holding its source files" in run.stderr, run.stderr @@ -338,8 +364,7 @@ def test_a_stale_pyc_beside_its_source_in_a_zip_is_passed_over_as_zipimport_pass env.pop("NUMBOX_JIT_OPTIONS", None) run = subprocess.run([sys.executable, "-W", "always", "-c", IMPORT_LIBM], capture_output=True, text=True, env=env, cwd=str(tmp_path)) - if windows_path_sep_bug in run.stderr: - pytest.skip(windows_path_sep_bug) + _skip_for_windows_separator_bug(run.stderr) assert run.returncode == 0 and run.stdout.strip().endswith("libm.py"), run.stderr if _zip_is_cached(): assert "compiles without a cache" not in run.stderr, run.stderr @@ -366,8 +391,7 @@ def test_a_stale_pyc_whose_source_in_the_zip_does_not_compile_is_passed_over(tmp NUMBA_CACHE_DIR=str(tmp_path / "cache")) env.pop("NUMBOX_JIT_OPTIONS", None) run = _run(env, tmp_path) - if windows_path_sep_bug in run.stderr: - pytest.skip(windows_path_sep_bug) + _skip_for_windows_separator_bug(run.stderr) assert run.returncode == 0 and str(archive) in run.stdout, run.stderr if _zip_is_cached(): assert "compiles without a cache" not in run.stderr, run.stderr @@ -393,8 +417,7 @@ def test_a_stray_pyc_in_a_zip_that_nothing_imports_is_passed_over(tmp_path, dama NUMBA_CACHE_DIR=str(tmp_path / "cache")) env.pop("NUMBOX_JIT_OPTIONS", None) run = _run(env, tmp_path) - if windows_path_sep_bug in run.stderr: - pytest.skip(windows_path_sep_bug) + _skip_for_windows_separator_bug(run.stderr) assert run.returncode == 0 and str(archive) in run.stdout, run.stderr if _zip_is_cached(): assert "compiles without a cache" not in run.stderr, run.stderr @@ -420,7 +443,8 @@ def test_a_zip_import_with_no_writable_user_cache_directory_compiles_uncached_wi assert run.stderr.count("compiles without a cache") == 1, run.stderr assert "NUMBA_CACHE_DIR has no effect here" in run.stderr if _zip_is_cached(): - assert "make that directory, " in run.stderr and "import it from a .zip" not in run.stderr + assert "so make room there, or make it writable" in run.stderr, run.stderr + assert "import it from a .zip" not in run.stderr finally: home.chmod(0o755) @@ -474,7 +498,7 @@ def test_a_zip_import_whose_user_cache_directory_is_too_long_is_told_so(tmp_path assert run.stderr.count("compiles without a cache") == 1, run.stderr if _zip_is_cached(): assert "too long for the file system, so put that directory" in run.stderr, run.stderr - assert "make that directory, " not in run.stderr, run.stderr + assert "make room there" not in run.stderr, run.stderr def test_a_moved_zip_whose_pyc_members_name_its_old_path_compiles_uncached_and_is_told_why(tmp_path): @@ -493,8 +517,7 @@ def test_a_moved_zip_whose_pyc_members_name_its_old_path_compiles_uncached_and_i env.pop("NUMBOX_JIT_OPTIONS", None) run = subprocess.run([sys.executable, "-W", "always", "-c", IMPORT_LIBM], capture_output=True, text=True, env=env, cwd=str(tmp_path)) - if windows_path_sep_bug in run.stderr: - pytest.skip(windows_path_sep_bug) + _skip_for_windows_separator_bug(run.stderr) assert run.returncode == 0 and str(moved) in run.stdout, run.stderr assert run.stderr.count("compiles without a cache") == 1, run.stderr if _zip_is_cached(): @@ -518,7 +541,8 @@ def test_a_zip_import_whose_user_cache_directory_hangs_off_a_dangling_link_is_to assert run.returncode == 0 and str(archive) in run.stdout, run.stderr assert run.stderr.count("compiles without a cache") == 1, run.stderr if _zip_is_cached(): - assert "make that directory, " in run.stderr and "its .pyc was compiled" not in run.stderr, run.stderr + assert "so make room there, or make it writable" in run.stderr, run.stderr + assert "its .pyc was compiled" not in run.stderr, run.stderr @needs_a_directory_it_cannot_write @@ -562,7 +586,9 @@ def test_a_zip_import_whose_location_for_one_directory_stopped_being_writable_ta # listing that took a directory so named above it listed nothing. A # directory of .pyc members compiled to name the archive, as compileall -d # does, stands in the listing like one of .py members: a listing that let - # only .py members stand never asked for it. + # only .py members stand never asked for it. From numba 0.68 the archive + # is opened for each member's source, and the directory so named above + # it, or a directory of .pyc members, takes the fallback instead, told so. (tmp_path / parent).mkdir(exist_ok=True) archive = _archive(tmp_path / parent / "numbox.zip", bytecode) home = tmp_path / "home" @@ -572,10 +598,20 @@ def test_a_zip_import_whose_location_for_one_directory_stopped_being_writable_ta env.pop("NUMBOX_JIT_OPTIONS", None) warm = _run(env, tmp_path) assert warm.returncode == 0, warm.stderr - if "use a path without a .zip-named directory" in warm.stderr: - pytest.skip("numba's caching doesn't distinguish directory with .zip in its name from an archive from 0.68 on") - if "put all source .py file(s) in .zip" in warm.stderr: - pytest.skip("numba's caching demands .py sources in a .zip from 0.68 on") + if numba_version >= 68 and parent: + assert warm.stderr.count("compiles without a cache") == 1, warm.stderr + assert (f"numba takes the first part of the path ending in .zip, {tmp_path / parent}, for the archive, and it " + "is a directory, which numba 0.68 and later open to read the member's source: put the archive under " + "a path with no such directory above it, or NUMBOX_JIT_OPTIONS") in warm.stderr, warm.stderr + return + if numba_version >= 68 and bytecode: + assert warm.stderr.count("compiles without a cache") == 1, warm.stderr + assert ("numba 0.68 and later read a .zip member's source from the archive, and this one holds none under " + "numbox/core/bindings/__init__.py, the module shipped as .pyc alone: to cache, install numbox with " + "its source files on disk, unpacked from any archive, or import it from a .zip holding its source " + "files, which numba 0.61 and later cache in the user's cache directory; or NUMBOX_JIT_OPTIONS" + ) in warm.stderr, warm.stderr + return if not _zip_is_cached(): pytest.skip("numba caches a .zip from 0.61 on") # Under XDG_CACHE_HOME on Linux, under Library/Caches on macOS. @@ -588,11 +624,73 @@ def test_a_zip_import_whose_location_for_one_directory_stopped_being_writable_ta assert run.stderr.count("compiles without a cache") == 1, run.stderr # The directory to make writable is the location that lost it, not the # user's cache directory above it, which the warning named. - assert f"make that directory, {locations[0]}, writable" in run.stderr, run.stderr + assert (f"no file can be written in that directory, {locations[0]} (Permission denied), so make room there, " + "or make it writable") in run.stderr, run.stderr finally: locations[0].chmod(0o755) +def test_a_pyc_member_alone_beside_sourced_ones_in_a_zip_is_asked_for_itself(tmp_path): + # numba 0.68 and later read each member's source from the archive by the + # name its code gives, so a module shipped as .pyc alone fails where the + # .py members beside it pass: a listing that let one member stand for the + # directory never asked for it, and libm died at its first binding. + archive = _archive(tmp_path / "numbox.zip", ("numbox/core/bindings/libm.py",)) + home = tmp_path / "home" + home.mkdir() + env = dict(os.environ, PYTHONPATH=str(archive), HOME=str(home), XDG_CACHE_HOME=str(home / "cache")) + env.pop("NUMBA_CACHE_DIR", None) + env.pop("NUMBOX_JIT_OPTIONS", None) + run = subprocess.run([sys.executable, "-W", "always", "-c", IMPORT_LIBM], capture_output=True, text=True, + env=env, cwd=str(tmp_path)) + _skip_for_windows_separator_bug(run.stderr) + assert run.returncode == 0 and str(archive) in run.stdout, run.stderr + if numba_version >= 68: + assert run.stderr.count("compiles without a cache") == 1, run.stderr + assert ("holds none under numbox/core/bindings/libm.py, the module shipped as .pyc alone: to cache, install " + "numbox with its source files on disk") in run.stderr, run.stderr + else: + assert ("compiles without a cache" not in run.stderr) == _zip_is_cached(), run.stderr + + +def test_a_pyc_beside_its_source_in_a_zip_compiled_to_name_another_path_in_it_is_asked_for_itself(tmp_path): + # zipimport runs a .pyc before the .py beside it, a hash-based one it + # does not check whatever the .py's age, and numba reads the member's + # source by the name the .pyc's code gives, from numba 0.68 the source + # too; compiled to name another path in the archive, compileall's -d + # with a path the archive does not hold, that name is of no member. The + # listing took a .pyc naming the archive for one standing for its + # directory, like its .py, and never asked: libm died at decoration. + archive = tmp_path / "numbox.zip" + libm = REPO / "numbox" / "core" / "bindings" / "libm.py" + compiled = py_compile.compile(str(libm), cfile=str(tmp_path / "libm.pyc"), + dfile=os.path.join(str(archive), "src", "numbox", "core", "bindings", "libm.py"), + doraise=True, invalidation_mode=py_compile.PycInvalidationMode.UNCHECKED_HASH) + with zipfile.ZipFile(archive, "w") as zipped: + for source in sorted((REPO / "numbox").rglob("*.py")): + zipped.write(source, str(source.relative_to(REPO))) + zipped.write(compiled, "numbox/core/bindings/libm.pyc") + home = tmp_path / "home" + home.mkdir() + env = dict(os.environ, PYTHONPATH=str(archive), HOME=str(home), XDG_CACHE_HOME=str(home / "cache")) + env.pop("NUMBA_CACHE_DIR", None) + env.pop("NUMBOX_JIT_OPTIONS", None) + run = subprocess.run([sys.executable, "-W", "always", "-c", IMPORT_LIBM], capture_output=True, text=True, + env=env, cwd=str(tmp_path)) + _skip_for_windows_separator_bug(run.stderr) + assert run.returncode == 0 and str(archive) in run.stdout, run.stderr + if numba_version >= 68: + assert run.stderr.count("compiles without a cache") == 1, run.stderr + assert ("holds none under src/numbox/core/bindings/libm.py" not in run.stderr + or "shipped as .pyc alone" not in run.stderr), run.stderr + assert ("by the name the module's code gives, src/numbox/core/bindings/libm.py, which " + "numbox/core/bindings/libm.pyc was compiled to name, and this one holds none so named, while it holds " + "the module's source, numbox/core/bindings/libm.py: to cache, compile the .pyc to name its path in the " + "archive, or leave it out") in run.stderr, run.stderr + else: + assert ("compiles without a cache" not in run.stderr) == _zip_is_cached(), run.stderr + + @pytest.mark.skipif(os.name == "nt", reason="a symlink needs a privilege on Windows") def test_symlinks_that_cycle_inside_the_package_are_walked_once(tmp_path): # Two links pointing up the package, followed with no memory of where the @@ -667,9 +765,10 @@ def test_a_frozen_application_is_told_its_user_cache_directory(tmp_path): assert run.returncode == 0 and str(site) in run.stdout, run.stderr assert run.stderr.count("compiles without a cache") == 1, run.stderr assert "frozen application" in run.stderr, run.stderr - # The frozen error is the no-locator one, which names no directory, - # so the warning names numba's: the user cache directory under the home. - named = re.search(r"make that directory, (.+?), writable", run.stderr) + # The frozen error is the no-locator one, which names no directory + # and gives no reason, so the warning names numba's, the user cache + # directory under the home, as one numba could not use. + named = re.search(r"numba could not use that directory, (.+?): make it writable", run.stderr) assert named and Path(named.group(1)).is_relative_to(home), run.stderr assert "install numbox with its source files" not in run.stderr finally: @@ -724,197 +823,4776 @@ def test_a_location_too_close_to_the_path_limit_for_numbas_files_compiles_uncach run = _run(env, tmp_path) assert run.returncode == 0 and str(REPO / "numbox" / "core") in run.stdout, run.stderr assert run.stderr.count("compiles without a cache") == 1, run.stderr - assert "the path is too long for the file system: a shorter NUMBA_CACHE_DIR, or none" in run.stderr, run.stderr + assert "the path is too long for the file system: a shorter NUMBA_CACHE_DIR; or NUMBOX_JIT_OPTIONS" in ( + run.stderr), run.stderr assert not _index_files(cache_dir) -@pytest.mark.parametrize("placement", ["too long", "archive"]) -def test_the_remedies_that_name_a_package_name_the_one_given(tmp_path, placement): - # A package built on numbox puts the question for its own files and takes - # the remedy from cache_remedy; the two remedies that tell the reader to - # install a package again named numbox whichever package had asked. +@needs_a_directory_it_cannot_write +@pytest.mark.skipif(sys.platform != "linux", reason="a path of 4096 bytes is Linux's limit") +def test_a_read_only_install_whose_user_cache_directory_is_too_long_is_told_to_shorten_that(tmp_path): + # Where the source's __pycache__ cannot be written and NUMBA_CACHE_DIR is + # not set, numba caches under the user's cache directory, and a location + # there too long for numba's files was told "a shorter NUMBA_CACHE_DIR, or + # none, where it is set, else numbox installed at a shorter path": a + # variable that was not set and a reinstall that changes nothing. The first + # module asked has its location there end 20 bytes short of the limit. + from numba.core.caching import _CacheLocator + site = tmp_path / "site" + shutil.copytree(REPO / "numbox", site / "numbox", ignore=shutil.ignore_patterns("__pycache__")) + location = _CacheLocator.get_suitable_cache_subpath(str(site / "numbox" / "core" / "configurations.py")) + xdg = _directory_of_length(tmp_path / "xdg", 4096 - 20 - 1 - len(location) - len("/numba")) + read_only = [path for path in site.rglob("*") if path.is_dir()] + for path in read_only: + path.chmod(0o555) + try: + env = dict(os.environ, PYTHONPATH=str(site), XDG_CACHE_HOME=str(xdg)) + env.pop("NUMBA_CACHE_DIR", None) + env.pop("NUMBOX_JIT_OPTIONS", None) + run = _run(env, tmp_path) + assert run.returncode == 0 and str(site) in run.stdout, run.stderr + assert run.stderr.count("compiles without a cache") == 1, run.stderr + assert "at a shorter path, through XDG_CACHE_HOME or HOME" in run.stderr, run.stderr + assert "a shorter NUMBA_CACHE_DIR" not in run.stderr and "numbox installed" not in run.stderr, run.stderr + cured = _run(dict(env, XDG_CACHE_HOME=str(tmp_path / "short")), tmp_path, warnings="error") + assert cured.returncode == 0, cured.stderr + assert _index_files(tmp_path / "short") + finally: + for path in read_only: + path.chmod(0o755) + + +# Each case: where NUMBA_CACHE_DIR is, where the source is, and which of numba's +# locations for it is too long; "user" stands for the user's cache directory. +LOCATION_CASES = { + "NUMBA_CACHE_DIR": ("cache", "site", "NUMBA_CACHE_DIR"), + "beside the source": (None, "site", "beside"), + "the user's cache directory": (None, "site", "user"), + "beside the source, NUMBA_CACHE_DIR passed over": ("cache", "site", "beside"), + "beside a source under NUMBA_CACHE_DIR": ("cache", "cache/site", "beside"), + "the user's cache directory under NUMBA_CACHE_DIR": ("user-cache", "site", "user"), + "beside a source under the user's cache directory": (None, "user-cache/numba/site", "beside"), +} + + +@pytest.mark.parametrize("named", ["the location", "a file in it"]) +@pytest.mark.parametrize("case", list(LOCATION_CASES)) +def test_a_location_too_long_is_told_what_shortens_the_one_it_is(tmp_path, monkeypatch, case, named): + # numba takes a directory under NUMBA_CACHE_DIR where that is set, else the + # __pycache__ beside the source, else a directory under the user's cache + # directory, and the error names the one that is too long. One remedy for + # all three named a variable that was not set and a reinstall that changes + # nothing where the location was under the user's cache directory, and + # telling them apart by which directory a location starts with took one for + # another where they nest: an install under either cache directory, or + # NUMBA_CACHE_DIR above the user's. The user's cache directory is one under + # tmp_path, so that every case runs on every platform. numbox's own check + # names the location; numba's first save names a file in it, and a caller + # can pass that error on. + import numba + import numbox.core.configurations as configurations + from numba.core.caching import _CacheLocator + + class UserCacheUnderTmp: + def __init__(self, appname, appauthor): + self.user_cache_dir = str(tmp_path / "user-cache" / "numba") + + monkeypatch.setattr(configurations, "AppDirs", UserCacheUnderTmp) + cache_dir, source_dir, which = LOCATION_CASES[case] + monkeypatch.setattr(numba.config, "CACHE_DIR", str(tmp_path / cache_dir) if cache_dir else "") + py_file = tmp_path / source_dir / "package" / "module.py" + py_file.parent.mkdir(parents=True) + py_file.write_text("") + subpath = _CacheLocator.get_suitable_cache_subpath(str(py_file)) + user_cache_dir = tmp_path / "user-cache" / "numba" + # A location other than NUMBA_CACHE_DIR's where the variable is set means + # numba passed it over, so the warning names it rather than telling the + # reader to set what is set. + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + if cache_dir: + instead = (f", or NUMBA_CACHE_DIR, which is set to {tmp_path / cache_dir} and numba could not use, made a " + "writable directory at a short path") + else: + instead = ", or NUMBA_CACHE_DIR set to a short path" + where, cure = { + "NUMBA_CACHE_DIR": (tmp_path / "cache" / subpath, "a shorter NUMBA_CACHE_DIR"), + "beside": (py_file.parent / "__pycache__", f"numbduck installed at a shorter path{instead}"), + "user": (user_cache_dir / subpath, + f"the user's cache directory, {user_cache_dir}, at a shorter path{configurations._moved_through()}" + f"{instead}"), + }[which] + if named == "a file in it": + where = where / "module.f-1.py312.nbi" + failure = OSError(errno.ENAMETOOLONG, "File name too long", str(where)) + remedy = configurations.cache_remedy(str(py_file), failure, "silence", package="numbduck") + assert remedy == f"the path is too long for the file system: {cure}; or silence", remedy + + +def test_a_relative_numba_cache_dir_is_matched_as_the_location_numba_names(tmp_path, monkeypatch): + # numba joins NUMBA_CACHE_DIR as set to the name it gives the source's + # directory, so set relative, to the directory the process runs from, the + # location is relative, and so is the path numba's error, and the + # package's own, names. The remedy writes the locations absolute, and the + # path as named matched none of them, so it was told as a location that + # is none of numba's, with the variable to be made a writable directory + # at a short path, where a shorter NUMBA_CACHE_DIR shortens it. + import numba + import numbox.core.configurations as configurations + from numba.core.caching import _CacheLocator + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(numba.config, "CACHE_DIR", "cache") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + py_file = tmp_path / "site" / "package" / "module.py" + py_file.parent.mkdir(parents=True) + py_file.write_text("") + location = os.path.join("cache", _CacheLocator.get_suitable_cache_subpath(str(py_file))) + failure = OSError(errno.ENAMETOOLONG, "File name too long", location) + remedy = configurations.cache_remedy(str(py_file), failure, "silence") + assert remedy == "the path is too long for the file system: a shorter NUMBA_CACHE_DIR; or silence", remedy + + +@pytest.mark.parametrize("locators, cure", [ + ("", "a shorter NUMBA_CACHE_DIR"), + ("UserWideCacheLocator", "the user's cache directory, {user_cache_dir}, at a shorter path{moved_through}"), + ("UserWideCacheLocator,UserProvidedCacheLocator", + "the user's cache directory, {user_cache_dir}, at a shorter path{moved_through}"), + ("InTreeCacheLocator,UserProvidedCacheLocator,UserWideCacheLocator", "a shorter NUMBA_CACHE_DIR"), +]) +def test_a_location_too_long_that_is_numba_cache_dirs_and_the_users_cache_directorys_is_told_by_the_locator_order( + tmp_path, monkeypatch, locators, cure): + # NUMBA_CACHE_DIR set to the user's cache directory makes one path of two + # of numba's locations, and which locator took it is the order's to say: + # numba's own order tries the variable first, so a shorter NUMBA_CACHE_DIR + # shortens it, and nothing else does: numba's own check passed the + # location, so numba keeps taking the variable, and the user's cache + # directory at a shorter path, once offered beside it, moved nothing numba + # reads; a list that tries the user's cache directory before the + # variable, or never reads the variable, took it as the user's cache + # directory, and the variable's remedy, matched first whatever the order, + # told the reader to shorten a variable numba did not read. + import numba + import numbox.core.configurations as configurations + from numba.core.caching import _CacheLocator + + class UserCacheUnderTmp: + def __init__(self, appname, appauthor): + self.user_cache_dir = str(tmp_path / "user-cache" / "numba") + + monkeypatch.setattr(configurations, "AppDirs", UserCacheUnderTmp) + user_cache_dir = tmp_path / "user-cache" / "numba" + monkeypatch.setattr(numba.config, "CACHE_DIR", str(user_cache_dir)) + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", locators, raising=False) + py_file = tmp_path / "site" / "package" / "module.py" + py_file.parent.mkdir(parents=True) + py_file.write_text("") + location = user_cache_dir / _CacheLocator.get_suitable_cache_subpath(str(py_file)) + failure = OSError(errno.ENAMETOOLONG, "File name too long", str(location)) + remedy = configurations.cache_remedy(str(py_file), failure, "silence") + expected = cure.format(user_cache_dir=user_cache_dir, moved_through=configurations._moved_through()) + assert remedy == f"the path is too long for the file system: {expected}; or silence", remedy + + +@pytest.mark.parametrize("cache_dir_set", [False, True]) +@pytest.mark.parametrize("locators, offered", [ + ("", True), + ("IPythonCacheLocator,UserProvidedCacheLocator,InTreeCacheLocator,UserWideCacheLocator", True), + ("InTreeCacheLocator,UserProvidedCacheLocator", False), +]) +def test_a_location_too_long_that_is_none_of_numbas_is_named_with_the_variable_numba_tries_first( + tmp_path, monkeypatch, locators, offered, cache_dir_set): + # A caller can pass on an error naming a location that is none of the three + # numba builds from the file. The remedy names it and offers NUMBA_CACHE_DIR + # where numba tries the variable before any other location, the IPython + # locator ahead taking no file on disk; it read "NUMBA_CACHE_DIR set to a + # short path, which numba takes first", the alternative's clause cut from + # its "or" and nothing said of the location. Set, the variable is named + # without the claim that numba passed it over: a location that is none of + # numba's shows nothing of what numba did with the variable, where the + # remedy said it was one "numba could not use", as it says of a set + # variable passed over for a location numba took. With another locator + # ahead of the variable's, the variable is not offered, and no case here + # held that: the offer with the position dropped passed every one. + import numba from numbox.core.configurations import cache_remedy - if placement == "too long": - py_file = tmp_path / "ducklib.py" - py_file.write_text("") - failure = OSError(errno.ENAMETOOLONG, "File name too long", str(tmp_path / "cache")) - named = "else {} installed at a shorter path" + cache_dir = tmp_path / "cache" + monkeypatch.setattr(numba.config, "CACHE_DIR", str(cache_dir) if cache_dir_set else "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", locators, raising=False) + py_file = tmp_path / "package" / "module.py" + py_file.parent.mkdir() + py_file.write_text("") + elsewhere = tmp_path / "elsewhere" / "cache" + failure = OSError(errno.ENAMETOOLONG, "File name too long", str(elsewhere)) + remedy = cache_remedy(str(py_file), failure, "silence") + if not offered: + instead = "" + elif cache_dir_set: + instead = f", or NUMBA_CACHE_DIR, which is set to {cache_dir}, made a writable directory at a short path" else: - py_file = tmp_path / "numbduck-0.0.0-py3-none-any.whl" / "numbduck" / "ducklib.py" - failure = RuntimeError(f"cannot cache function 'f': no locator available for file '{py_file}'") - named = "install {} with its source files on disk" - remedy = cache_remedy(str(py_file), failure, "silence", package="numbduck") - assert named.format("numbduck") in remedy and "numbox" not in remedy, remedy - assert named.format("numbox") in cache_remedy(str(py_file), failure, "silence") + instead = ", or NUMBA_CACHE_DIR set to a short path" + assert remedy == ( + f"the path is too long for the file system: that location, {elsewhere}, at a shorter path{instead}; or silence" + ), remedy -def _function_names(tree): - """Each function's qualified name as numba names its cache files, angle brackets dropped, with its line.""" - def walk(node, scope): - for child in ast.iter_child_nodes(node): - if isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef)): - yield ".".join(scope + [child.name]), child.lineno - yield from walk(child, scope + [child.name, "locals"]) - elif isinstance(child, ast.ClassDef): - yield from walk(child, scope + [child.name]) - else: - yield from walk(child, scope) - return walk(tree, []) +@pytest.mark.parametrize("filename", [None, ""]) +@pytest.mark.parametrize("cache_dir", [None, "cache", "user-cache/numba"]) +def test_a_location_too_long_that_the_error_does_not_name_is_told_the_locations_numba_could_have_taken( + tmp_path, monkeypatch, cache_dir, filename): + # numba's errors and the package's own check name the file refused, but a + # caller can pass on an ENAMETOOLONG that names nothing. The remedy then + # cannot tell which location numba took, and it lists the locations numba + # could have taken for the file, in numba's order, for the reader to put + # the one that is too long at a shorter path; it read "that location, + # None, at a shorter path", and matched the empty name against the working + # directory and its parent, which could have taken one location for + # another. NUMBA_CACHE_DIR set to the user's cache directory makes one + # path of two locations, and the list named it twice. An empty name is no + # name either, and it read "that location, , at a shorter path". + import numba + import numbox.core.configurations as configurations + from numba.core.caching import _CacheLocator + class UserCacheUnderTmp: + def __init__(self, appname, appauthor): + self.user_cache_dir = str(tmp_path / "user-cache" / "numba") -def test_numbas_longest_file_name_for_the_package_is_within_the_probes_bound(): - # The probe reserves LONGEST_CACHE_FILE_NAME bytes in a location for the - # files numba writes there: the module's stem, the function's qualified - # name, a line number, the interpreter tag with its abiflags and an index - # number, written under a 21-byte temporary name. Every function of the - # package must fit, decorated or not, and so must the builder's generated - # kernel, anchored to its file under a name of 64 hex characters. - from numbox.core.configurations import LONGEST_CACHE_FILE_NAME - tag = f".py{sys.version_info[0]}{sys.version_info[1]}t.99.nbc.tmp.0123456789abcdef" - names = [f"{path.stem}.{qualname}-{lineno}{tag}" - for path in sorted((REPO / "numbox").rglob("*.py")) - for qualname, lineno in _function_names(ast.parse(path.read_text(encoding="utf-8")))] - names.append(f"builder._make_{'0' * 64}-9999{tag}") - longest = max(names, key=len) - assert len(names) > 500 - assert len(longest) <= LONGEST_CACHE_FILE_NAME, longest + monkeypatch.setattr(configurations, "AppDirs", UserCacheUnderTmp) + cache_dir = tmp_path / cache_dir if cache_dir else None + monkeypatch.setattr(numba.config, "CACHE_DIR", str(cache_dir) if cache_dir else "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + py_file = tmp_path / "site" / "package" / "module.py" + py_file.parent.mkdir(parents=True) + py_file.write_text("") + # The working directory is the source's __pycache__, so an empty name + # matched against it would take the in-tree location for the one named. + (py_file.parent / "__pycache__").mkdir() + monkeypatch.chdir(py_file.parent / "__pycache__") + subpath = _CacheLocator.get_suitable_cache_subpath(str(py_file)) + locations = [str(py_file.parent / "__pycache__"), str(tmp_path / "user-cache" / "numba" / subpath)] + if cache_dir: + locations.insert(0, str(cache_dir / subpath)) + locations = list(dict.fromkeys(locations)) + instead = f", or NUMBA_CACHE_DIR, which is set to {cache_dir}, made a writable directory at a short path" + else: + instead = ", or NUMBA_CACHE_DIR set to a short path" + failure = OSError(errno.ENAMETOOLONG, "File name too long", filename) + remedy = configurations.cache_remedy(str(py_file), failure, "silence") + assert remedy == ( + f"the path is too long for the file system: the location numba took, one of {', '.join(locations[:-1])} or " + f"{locations[-1]}, at a shorter path{instead}; or silence"), remedy -ANOTHER_ERROR_AT_THE_CACHE_SET_UP = ( - "import numba.core.caching as caching\n" - "def refuse(self, py_func):\n" - " raise RuntimeError('a locator of another kind refused')\n" - "caching.CompileResultCacheImpl.__init__ = refuse\n" - "import numbox.core.configurations\n" -) +@pytest.mark.skipif(numba_version < 62, reason="NUMBA_CACHE_LOCATOR_CLASSES arrived in numba 0.62") +@pytest.mark.parametrize("failure, told", [ + pytest.param(OSError(errno.ENAMETOOLONG, "File name too long"), + "the path is too long for the file system: the location numba took, {location}, at a shorter path", + id="too long"), + pytest.param(OSError(errno.EACCES, "Permission denied"), + "the location numba took, {location}, where no file can be written (Permission denied): make room " + "there, or make it writable", id="unwritable"), +]) +def test_an_error_that_names_no_file_under_one_locator_is_told_its_one_location(tmp_path, monkeypatch, failure, told): + # A list of one locator leaves numba one location to take for the file, so + # an error that names no file is told that location. The list of the + # locations numba could have taken read "one of" before the one path. + import numba + import numbox.core.configurations as configurations + from numba.core.caching import _CacheLocator + class UserCacheUnderTmp: + def __init__(self, appname, appauthor): + self.user_cache_dir = str(tmp_path / "user-cache" / "numba") -def test_an_error_that_is_not_the_caches_is_raised_as_it_was(tmp_path): - # The fallback answers two errors of numba's cache set-up, no locator and - # a directory that cannot be written. A RuntimeError of another kind at - # the same step, a locator of the user's own refusing, say, is not its to - # turn into an uncached import with a remedy that does not apply. - env = dict(os.environ, NUMBA_CACHE_DIR=str(tmp_path / "cache")) - env.pop("NUMBOX_JIT_OPTIONS", None) - run = subprocess.run([sys.executable, "-W", "always", "-c", ANOTHER_ERROR_AT_THE_CACHE_SET_UP], - capture_output=True, text=True, env=env, cwd=str(tmp_path)) - assert run.returncode != 0 and "a locator of another kind refused" in run.stderr, run.stderr - assert "compiles without a cache" not in run.stderr + monkeypatch.setattr(configurations, "AppDirs", UserCacheUnderTmp) + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "UserWideCacheLocator", raising=False) + py_file = tmp_path / "site" / "package" / "module.py" + py_file.parent.mkdir(parents=True) + py_file.write_text("") + location = tmp_path / "user-cache" / "numba" / _CacheLocator.get_suitable_cache_subpath(str(py_file)) + remedy = configurations.cache_remedy(str(py_file), failure, "silence") + assert remedy == f"{told.format(location=location)}; or silence", remedy -# The code numbox generates at run time, each child importing the helpers of -# the test module that exercises it. -MAKE_A_STRUCTREF = ( - "from numba.core.types import StructRef, float32\n" - "from numba.experimental.structref import register\n" - "from numbox.utils.highlevel import make_structref\n" - "@register\n" - "class TypeClass(StructRef):\n" - " pass\n" - "Struct = make_structref('Struct', {'value': float32}, TypeClass)\n" - "assert Struct(2.5).value == 2.5\n" - "print('made')\n" -) +# Each case: where NUMBA_CACHE_DIR is, NUMBA_CACHE_LOCATOR_CLASSES, which of +# numba's locations the error names, and the alternative offered after it. +UNWRITABLE_CASES = { + "NUMBA_CACHE_DIR's": ("cache", "", "NUMBA_CACHE_DIR", ", or NUMBA_CACHE_DIR set to another writable directory"), + "beside the source, NUMBA_CACHE_DIR unset": (None, "", "beside", ", or NUMBA_CACHE_DIR set to a writable directory"), + "beside the source, NUMBA_CACHE_DIR passed over": ( + "cache", "", "beside", + ", or NUMBA_CACHE_DIR, which is set to {cache_dir} and numba could not use, made a writable directory at a " + "short path"), + "the user's cache directory, another locator first": (None, "UserWideCacheLocator", "user", ""), + "none of numba's": (None, "", "none", ", or NUMBA_CACHE_DIR set to a writable directory"), + "unnamed, NUMBA_CACHE_DIR unset": (None, "", "unnamed", ", or NUMBA_CACHE_DIR set to a writable directory"), + "unnamed, NUMBA_CACHE_DIR set": ( + "cache", "", "unnamed", + ", or NUMBA_CACHE_DIR, which is set to {cache_dir}, made a writable directory at a short path"), +} -REGISTER_AN_AGGREGATE = ( - "from test.core.test_sqlite_udf_helpers import (_open_memory, _make_table, _read1_int64, sum_state_type,\n" - " sum_init, sum_step, sum_finalize)\n" - "from numbox.core.bindings.sqlite.udf_helpers import register_aggregate\n" - "from numbox.core.bindings.sqlite.conn import sqlite3_close\n" - "db = _open_memory()\n" - "_make_table(db, [1, 2, 3, 4, 5])\n" - "register_aggregate(db, 'my_sum', 1, sum_state_type, sum_init, sum_step, sum_finalize)\n" - "value, _ = _read1_int64(db, 'SELECT __cap(my_sum(v)) FROM t')\n" - "sqlite3_close(db)\n" - "assert value == 15, value\n" - "print('summed')\n" -) -COMPILE_A_KERNEL = ( - "from numbox.core.variable.variable import Graph\n" - "from numbox.core.variable.compile_kernel import compile_kernel\n" - "def f(x):\n" - " return x + 1.0\n" - "graph = Graph({'calc': [{'name': 'y', 'inputs': {'x': 'ext'}, 'formula': f}]}, ['ext'])\n" - "kernel = compile_kernel(graph, 'calc.y')\n" - "assert kernel.execute({'ext': {'x': 1.0}})['calc.y'] == 2.0\n" - "print('executed')\n" -) +@pytest.mark.parametrize("error", [(errno.ENOSPC, "No space left on device"), (errno.EACCES, "Permission denied")]) +@pytest.mark.parametrize("case", list(UNWRITABLE_CASES)) +def test_a_location_numba_took_where_no_file_can_be_written_is_told_the_location_and_the_reason( + tmp_path, monkeypatch, case, error): + # numba takes a location its own check could write a temporary file in, + # and the package's check then writes a named file there, which a full + # disk, a quota or permissions changed since can refuse. The location is + # the one numba took, so "NUMBA_CACHE_DIR is set to X, which numba could + # not use: set it to a writable directory at a short path", the answer for + # a variable numba passed over, was false on both counts; the remedy names + # the location and the reason, and offers the variable as the alternative + # where numba tries it before that location, or set elsewhere where the + # location is the variable's own. A location that is none of numba's for + # the file, which a caller can pass on, is named as the error names it, + # as the too-long remedy names one; "numba caches this file in" claimed it + # for numba. An error naming no file, a caller's too, is told the + # locations numba could have taken, as the too-long remedy tells them; it + # fell past this branch to the no-locator remedy, which dropped the reason + # and said the variable, set and writable, was one numba could not use. + import numba + import numbox.core.configurations as configurations + from numba.core.caching import _CacheLocator -BUILD_A_DERIVE = ( - "from numpy import isclose\n" - "from numbox.core.work.builder import Derived, End, make_graph\n" - "x = End(name='x', init_value=3.14)\n" - "def twice(x):\n" - " return 2 * x\n" - "y = Derived(name='y', init_value=0.0, derive=twice, sources=(x,))\n" - "access = make_graph(y)\n" - "access.y.calculate()\n" - "assert isclose(access.y.data, 6.28), access.y.data\n" - "print('derived')\n" -) + class UserCacheUnderTmp: + def __init__(self, appname, appauthor): + self.user_cache_dir = str(tmp_path / "user-cache" / "numba") -REGISTER_A_TVF = ( - "import numpy as np\n" - "from test.core.test_sqlite_tvf import _open, _select_int, _series, _OUT\n" - "from numbox.core.bindings.sqlite.tvf import register_tvf\n" - "from numbox.core.bindings.sqlite.conn import sqlite3_close\n" - "db = _open()\n" - "handle = register_tvf(db.value, 'series', (np.int64, np.int64), _OUT, _series)\n" - "rc, rows = _select_int(db, 'SELECT n FROM series(2, 5)')\n" - "assert rc == 0 and [row[0] for row in rows] == [2, 3, 4], (rc, rows)\n" - "sqlite3_close(db.value)\n" - "print('selected')\n" -) + monkeypatch.setattr(configurations, "AppDirs", UserCacheUnderTmp) + cache_dir, locators, which, offer = UNWRITABLE_CASES[case] + monkeypatch.setattr(numba.config, "CACHE_DIR", str(tmp_path / cache_dir) if cache_dir else "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", locators, raising=False) + py_file = tmp_path / "site" / "package" / "module.py" + py_file.parent.mkdir(parents=True) + py_file.write_text("") + subpath = _CacheLocator.get_suitable_cache_subpath(str(py_file)) + location = { + "NUMBA_CACHE_DIR": tmp_path / "cache" / subpath, + "beside": py_file.parent / "__pycache__", + "user": tmp_path / "user-cache" / "numba" / subpath, + "none": tmp_path / "elsewhere" / "cache", + "unnamed": None, + }[which] + number, reason = error + if which == "unnamed": + failure = OSError(number, reason) + could_have_taken = [py_file.parent / "__pycache__", tmp_path / "user-cache" / "numba" / subpath] + if cache_dir: + could_have_taken.insert(0, tmp_path / "cache" / subpath) + opening = f"the location numba took, one of {', '.join(map(str, could_have_taken[:-1]))} or {could_have_taken[-1]}," + else: + failure = OSError(number, reason, str(location)) + opening = f"that location, {location}," if which == "none" else f"numba caches this file in {location}," + remedy = configurations.cache_remedy(str(py_file), failure, "silence") + assert remedy == ( + f"{opening} where no file can be written ({reason}): make room there, or make it " + f"writable{offer.format(cache_dir=tmp_path / 'cache')}; or silence"), remedy -@needs_a_directory_it_cannot_write -@pytest.mark.parametrize("child, word", [ - (MAKE_A_STRUCTREF, "made"), (REGISTER_AN_AGGREGATE, "summed"), (REGISTER_A_TVF, "selected"), -], ids=["make_structref", "register_aggregate", "register_tvf"]) -def test_generated_code_compiles_uncached_where_its_anchor_cannot_be_written(tmp_path, child, word): - # The code numbox generates at run time is anchored to a file under - # NUMBA_CACHE_DIR or the user's cache directory, and the anchor was written - # whatever the cache option said. So with the tree writable, where the - # package's own functions cache, and the user's cache directory not, - # make_structref and the sqlite registrations died at the anchor's - # directory. The anchor is written to be cached from, and where it cannot - # be, the code compiles without a cache after a warning; NUMBA_CACHE_DIR at - # a writable directory cures it and holds the anchor. - home = tmp_path / "home" - home.mkdir() - home.chmod(0o555) - try: - env = dict(os.environ, PYTHONPATH=str(REPO), HOME=str(home), XDG_CACHE_HOME=str(home / "cache")) - env.pop("NUMBA_CACHE_DIR", None) - env.pop("NUMBOX_JIT_OPTIONS", None) - run = subprocess.run([sys.executable, "-W", "always", "-c", child], capture_output=True, text=True, - env=env, cwd=str(tmp_path)) - assert run.returncode == 0 and word in run.stdout, run.stderr - assert "compiles without a cache" in run.stderr and "Set NUMBA_CACHE_DIR" in run.stderr, run.stderr - cured = subprocess.run([sys.executable, "-W", "error::RuntimeWarning", "-c", child], capture_output=True, - text=True, env=dict(env, NUMBA_CACHE_DIR=str(tmp_path / "cache")), cwd=str(tmp_path)) - assert cured.returncode == 0 and word in cured.stdout, cured.stderr - assert list((tmp_path / "cache").rglob("*.py")), "no anchor under NUMBA_CACHE_DIR" - finally: - home.chmod(0o755) +@pytest.mark.parametrize("placement", [ + "a source on disk", "a .zip member", "a .zip member in an ipykernel directory, naming IPython's location", + "a .zip member, naming a location that is none of numba's", +]) +def test_an_error_with_no_strerror_is_told_its_own_text_as_the_reason(tmp_path, monkeypatch, placement): + # is_a_cache_error admits any OSError, and a caller can pass on one built + # from a message alone, with no errno, no strerror and no filename, or one + # naming a file with no strerror. The reason read "(None)" and the message + # was lost, for a source on disk; the remedies for a source not on disk + # give the reason the same way, and nothing held them to it. + import types + import numba + import numbox.core.configurations as configurations + from numba.core.caching import _CacheLocator + class UserCacheUnderTmp: + def __init__(self, appname, appauthor): + self.user_cache_dir = str(tmp_path / "user-cache" / "numba") -@needs_a_directory_it_cannot_write -@pytest.mark.parametrize("child, word, warns", [ - (MAKE_A_STRUCTREF, "made", True), (REGISTER_AN_AGGREGATE, "summed", True), (REGISTER_A_TVF, "selected", True), - (COMPILE_A_KERNEL, "executed", True), (BUILD_A_DERIVE, "derived", False), -], ids=["make_structref", "register_aggregate", "register_tvf", "compile_kernel", "derive"]) -def test_a_warm_anchor_in_a_directory_that_stopped_being_writable(tmp_path, child, word, warns): - # The anchor is on disk from an earlier run, in a NUMBA_CACHE_DIR that can - # no longer be written. numba then caches the code in the user's cache - # directory, and a check on the anchor's own directory would have turned - # caching off where numba had a location. With the user's cache directory - # unwritable too there is none, and a check that only wrote the anchor - # would have let the first decorated function die at numba's set-up. The - # builder's derive falls back without a word, as it did for an anchor it - # could not write. - home = tmp_path / "home" + monkeypatch.setattr(configurations, "AppDirs", UserCacheUnderTmp) + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + ipython_dir = tmp_path / "ipython" + paths = types.ModuleType("IPython.paths") + paths.get_ipython_cache_dir = lambda: str(ipython_dir) + monkeypatch.setitem(sys.modules, "IPython", types.ModuleType("IPython")) + monkeypatch.setitem(sys.modules, "IPython.paths", paths) + failure = OSError("Disk quota exceeded on /home") + user_cache_dir = tmp_path / "user-cache" / "numba" + not_on_disk = ( + "numba caches a .zip, or a frozen application, in the user's cache directory, and NUMBA_CACHE_DIR has no " + "effect here, because the source is not a file on disk: ") + if placement == "a source on disk": + py_file = tmp_path / "site" / "package" / "module.py" + py_file.parent.mkdir(parents=True) + py_file.write_text("") + subpath = _CacheLocator.get_suitable_cache_subpath(str(py_file)) + expected = ( + f"the location numba took, one of {py_file.parent / '__pycache__'} or {user_cache_dir / subpath}, where no " + f"file can be written ({failure}): make room there, or make it writable, or NUMBA_CACHE_DIR set to a " + "writable directory; or silence") + elif placement == "a .zip member": + py_file = tmp_path / "bundle.zip" / "package" / "module.py" + subpath = _CacheLocator.get_suitable_cache_subpath(str(py_file)) + expected = not_on_disk + ( + f"no file can be written in the location numba took, {user_cache_dir / subpath} ({failure}), so make " + "room there, or make it writable, or silence") + elif placement.startswith("a .zip member in an ipykernel directory"): + py_file = tmp_path / "bundle.zip" / "ipykernel_123" / "cell.py" + failure = OSError(errno.ENOSPC, None, str(ipython_dir / "numba_cache")) + expected = ( + f"numba caches this file in {ipython_dir / 'numba_cache'}, where no file can be written ({failure}): make " + "room there, or make it writable; or silence") + else: + py_file = tmp_path / "bundle.zip" / "package" / "module.py" + failure = OSError(errno.ENOSPC, None, str(tmp_path / "elsewhere" / "cache")) + expected = ( + "NUMBA_CACHE_DIR has no effect here, because the source is not a file on disk: no file can be written at " + f"that location, {tmp_path / 'elsewhere' / 'cache'} ({failure}), so make room " + "there, or make it writable, or silence") + if numba_version < 61 and placement in ("a .zip member", "a .zip member, naming a location that is none of numba's"): + # numba's .zip locator arrived in 0.61; before it, no locator takes a + # .zip member in no ipykernel directory, and the error gets the remedy + # numba's no-locator one does. + expected = f"{ARCHIVE_REMEDY}; or silence" + assert failure.strerror is None and "None" not in expected.replace(str(failure), "") + remedy = configurations.cache_remedy(str(py_file), failure, "silence") + assert remedy == expected, remedy + + +@pytest.mark.parametrize("locators, ipython, expected", [ + pytest.param( + "IPythonCacheLocator,InTreeCacheLocator", "IPython.paths", + "IPython's cache directory, {ipython_dir}, at a shorter path", + marks=pytest.mark.skipif(numba_version < 62, reason="NUMBA_CACHE_LOCATOR_CLASSES arrived in 0.62")), + ("", "IPython.paths", + "IPython's cache directory, {ipython_dir}, at a shorter path, or NUMBA_CACHE_DIR set to a short path"), + ("", "IPython.utils.path", + "IPython's cache directory, {ipython_dir}, at a shorter path, or NUMBA_CACHE_DIR set to a short path"), + ("", None, "that location, {named}, at a shorter path, or NUMBA_CACHE_DIR set to a short path"), +]) +def test_a_cell_file_numbas_ipython_locator_takes_is_told_ipythons_cache_directory( + tmp_path, monkeypatch, locators, ipython, expected): + # numba's IPython locator takes a file on disk in an ipykernel directory, + # a notebook's cell file, and caches it in numba_cache under IPython's + # cache directory, with no directory of its own per file. That location + # too long for the file system got "that location, X", the wording for a + # location that is none of numba's. numba imports IPython for the location + # when it makes it and catches only OSError there, so without IPython it + # raises ImportError at decoration, no cache error; an error a caller + # passes on naming that location is then told it as none of numba's. + import types + import numba + import numbox.core.configurations as configurations + + class UserCacheUnderTmp: + def __init__(self, appname, appauthor): + self.user_cache_dir = str(tmp_path / "user-cache" / "numba") + + monkeypatch.setattr(configurations, "AppDirs", UserCacheUnderTmp) + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", locators, raising=False) + ipython_dir = tmp_path / "ipython" + if ipython: + # The module numba imports the location from: IPython.paths, or + # IPython.utils.path in an older IPython, which numba falls back to + # when IPython.paths does not import. + paths = types.ModuleType(ipython) + paths.get_ipython_cache_dir = lambda: str(ipython_dir) + monkeypatch.setitem(sys.modules, "IPython", types.ModuleType("IPython")) + monkeypatch.setitem(sys.modules, "IPython.paths", paths if ipython == "IPython.paths" else None) + monkeypatch.setitem(sys.modules, "IPython.utils", types.ModuleType("IPython.utils")) + monkeypatch.setitem(sys.modules, "IPython.utils.path", paths if ipython == "IPython.utils.path" else None) + else: + monkeypatch.setitem(sys.modules, "IPython", None) + monkeypatch.setitem(sys.modules, "IPython.paths", None) + monkeypatch.setitem(sys.modules, "IPython.utils.path", None) + py_file = tmp_path / "ipykernel_123" / "cell.py" + py_file.parent.mkdir() + py_file.write_text("") + named = ipython_dir / "numba_cache" / "cell-1.py312.nbi" + failure = OSError(errno.ENAMETOOLONG, "File name too long", str(named)) + remedy = configurations.cache_remedy(str(py_file), failure, "silence") + expected = expected.format(ipython_dir=ipython_dir, named=named) + assert remedy == f"the path is too long for the file system: {expected}; or silence", remedy + + +@pytest.mark.parametrize("parent, name", [("", "bundle.zip"), ("container.zip", "bundle.zip"), ("", "bundle.egg")], + ids=["the archive", "under a directory named .zip", "an .egg"]) +def test_a_zip_member_in_an_ipykernel_directory_is_asked_with_the_probes_source(tmp_path, monkeypatch, parent, name): + # numba's IPython locator takes a file in an ipykernel directory, on disk + # or not, and reads the function's source when it does: from the file, or + # through the module's loader for a member of a .zip. The probe the check + # compiles has no module, so for such a member inspect found no source, + # and the check raised the OSError from reading it, which is no cache + # error and which the remedy, taking an OSError for a source not on disk + # as the .zip's, answered with the user's cache directory made writable, + # a directory numba never tried. The probe's module reads the member's + # source for inspect, as the member's own does, and the question goes on + # to IPython's cache directory, as numba's does. zipimport finds the archive + # below a directory named .zip, and whatever its name, an .egg as Spark + # ships; a check that took the first part named .zip for the archive found + # no member in the directory, or no archive in the .egg, and raised there. + import linecache + import types + import numba + import numbox.core.configurations as configurations + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + ipython_dir = tmp_path / "ipython" + paths = types.ModuleType("IPython.paths") + paths.get_ipython_cache_dir = lambda: str(ipython_dir) + monkeypatch.setitem(sys.modules, "IPython", types.ModuleType("IPython")) + monkeypatch.setitem(sys.modules, "IPython.paths", paths) + (tmp_path / parent).mkdir(exist_ok=True) + archive = tmp_path / parent / name + with zipfile.ZipFile(archive, "w") as zipped: + zipped.writestr("ipykernel_123/cell.py", "def f():\n pass\n") + py_file = str(archive / "ipykernel_123" / "cell.py") + configurations.check_cache_location(py_file, configurations.LONGEST_CACHE_FILE_NAME) + assert (ipython_dir / "numba_cache").is_dir() + # What linecache holds from then on is the member's own, as any reader leaves it. + assert linecache.getlines(py_file) == ["def f():\n", " pass\n"] + linecache.cache.pop(py_file, None) + + +@pytest.mark.parametrize("longest", [LONGEST_CACHE_FILE_NAME, 0], + ids=["with the package's bound on names", "with no bound, as a package may ask"]) +def test_a_zip_member_in_an_ipykernel_directory_under_a_temporary_ipython_directory_takes_the_fallback( + tmp_path, monkeypatch, longest): + # numba's IPython locator asks IPython for its cache directory at every + # save, and IPython, able to write neither that directory nor its own, + # gives a new temporary directory on every call: numba makes numba_cache + # in one and saves in another, dying with FileNotFoundError at the + # function's first call. The check made numba_cache the same way and + # passed the file, where before the probe's source was given it raised + # inspect's error and the package fell back. With a bound on names the + # file the check makes in the location raises first, under the next + # directory IPython gives; without one the location itself is asked for. + import tempfile + import types + import numba + import numbox.core.configurations as configurations + + class UserCacheUnderTmp: + def __init__(self, appname, appauthor): + self.user_cache_dir = str(tmp_path / "user-cache" / "numba") + + monkeypatch.setattr(configurations, "AppDirs", UserCacheUnderTmp) + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + paths = types.ModuleType("IPython.paths") + paths.get_ipython_cache_dir = lambda: tempfile.mkdtemp(dir=tmp_path) + monkeypatch.setitem(sys.modules, "IPython", types.ModuleType("IPython")) + monkeypatch.setitem(sys.modules, "IPython.paths", paths) + archive = tmp_path / "bundle.zip" + with zipfile.ZipFile(archive, "w") as zipped: + zipped.writestr("ipykernel_123/cell.py", "def f():\n pass\n") + py_file = str(archive / "ipykernel_123" / "cell.py") + with pytest.raises(FileNotFoundError) as raised: + configurations.check_cache_location(py_file, longest) + assert raised.value.errno == errno.ENOENT + assert raised.value.filename.startswith(str(tmp_path)) and raised.value.filename.endswith("numba_cache") + assert not os.path.exists(raised.value.filename) + assert configurations.is_a_cache_error(raised.value) + + def left_behind(): + return len([path for path in tmp_path.iterdir() if path.name.startswith("tmp")]) + + # Each call leaves a directory behind: numba asks IPython twice making + # the directory, the check once more for the one numba would save in, + # and the remedy twice, the second to tell a new answer from the first. + # The check asked at every step, and left four. + assert left_behind() == 3 + assert configurations.cache_remedy(py_file, raised.value, "silence") == ( + "numba caches this file in numba_cache under IPython's cache directory, and IPython, able to write neither " + "that directory nor its own, gives a new temporary directory on every call, so numba makes numba_cache in one " + "and saves in another: set IPYTHONDIR, or XDG_CACHE_HOME on a POSIX system, to a writable directory; or silence") + assert left_behind() == 5 + + +@pytest.mark.parametrize("locators", [ + pytest.param("IPythonCacheLocator,InTreeCacheLocator", id="the IPython locator listed first", + marks=pytest.mark.skipif(numba_version < 62, reason="NUMBA_CACHE_LOCATOR_CLASSES arrived in 0.62")), + pytest.param("", id="numba's order, the locators before IPython's passing the file over", + marks=needs_a_directory_it_cannot_write), +]) +def test_a_cell_file_on_disk_under_a_temporary_ipython_directory_is_told_so(tmp_path, monkeypatch, locators): + # A cell file on disk in an ipykernel directory reaches numba's IPython + # locator where it is listed first, or once the in-tree and user-wide + # locators passed the file over, and IPython giving a new temporary + # directory on every call dies the same way a .zip member does; the + # remedy, written for the member alone, told the file on disk to make + # room in the temporary directory, and offered NUMBA_CACHE_DIR. + import tempfile + import types + import numba + import numba.core.caching + import numbox.core.configurations as configurations + + class UserCacheUnderTmp: + def __init__(self, appname, appauthor): + self.user_cache_dir = str(tmp_path / "read-only" / "user-cache" / "numba") + + # numba's own user-wide locator reads the directory through its caching + # module, the remedy through this one. + monkeypatch.setattr(numba.core.caching, "AppDirs", UserCacheUnderTmp) + monkeypatch.setattr(configurations, "AppDirs", UserCacheUnderTmp) + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", locators, raising=False) + paths = types.ModuleType("IPython.paths") + paths.get_ipython_cache_dir = lambda: tempfile.mkdtemp(dir=tmp_path) + monkeypatch.setitem(sys.modules, "IPython", types.ModuleType("IPython")) + monkeypatch.setitem(sys.modules, "IPython.paths", paths) + kernel = tmp_path / "read-only" / "ipykernel_123" + kernel.mkdir(parents=True) + py_file = kernel / "cell.py" + py_file.write_text("def f():\n pass\n") + read_only = [tmp_path / "read-only", kernel] + if not locators: + # The in-tree locator cannot make __pycache__ beside the cell, the + # user-wide one cannot make the user's cache directory. + for path in read_only: + path.chmod(0o555) + try: + with pytest.raises(FileNotFoundError) as raised: + configurations.check_cache_location(str(py_file), configurations.LONGEST_CACHE_FILE_NAME) + assert raised.value.filename.startswith(str(tmp_path)) and raised.value.filename.endswith("numba_cache") + remedy = configurations.cache_remedy(str(py_file), raised.value, "silence") + finally: + for path in read_only: + path.chmod(0o755) + assert remedy == ( + "numba caches this file in numba_cache under IPython's cache directory, and IPython, able to write neither " + "that directory nor its own, gives a new temporary directory on every call, so numba makes numba_cache in one " + "and saves in another: set IPYTHONDIR, or XDG_CACHE_HOME on a POSIX system, " + "to a writable directory; or silence"), remedy + + +@pytest.mark.parametrize("named", ["the directory", "a file numba writes in it"]) +def test_a_cell_file_on_disk_refused_its_pycache_is_told_that_under_a_temporary_ipython_directory_too( + tmp_path, monkeypatch, named): + # The in-tree locator took the cell file and the __pycache__ beside it + # refused a file: that is the remedy, whatever IPython would give for its + # directory, which numba never asked; a remedy that asked IPython first + # told IPYTHONDIR for an error naming the __pycache__, or a file in it, + # as the first save's error does, which nothing told apart from one + # naming the directory itself. + import tempfile + import types + import numba + import numbox.core.configurations as configurations + + class UserCacheUnderTmp: + def __init__(self, appname, appauthor): + self.user_cache_dir = str(tmp_path / "user-cache" / "numba") + + monkeypatch.setattr(configurations, "AppDirs", UserCacheUnderTmp) + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + paths = types.ModuleType("IPython.paths") + paths.get_ipython_cache_dir = lambda: tempfile.mkdtemp(dir=tmp_path) + monkeypatch.setitem(sys.modules, "IPython", types.ModuleType("IPython")) + monkeypatch.setitem(sys.modules, "IPython.paths", paths) + kernel = tmp_path / "ipykernel_123" + kernel.mkdir() + py_file = kernel / "cell.py" + py_file.write_text("def f():\n pass\n") + refused = kernel / "__pycache__" + if named != "the directory": + refused = refused / "cell-1.py312.nbi.tmp.0123456789ab" + failure = OSError(errno.EACCES, "Permission denied", str(refused)) + remedy = configurations.cache_remedy(str(py_file), failure, "silence") + assert remedy == ( + f"numba caches this file in {kernel / '__pycache__'}, where no file can be written (Permission denied): make " + "room there, or make it writable, or NUMBA_CACHE_DIR set to a writable directory; or silence"), remedy + # IPython is asked for its location once, to tell it from the error's, + # and never whether it gives a new one on every call. + assert len([path for path in tmp_path.iterdir() if path.name.startswith("tmp")]) == 1 + + +@pytest.mark.skipif(numba_version < 62, reason="NUMBA_CACHE_LOCATOR_CLASSES arrived in numba 0.62") +def test_a_cell_under_a_locator_list_without_the_ipython_locator_is_told_to_list_it(tmp_path, monkeypatch): + # numba's IPython locator alone takes a cell, so a list without it leaves + # numba no locator for one, and the remedy offered source files on disk, + # locators for a file on disk and a .zip, none of which takes a cell. + import linecache + import types + import numba + import numbox.core.configurations as configurations + + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "ZipCacheLocator", raising=False) + paths = types.ModuleType("IPython.paths") + paths.get_ipython_cache_dir = lambda: str(tmp_path / "ipython") + monkeypatch.setitem(sys.modules, "IPython", types.ModuleType("IPython")) + monkeypatch.setitem(sys.modules, "IPython.paths", paths) + cell = "" + source = "def f():\n pass\n" + monkeypatch.setitem(linecache.cache, cell, (len(source), None, source.splitlines(True), cell)) + with pytest.raises(RuntimeError, match="no locator available") as raised: + configurations.check_cache_location(cell, configurations.LONGEST_CACHE_FILE_NAME) + assert configurations.is_a_cache_error(raised.value) + assert configurations.cache_remedy(cell, raised.value, "silence") == ( + "numba caches a notebook cell through IPythonCacheLocator alone, and NUMBA_CACHE_LOCATOR_CLASSES, " + "ZipCacheLocator, leaves it out: list it; or silence") + + +@needs_a_directory_it_cannot_write +@pytest.mark.parametrize("ipython", ["left alone", "giving a temporary directory per call"]) +def test_a_source_on_disk_numba_cannot_read_is_told_to_make_it_readable(tmp_path, monkeypatch, ipython): + # numba 0.68 and later read a source on disk for the stamp, and its + # IPython locator reads the function's source in every version, so a file + # that cannot be read raises PermissionError naming the source; the + # remedy told it as the location numba took, to make room in or make + # writable, and, under an IPython giving temporary directories, told + # IPYTHONDIR, the error naming no location of another locator's. + import tempfile + import types + import numba + import numbox.core.configurations as configurations + + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + if ipython != "left alone": + paths = types.ModuleType("IPython.paths") + paths.get_ipython_cache_dir = lambda: tempfile.mkdtemp(dir=tmp_path) + monkeypatch.setitem(sys.modules, "IPython", types.ModuleType("IPython")) + monkeypatch.setitem(sys.modules, "IPython.paths", paths) + kernel = tmp_path / "ipykernel_123" + kernel.mkdir() + py_file = kernel / "cell.py" + py_file.write_text("def f():\n pass\n") + py_file.chmod(0) + try: + if numba_version >= 68: + with pytest.raises(PermissionError) as raised: + configurations.check_cache_location(str(py_file), configurations.LONGEST_CACHE_FILE_NAME) + assert raised.value.filename == str(py_file) + failure = raised.value + else: + failure = PermissionError(errno.EACCES, "Permission denied", str(py_file)) + assert configurations.is_a_cache_error(failure) + remedy = configurations.cache_remedy(str(py_file), failure, "silence") + finally: + py_file.chmod(0o644) + assert remedy == ( + f"numba reads the function's source from its file, {py_file}, which cannot be read (Permission denied): make it " + "readable; or silence"), remedy + + +@pytest.mark.skipif(numba_version < 61, reason="numba's .zip locator arrived in 0.61") +@pytest.mark.parametrize("locators", [ + pytest.param("", id="numba's order"), + pytest.param("ZipCacheLocator,InTreeCacheLocator", id="the .zip locator listed first", + marks=pytest.mark.skipif(numba_version < 62, reason="NUMBA_CACHE_LOCATOR_CLASSES arrived in 0.62")), + pytest.param("UserProvidedCacheLocator,InTreeCacheLocator,UserWideCacheLocator", id="no .zip locator listed", + marks=pytest.mark.skipif(numba_version < 62, reason="NUMBA_CACHE_LOCATOR_CLASSES arrived in 0.62")), +]) +def test_a_source_on_disk_under_a_directory_named_zip_is_told_the_directory_once_the_locators_before_passed( + tmp_path, monkeypatch, locators): + # numba's .zip locator takes a file on disk for the ".zip" in its path once + # the in-tree and user-wide locators passed it over, takes the directory so + # named for the archive and, from 0.68, opens it for the source, which a + # directory refuses; the remedy told the reader to make room in that + # directory, writable already and no cache location, where the member's + # remedy knew the case. + import numba + import numba.core.caching + import numbox.core.configurations as configurations + + class UserCacheUnderTmp: + def __init__(self, appname, appauthor): + self.user_cache_dir = str(tmp_path / "read-only" / "user-cache" / "numba") + + monkeypatch.setattr(numba.core.caching, "AppDirs", UserCacheUnderTmp) + monkeypatch.setattr(configurations, "AppDirs", UserCacheUnderTmp) + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", locators, raising=False) + container = tmp_path / "read-only" / "data.zip" + package = container / "pkg" + package.mkdir(parents=True) + py_file = package / "m.py" + py_file.write_text("x = 1\n") + if locators and "ZipCacheLocator" not in locators: + # Under a list with no .zip locator the error naming the directory is + # a caller's, and is told as the location it names; the remedy spoke + # of the list putting the .zip locator first. + expected = ( + f"that location, {container}, where no file can be written (Is a directory): make room there, or make it " + "writable, or NUMBA_CACHE_DIR set to a writable directory; or silence") + elif locators: + # Listed first, the .zip locator takes the file before any locator for + # a file on disk; the remedy spoke of locators before it passing the + # file over, where there were none. + expected = ( + "numba's .zip locator, which NUMBA_CACHE_LOCATOR_CLASSES puts before every locator for a source file on " + f"disk, takes the first part of the path ending in .zip, {container}, for an archive, and it is a " + "directory, which numba 0.68 and later open for the source: list UserProvidedCacheLocator, with " + "NUMBA_CACHE_DIR set, InTreeCacheLocator or UserWideCacheLocator before it, or put the file under a path " + "with no such directory; or silence") + else: + expected = ( + "numba's .zip locator, reached once the locators before it passed this file over, takes the first part of " + f"the path ending in .zip, {container}, for an archive, and it is a directory, which numba 0.68 and later " + "open for the source: make one of those locators' locations writable, or NUMBA_CACHE_DIR set to a " + "writable directory, or put the file under a path with no such directory; or silence") + failure = IsADirectoryError(errno.EISDIR, "Is a directory", str(container)) + assert configurations.cache_remedy(str(py_file), failure, "silence") == expected + if numba_version < 68 or os.name == "nt" or os.geteuid() == 0 or locators: + return + # The in-tree locator cannot make __pycache__ beside the source, the + # user-wide one cannot make the user's cache directory. + read_only = [tmp_path / "read-only", package] + for path in read_only: + path.chmod(0o555) + try: + with pytest.raises(IsADirectoryError) as raised: + configurations.check_cache_location(str(py_file), configurations.LONGEST_CACHE_FILE_NAME) + assert raised.value.filename == str(container) + assert configurations.is_a_cache_error(raised.value) + assert configurations.cache_remedy(str(py_file), raised.value, "silence") == expected + finally: + for path in read_only: + path.chmod(0o755) + + +def test_an_ipython_that_raises_for_its_cache_directory_leaves_the_remedy_a_remedy(tmp_path, monkeypatch): + # The remedy asks IPython for its cache directory as numba's locator does, + # and took OSError and ImportError for no directory; anything else escaped, + # and the import died with IPython's error where the warning was due. + import types + import numba + import numbox.core.configurations as configurations + + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + paths = types.ModuleType("IPython.paths") + + def raises(): + raise RuntimeError("IPython misbehaved") + + paths.get_ipython_cache_dir = raises + monkeypatch.setitem(sys.modules, "IPython", types.ModuleType("IPython")) + monkeypatch.setitem(sys.modules, "IPython.paths", paths) + cell = "" + failure = RuntimeError(f"cannot cache function 'f': no locator available for file '{cell}'") + remedy = configurations.cache_remedy(cell, failure, "silence") + assert remedy == ( + "numba caches this file in numba_cache under IPython's cache directory, which IPython raised for, under a " + "home it cannot write: make that home writable; or silence"), remedy + + +def test_an_error_naming_no_file_for_an_ipykernel_source_on_disk_is_not_blamed_on_ipython(tmp_path, monkeypatch): + # A cell file on disk in an ipykernel directory that the in-tree locator + # took, with an error naming no file, a full disk at the first save, under + # an IPython giving temporary directories: the error shows nothing of + # which locator took the file, and the remedy, finding no other location + # named, blamed IPython's temporary directory and told IPYTHONDIR. + import tempfile + import types + import numba + import numbox.core.configurations as configurations + + class UserCacheUnderTmp: + def __init__(self, appname, appauthor): + self.user_cache_dir = str(tmp_path / "user-cache" / "numba") + + monkeypatch.setattr(configurations, "AppDirs", UserCacheUnderTmp) + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + paths = types.ModuleType("IPython.paths") + paths.get_ipython_cache_dir = lambda: tempfile.mkdtemp(dir=tmp_path) + monkeypatch.setitem(sys.modules, "IPython", types.ModuleType("IPython")) + monkeypatch.setitem(sys.modules, "IPython.paths", paths) + kernel = tmp_path / "ipykernel_123" + kernel.mkdir() + py_file = kernel / "cell.py" + py_file.write_text("def f():\n pass\n") + remedy = configurations.cache_remedy(str(py_file), OSError(errno.ENOSPC, "No space left on device"), "silence") + assert remedy.startswith("the location numba took, one of "), remedy + assert str(kernel / "__pycache__") in remedy and "IPYTHONDIR" not in remedy, remedy + assert remedy.endswith("where no file can be written (No space left on device): make room there, or make it " + "writable, or NUMBA_CACHE_DIR set to a writable directory; or silence"), remedy + + +@pytest.mark.skipif(numba_version < 62, reason="NUMBA_CACHE_LOCATOR_CLASSES arrived in numba 0.62") +def test_a_name_too_long_under_the_zip_locator_listed_before_the_in_tree_one_is_not_offered_the_pycache( + tmp_path, monkeypatch): + # numba's .zip locator, listed first, takes a source on disk with .zip in + # its path and reaches nothing after it, so the in-tree locator listed + # after it never caches beside the source; the remedy for a name too long + # offered the __pycache__ beside the source all the same, reading the + # whole list for the locators numba reaches. + import numba + from numba.core.caching import _CacheLocator + import numbox.core.configurations as configurations + + class UserCacheUnderTmp: + def __init__(self, appname, appauthor): + self.user_cache_dir = str(tmp_path / "user-cache" / "numba") + + monkeypatch.setattr(configurations, "AppDirs", UserCacheUnderTmp) + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "ZipCacheLocator,InTreeCacheLocator", raising=False) + limit = configurations._name_limit(str(tmp_path)) + parent = tmp_path / ("d" * (limit - 34) + ".zip") + parent.mkdir() + py_file = parent / "module.py" + py_file.write_text("def f():\n pass\n") + location = os.path.join(str(tmp_path / "user-cache" / "numba"), _CacheLocator.get_suitable_cache_subpath(str(py_file))) + remedy = configurations.cache_remedy(str(py_file), OSError(errno.ENAMETOOLONG, "File name too long", location), "silence") + assert remedy.startswith("numba names the directory it caches this file in after the source's directory, "), remedy + assert remedy.endswith( + ": put the file in a directory of a shorter name, or list InTreeCacheLocator before ZipCacheLocator, which " + "caches beside the source, in its __pycache__, under no such name; or silence"), remedy + assert "writable" not in remedy + + +@pytest.mark.skipif(numba_version < 62, reason="NUMBA_CACHE_LOCATOR_CLASSES arrived in numba 0.62") +def test_a_path_too_long_the_user_wide_and_zip_locators_share_is_offered_the_variable_listed_between_them( + tmp_path, monkeypatch): + # A source on disk with .zip in its path is one the user-wide and .zip + # locators give one path under the user's cache directory; with the + # user-provided locator listed between them, a path too long that the + # user-wide one passed over and the .zip one took is cured by + # NUMBA_CACHE_DIR too, which numba tried between. The remedy read the + # location as the user-wide locator's, the first to match, and left the + # variable out. + import numba + from numba.core.caching import _CacheLocator + import numbox.core.configurations as configurations + + user_cache_dir = tmp_path / "user-cache" / "numba" + + class UserCacheUnderTmp: + def __init__(self, appname, appauthor): + self.user_cache_dir = str(user_cache_dir) + + monkeypatch.setattr(configurations, "AppDirs", UserCacheUnderTmp) + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", + "UserWideCacheLocator,UserProvidedCacheLocator,ZipCacheLocator", raising=False) + py_file = tmp_path / "data.zip" / "pkg" / "module.py" + py_file.parent.mkdir(parents=True) + py_file.write_text("def f():\n pass\n") + location = user_cache_dir / _CacheLocator.get_suitable_cache_subpath(str(py_file)) + remedy = configurations.cache_remedy( + str(py_file), OSError(errno.ENAMETOOLONG, "File name too long", str(location)), "silence") + assert remedy == ( + f"the path is too long for the file system: the user's cache directory, {user_cache_dir}, at a shorter " + f"path{configurations._moved_through()}, or NUMBA_CACHE_DIR set to a short path; or silence"), remedy + + +@pytest.mark.skipif(numba_version < 61, reason="numba's .zip locator arrived in 0.61") +def test_a_name_too_long_behind_the_zip_locators_error_for_a_path_with_zip_in_it_is_told_the_name( + tmp_path, monkeypatch): + # A source on disk in a directory whose name holds .zip without ending in + # it, long enough that the directory numba names after it is too long for + # the file system: the locators that name theirs so pass the file over, + # the in-tree one cannot cache beside the source, and numba's .zip locator + # raises for the ".zip" in the path. The remedy read the .zip locator's + # error alone and asked for NUMBA_CACHE_DIR, whose directory is named the + # same way and cannot be made either. + import numba + import numbox.core.configurations as configurations + + class UserCacheUnderTmp: + def __init__(self, appname, appauthor): + self.user_cache_dir = str(tmp_path / "user-cache" / "numba") + + monkeypatch.setattr(numba.core.caching, "AppDirs", UserCacheUnderTmp) + monkeypatch.setattr(configurations, "AppDirs", UserCacheUnderTmp) + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + limit = configurations._name_limit(str(tmp_path)) + package = tmp_path / ("proj.zip_" + "x" * (limit - 20)) + package.mkdir() + py_file = package / "module.py" + py_file.write_text("def f():\n pass\n") + # The in-tree locator cannot make __pycache__ beside the source. + (package / "__pycache__").write_text("") + with pytest.raises(ValueError, match="No zip file found") as raised: + configurations.check_cache_location(str(py_file), configurations.LONGEST_CACHE_FILE_NAME) + assert configurations.is_a_cache_error(raised.value) + remedy = configurations.cache_remedy(str(py_file), raised.value, "silence") + assert remedy.startswith("numba names the directory it caches this file in after the source's directory, "), remedy + assert remedy.endswith( + f": put the file in a directory of a shorter name, or make the __pycache__ beside the source, " + f"{package / '__pycache__'}, writable, where InTreeCacheLocator caches it under no such name; or silence"), remedy + assert "NUMBA_CACHE_DIR" not in remedy + + +@pytest.mark.skipif(not hasattr(os, "statvfs"), reason="the file system's name limit is asked through statvfs") +def test_the_name_limit_is_the_file_systems_own_asked_of_the_nearest_directory_that_exists(tmp_path, monkeypatch): + # The longest name a file can have is the file system's to say, 255 nearly + # everywhere; the remedy reads it from the nearest directory above the + # location that exists, and 255 where the system cannot say. Nothing told + # a limit the remedy made up from one the file system gave: the tests + # took their expected limit from the remedy's own answer. + import numbox.core.configurations as configurations + asked = [] + real = os.statvfs + + class Limit: + f_namemax = 80 + + def statvfs(path): + asked.append(path) + return Limit() + + monkeypatch.setattr(os, "statvfs", statvfs) + assert configurations._name_limit(str(tmp_path / "not" / "yet" / "made")) == 80 + assert asked == [str(tmp_path)] + monkeypatch.setattr(os, "statvfs", real) + monkeypatch.delattr(os, "statvfs") + assert configurations._name_limit(str(tmp_path)) == 255 + + def refusing(path): + raise OSError(errno.EACCES, "Permission denied", path) + + # A statvfs that raises, for a directory the user cannot search, leaves + # 255 too; no test held the fallback, and one answering 0 passed. + monkeypatch.setattr(os, "statvfs", refusing, raising=False) + assert configurations._name_limit(str(tmp_path)) == 255 + assert configurations._name_limit(None) == 255 + + class NoLimit: + f_namemax = 0 + + # A file system that reports 0, no limit of its own, left 0 for the + # limit, and every name was too long for it. + monkeypatch.setattr(os, "statvfs", lambda path: NoLimit()) + assert configurations._name_limit(str(tmp_path)) == 255 + + +def test_is_a_cache_error_admits_numbas_errors_by_their_text_and_no_other(): + # The line between numba's cache errors and the rest: any OSError; the + # .zip locator's ValueError by its text; numba's no-locator RuntimeError + # by its text; and, from numba 0.68, zipfile's KeyError for a member the + # archive does not hold, by its text, so that a KeyError of the code's + # own, a dictionary's, is raised as it was and not turned into the + # warning. Nothing pinned the texts: a classifier admitting every + # KeyError passed every test. + from numbox.core.configurations import is_a_cache_error + assert is_a_cache_error(OSError(errno.EACCES, "Permission denied", "/x")) + assert is_a_cache_error(ValueError("No zip file found in path")) + assert not is_a_cache_error(ValueError("invalid literal for int()")) + assert is_a_cache_error(RuntimeError("cannot cache function 'f': no locator available for file 'x.py'")) + assert not is_a_cache_error(RuntimeError("cannot cache function 'f': something else")) + assert is_a_cache_error(KeyError("There is no item named 'pkg/m.py' in the archive")) == (numba_version >= 68) + assert not is_a_cache_error(KeyError("jit")) + assert not is_a_cache_error(KeyError("There is no item named 'pkg/m.py' in the archive")) or numba_version >= 68 + assert not is_a_cache_error(TypeError("x")) + + +def test_a_reader_of_the_member_meanwhile_reads_the_members_own_lines_for_a_file_not_on_disk(tmp_path, monkeypatch): + # The probe's two lines stood in the process-wide linecache under the + # member's name while numba picked the locator, and numba's IPython + # locator reading another function's source from the member meanwhile, in + # another thread, read the probe's two lines for it and raised inspect's + # error for a function past them, "lineno is out of bounds". The probe is + # a function of a module of its own, whose loader reads the member's + # source from the archive as the member's own module's does, and the lines + # linecache holds from then on are the member's own. + import inspect + import linecache + import types + import numba + import numbox.core.configurations as configurations + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + ipython_dir = tmp_path / "ipython" + paths = types.ModuleType("IPython.paths") + paths.get_ipython_cache_dir = lambda: str(ipython_dir) + monkeypatch.setitem(sys.modules, "IPython", types.ModuleType("IPython")) + monkeypatch.setitem(sys.modules, "IPython.paths", paths) + source = "import os\n\n\ndef cell():\n return os.sep\n" + archive = tmp_path / "bundle.zip" + with zipfile.ZipFile(archive, "w") as zipped: + zipped.writestr("ipykernel_123/cell.py", source) + py_file = str(archive / "ipykernel_123" / "cell.py") + linecache.cache.pop(py_file, None) + + class ReadingTheMember: + # zipimport, for the member's own module. + def get_source(self, name): + return source + + member = types.ModuleType("ipykernel_123_cell") + member.__file__ = py_file + member.__loader__ = ReadingTheMember() + exec(compile(source, py_file, "exec"), member.__dict__) + monkeypatch.setitem(sys.modules, member.__name__, member) + assert member.cell.__code__.co_firstlineno == 4 + seen = [] + real = configurations.CompileResultCacheImpl + + def reading_meanwhile(probe): + seen.append(inspect.getsource(member.cell)) + locator = real(probe) + seen.append(inspect.getsource(member.cell)) + return locator + + monkeypatch.setattr(configurations, "CompileResultCacheImpl", reading_meanwhile) + configurations.check_cache_location(py_file, configurations.LONGEST_CACHE_FILE_NAME) + assert seen == ["def cell():\n return os.sep\n"] * 2, seen + assert linecache.getlines(py_file) == source.splitlines(True) + assert (ipython_dir / "numba_cache").is_dir() + linecache.cache.pop(py_file, None) + + +@pytest.mark.parametrize("given", ["Path", "bytes"]) +def test_a_zip_member_in_an_ipykernel_directory_given_as_bytes_or_a_path_object_is_checked_as_its_string_is( + tmp_path, monkeypatch, given): + # check_cache_location takes any path-like, bytes too, and the probe's + # code carries the name as a string, as compile makes it; the probe's + # module and loader were given the path as os.fspath left it, bytes for + # bytes, and the loader raised TypeError, through inspect, at the first + # path operation asked of it for a member numba's IPython locator reads. + import linecache + import types + import numba + import numbox.core.configurations as configurations + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + ipython_dir = tmp_path / "ipython" + paths = types.ModuleType("IPython.paths") + paths.get_ipython_cache_dir = lambda: str(ipython_dir) + monkeypatch.setitem(sys.modules, "IPython", types.ModuleType("IPython")) + monkeypatch.setitem(sys.modules, "IPython.paths", paths) + archive = tmp_path / "bundle.zip" + with zipfile.ZipFile(archive, "w") as zipped: + zipped.writestr("ipykernel_123/cell.py", "def cell():\n return 1\n") + py_file = archive / "ipykernel_123" / "cell.py" + linecache.cache.pop(str(py_file), None) + as_given = py_file if given == "Path" else os.fsencode(py_file) + configurations.check_cache_location(as_given, configurations.LONGEST_CACHE_FILE_NAME) + assert (ipython_dir / "numba_cache").is_dir() + assert linecache.getlines(str(py_file)) == ["def cell():\n", " return 1\n"] + linecache.cache.pop(str(py_file), None) + + +@pytest.mark.parametrize("placement", ["a .zip member", "on disk"]) +def test_an_error_naming_no_file_under_a_temporary_ipython_directory_is_told_the_locations_with_its_reason( + tmp_path, monkeypatch, placement): + # An error naming no file is told the locations numba could have taken, + # with its reason, as the docstring says; the remedy for an IPython that + # gives a new temporary directory on every call, to set IPYTHONDIR, took + # a caller's error with a reason and no name too, a full disk's, for a + # .zip member in an ipykernel directory, and dropped the reason. The error + # numba's first save raises under such an IPython names the directory it + # could not save in. + import tempfile + import types + import numba + import numbox.core.configurations as configurations + + class UserCacheUnderTmp: + def __init__(self, appname, appauthor): + self.user_cache_dir = str(tmp_path / "user-cache" / "numba") + + monkeypatch.setattr(configurations, "AppDirs", UserCacheUnderTmp) + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + paths = types.ModuleType("IPython.paths") + paths.get_ipython_cache_dir = lambda: tempfile.mkdtemp(dir=tmp_path) + monkeypatch.setitem(sys.modules, "IPython", types.ModuleType("IPython")) + monkeypatch.setitem(sys.modules, "IPython.paths", paths) + if placement == "a .zip member": + archive = tmp_path / "bundle.zip" + with zipfile.ZipFile(archive, "w") as zipped: + zipped.writestr("ipykernel_123/cell.py", "def cell():\n return 1\n") + py_file = archive / "ipykernel_123" / "cell.py" + else: + py_file = tmp_path / "ipykernel_123" / "cell.py" + py_file.parent.mkdir() + py_file.write_text("def cell():\n return 1\n") + failure = OSError(errno.ENOSPC, "No space left on device") + assert failure.filename is None + remedy = configurations.cache_remedy(str(py_file), failure, "silence") + assert "IPYTHONDIR" not in remedy and "temporary" not in remedy, remedy + assert "the location numba took, " in remedy and "(No space left on device)" in remedy, remedy + assert "numba_cache" in remedy, remedy + # The .zip locator's location, under the user's cache directory, is among + # them from numba 0.61, where that locator arrived; a file on disk has + # the user-wide locator's there under every version. + others = placement == "on disk" or numba_version >= 61 + assert (str(tmp_path / "user-cache" / "numba") in remedy) == others, remedy + + +@pytest.mark.skipif(numba_version < 62, reason="NUMBA_CACHE_LOCATOR_CLASSES arrived in numba 0.62") +@pytest.mark.parametrize("placement", ["a .zip member", "on disk"]) +def test_a_locator_of_none_of_numbas_families_listed_before_the_ipython_one_is_told_its_own_location( + tmp_path, monkeypatch, placement): + # A class of none of numba's families takes any file for the remedy, so + # numba reaches none listed after it; the remedy for an IPython that gives + # a new temporary directory on every call, to set IPYTHONDIR, asked only + # whether a locator of IPython's family was listed before the .zip one, + # and blamed IPython for the error such a class raised naming its own + # location, for a file in an ipykernel directory, where the location the + # error names is the remedy. + import tempfile + import types + import numba + import numbox.core.configurations as configurations + from numba.core.caching import _CacheLocator + + where = tmp_path / "our-cache" + + class OurLocator(_CacheLocator): + def __init__(self, py_func, py_file): + self._py_file = py_file + + def get_cache_path(self): + return str(where) + + def get_source_stamp(self): + return 0 + + def get_disambiguator(self): + return "x" + + def ensure_cache_path(self): + raise PermissionError(errno.EACCES, "Permission denied", str(where)) + + @classmethod + def from_function(cls, py_func, py_file): + return cls(py_func, py_file) + + ours = types.ModuleType("our_locators") + ours.OurLocator = OurLocator + monkeypatch.setitem(sys.modules, "our_locators", ours) + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "our_locators.OurLocator,IPythonCacheLocator", + raising=False) + paths = types.ModuleType("IPython.paths") + paths.get_ipython_cache_dir = lambda: tempfile.mkdtemp(dir=tmp_path) + monkeypatch.setitem(sys.modules, "IPython", types.ModuleType("IPython")) + monkeypatch.setitem(sys.modules, "IPython.paths", paths) + if placement == "a .zip member": + archive = tmp_path / "bundle.zip" + with zipfile.ZipFile(archive, "w") as zipped: + zipped.writestr("ipykernel_123/cell.py", "def cell():\n return 1\n") + py_file = archive / "ipykernel_123" / "cell.py" + else: + py_file = tmp_path / "ipykernel_123" / "cell.py" + py_file.parent.mkdir() + py_file.write_text("def cell():\n return 1\n") + with pytest.raises(PermissionError) as raised: + configurations.check_cache_location(str(py_file), configurations.LONGEST_CACHE_FILE_NAME) + assert raised.value.filename == str(where) + remedy = configurations.cache_remedy(str(py_file), raised.value, "silence") + if placement == "a .zip member": + assert remedy == ( + "NUMBA_CACHE_DIR has no effect here, because the source is not a file on disk: no file can be written at " + f"that location, {where} (Permission denied), so make room there, or make it writable, or silence"), remedy + else: + assert remedy == ( + f"that location, {where}, where no file can be written (Permission denied): make room there, or make it " + "writable; or silence"), remedy + + +@needs_a_directory_it_cannot_write +@pytest.mark.skipif(numba_version < 61, reason="numba's .zip locator arrived in 0.61") +def test_a_zip_members_location_that_cannot_be_made_is_told_the_directory_refusing_it(tmp_path, monkeypatch): + # numba's .zip locator takes a member without making its location, and + # the first save makes it under the user's cache directory, as the check + # does; where a directory on the way is not writable, the error names the + # location, which is not there, and the remedy asked for room in it, or + # for it to be made writable, and never named the directory refusing. + import numba + import numbox.core.configurations as configurations + + class UserCacheUnderHome: + def __init__(self, appname, appauthor): + self.user_cache_dir = str(tmp_path / "home" / ".cache" / "numba") + + monkeypatch.setattr(configurations, "AppDirs", UserCacheUnderHome) + monkeypatch.setattr(numba.core.caching, "AppDirs", UserCacheUnderHome) + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + archive = tmp_path / "bundle.zip" + with zipfile.ZipFile(archive, "w") as zipped: + zipped.writestr("package/module.py", "def f():\n pass\n") + py_file = str(archive / "package" / "module.py") + refusing = tmp_path / "home" / ".cache" + refusing.mkdir(parents=True) + refusing.chmod(0o555) + try: + with pytest.raises(PermissionError) as raised: + configurations.check_cache_location(py_file, configurations.LONGEST_CACHE_FILE_NAME) + location = raised.value.filename + assert location.startswith(str(refusing / "numba") + os.sep) and not os.path.isdir(location) + remedy = configurations.cache_remedy(py_file, raised.value, "silence") + finally: + refusing.chmod(0o755) + assert remedy == ( + "numba caches a .zip, or a frozen application, in the user's cache directory, and NUMBA_CACHE_DIR has no " + f"effect here, because the source is not a file on disk: no file can be written in that directory, {location}, " + f"which cannot be made under {refusing} (Permission denied), so make room there, or make it writable, or " + "silence"), remedy + + +@pytest.mark.skipif(numba_version < 62, reason="NUMBA_CACHE_LOCATOR_CLASSES arrived in numba 0.62") +@pytest.mark.parametrize("placement", ["a .zip member", "on disk"]) +def test_a_locator_of_none_of_numbas_families_that_passes_the_file_over_leaves_the_ipython_one_reached( + tmp_path, monkeypatch, placement): + # A class of none of numba's families takes what its from_function takes, + # which the remedy asks it with a probe as numba does; the remedy took + # such a class for taking any file, so for one listed before + # IPythonCacheLocator that passed the file over, numba going on to + # IPython's locator, it read IPython's locator as never reached, and told + # the error numba's first save raises under an IPython giving a temporary + # directory per call as a location that is none of numba's, to make + # writable, where IPYTHONDIR is the remedy. + import tempfile + import types + import numba + import numbox.core.configurations as configurations + from numba.core.caching import _CacheLocator + + class OurPasser(_CacheLocator): + @classmethod + def from_function(cls, py_func, py_file): + return None + + ours = types.ModuleType("our_locators") + ours.OurPasser = OurPasser + monkeypatch.setitem(sys.modules, "our_locators", ours) + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "our_locators.OurPasser,IPythonCacheLocator", + raising=False) + paths = types.ModuleType("IPython.paths") + paths.get_ipython_cache_dir = lambda: tempfile.mkdtemp(dir=tmp_path) + monkeypatch.setitem(sys.modules, "IPython", types.ModuleType("IPython")) + monkeypatch.setitem(sys.modules, "IPython.paths", paths) + if placement == "a .zip member": + archive = tmp_path / "bundle.zip" + with zipfile.ZipFile(archive, "w") as zipped: + zipped.writestr("ipykernel_123/cell.py", "def cell():\n return 1\n") + py_file = archive / "ipykernel_123" / "cell.py" + else: + py_file = tmp_path / "ipykernel_123" / "cell.py" + py_file.parent.mkdir() + py_file.write_text("def cell():\n return 1\n") + with pytest.raises(FileNotFoundError) as raised: + configurations.check_cache_location(str(py_file), configurations.LONGEST_CACHE_FILE_NAME) + assert raised.value.filename.endswith(os.sep + "numba_cache") + remedy = configurations.cache_remedy(str(py_file), raised.value, "silence") + assert remedy == ( + "numba caches this file in numba_cache under IPython's cache directory, and IPython, able to write neither " + "that directory nor its own, gives a new temporary directory on every call, so numba makes numba_cache in one " + "and saves in another: set IPYTHONDIR, or XDG_CACHE_HOME on a POSIX system, " + "to a writable directory; or silence"), remedy + + +@pytest.mark.skipif(numba_version < 62, reason="NUMBA_CACHE_LOCATOR_CLASSES arrived in numba 0.62") +@pytest.mark.parametrize("placement", ["a .zip member", "on disk"]) +def test_a_location_of_a_locator_listed_after_a_class_of_none_of_numbas_families_that_took_the_file_is_none_of_numbas( + tmp_path, monkeypatch, placement): + # A class of none of numba's families that takes the file, asked with a + # probe as numba asks it, is the last locator numba reaches; the remedy + # read the ones listed after it as reached too, and told an error naming + # IPython's numba_cache as numba caching the file there, a location numba + # never reached. It is a location that is none of numba's for the file. + import types + import numba + import numbox.core.configurations as configurations + from numba.core.caching import _CacheLocator + + where = tmp_path / "our-cache" + + class OurLocator(_CacheLocator): + def __init__(self, py_func, py_file): + self._py_file = py_file + + def get_cache_path(self): + return str(where) + + def get_source_stamp(self): + return 0 + + def get_disambiguator(self): + return "x" + + @classmethod + def from_function(cls, py_func, py_file): + return cls(py_func, py_file) + + ours = types.ModuleType("our_locators") + ours.OurLocator = OurLocator + monkeypatch.setitem(sys.modules, "our_locators", ours) + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "our_locators.OurLocator,IPythonCacheLocator", + raising=False) + ipython_dir = tmp_path / "ipython" + paths = types.ModuleType("IPython.paths") + paths.get_ipython_cache_dir = lambda: str(ipython_dir) + monkeypatch.setitem(sys.modules, "IPython", types.ModuleType("IPython")) + monkeypatch.setitem(sys.modules, "IPython.paths", paths) + if placement == "a .zip member": + archive = tmp_path / "bundle.zip" + with zipfile.ZipFile(archive, "w") as zipped: + zipped.writestr("ipykernel_123/cell.py", "def cell():\n return 1\n") + py_file = archive / "ipykernel_123" / "cell.py" + else: + py_file = tmp_path / "ipykernel_123" / "cell.py" + py_file.parent.mkdir() + py_file.write_text("def cell():\n return 1\n") + configurations.check_cache_location(str(py_file), configurations.LONGEST_CACHE_FILE_NAME) + assert where.is_dir() and not ipython_dir.exists() + named = ipython_dir / "numba_cache" + remedy = configurations.cache_remedy(str(py_file), PermissionError(errno.EACCES, "Permission denied", str(named)), + "silence") + assert "numba caches this file in" not in remedy, remedy + if placement == "a .zip member": + assert remedy == ( + "NUMBA_CACHE_DIR has no effect here, because the source is not a file on disk: no file can be written at " + f"that location, {named} (Permission denied), so make room there, or make it writable, or silence"), remedy + else: + assert remedy == ( + f"that location, {named}, where no file can be written (Permission denied): make room there, or make it " + "writable; or silence"), remedy + + +@pytest.mark.skipif(numba_version < 62, reason="NUMBA_CACHE_LOCATOR_CLASSES arrived in numba 0.62") +@pytest.mark.parametrize("placement", ["a .zip member", "on disk"]) +def test_a_locator_of_none_of_numbas_families_whose_from_function_raises_takes_the_file(tmp_path, monkeypatch, placement): + # A class of none of numba's families whose from_function raises takes + # the file, the error being its own, as numba raises it at decoration: + # numba reaches none listed after it. Read as passing the file over, it + # would leave IPython's locator, listed after it, reached, and the error + # it raised naming its own location would be told as IPython's, to set + # IPYTHONDIR for. No test held the rule. + import tempfile + import types + import numba + import numbox.core.configurations as configurations + from numba.core.caching import _CacheLocator + + where = tmp_path / "our-cache" + + class OurRaiser(_CacheLocator): + @classmethod + def from_function(cls, py_func, py_file): + raise PermissionError(errno.EACCES, "Permission denied", str(where)) + + ours = types.ModuleType("our_locators") + ours.OurRaiser = OurRaiser + monkeypatch.setitem(sys.modules, "our_locators", ours) + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "our_locators.OurRaiser,IPythonCacheLocator", + raising=False) + paths = types.ModuleType("IPython.paths") + paths.get_ipython_cache_dir = lambda: tempfile.mkdtemp(dir=tmp_path) + monkeypatch.setitem(sys.modules, "IPython", types.ModuleType("IPython")) + monkeypatch.setitem(sys.modules, "IPython.paths", paths) + if placement == "a .zip member": + archive = tmp_path / "bundle.zip" + with zipfile.ZipFile(archive, "w") as zipped: + zipped.writestr("ipykernel_123/cell.py", "def cell():\n return 1\n") + py_file = archive / "ipykernel_123" / "cell.py" + else: + py_file = tmp_path / "ipykernel_123" / "cell.py" + py_file.parent.mkdir() + py_file.write_text("def cell():\n return 1\n") + assert configurations._takes(OurRaiser, str(py_file)) + with pytest.raises(PermissionError) as raised: + configurations.check_cache_location(str(py_file), configurations.LONGEST_CACHE_FILE_NAME) + assert raised.value.filename == str(where) + remedy = configurations.cache_remedy(str(py_file), raised.value, "silence") + if placement == "a .zip member": + assert remedy == ( + "NUMBA_CACHE_DIR has no effect here, because the source is not a file on disk: no file can be written at " + f"that location, {where} (Permission denied), so make room there, or make it writable, or silence"), remedy + else: + assert remedy == ( + f"that location, {where}, where no file can be written (Permission denied): make room there, or make it " + "writable; or silence"), remedy + + +@pytest.mark.skipif(numba_version < 61, reason="numba's .zip locator arrived in 0.61") +@on_windows_under_numba_0_68_0 +def test_a_zip_members_location_under_a_file_is_told_the_file_in_its_path(tmp_path, monkeypatch): + # The user's cache directory can lie under a file, XDG_CACHE_HOME set to + # a path through one; numba's first save, making the member's location, + # raises for it, and the remedy named the first directory above the + # location that is there, writable, and asked for room in it: the file + # in the way is what to move. + import numba + import numbox.core.configurations as configurations + + a_file = tmp_path / "home" / "afile" + a_file.parent.mkdir() + a_file.write_text("not a directory\n") + + class UserCacheUnderTheFile: + def __init__(self, appname, appauthor): + self.user_cache_dir = str(a_file / "c" / "numba") + + monkeypatch.setattr(configurations, "AppDirs", UserCacheUnderTheFile) + monkeypatch.setattr(numba.core.caching, "AppDirs", UserCacheUnderTheFile) + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + archive = tmp_path / "bundle.zip" + with zipfile.ZipFile(archive, "w") as zipped: + zipped.writestr("package/module.py", "def f():\n pass\n") + py_file = str(archive / "package" / "module.py") + with pytest.raises(OSError) as raised: + configurations.check_cache_location(py_file, configurations.LONGEST_CACHE_FILE_NAME) + location = raised.value.filename + assert location.startswith(str(a_file) + os.sep) and not os.path.exists(location) + remedy = configurations.cache_remedy(py_file, raised.value, "silence") + assert remedy == ( + "numba caches a .zip, or a frozen application, in the user's cache directory, and NUMBA_CACHE_DIR has no " + f"effect here, because the source is not a file on disk: no file can be written in that directory, {location}, " + f"which cannot be made, {a_file} being a file in its path ({raised.value.strerror}), so put that file " + "elsewhere, or silence"), remedy + + +@pytest.mark.skipif(numba_version < 62, reason="NUMBA_CACHE_LOCATOR_CLASSES arrived in numba 0.62") +@pytest.mark.parametrize("error", ["no archive in the path", "a directory named .zip"]) +def test_the_zip_locators_errors_after_a_class_of_none_of_numbas_families_that_took_the_file_are_a_callers( + tmp_path, monkeypatch, error): + # numba's .zip locator listed after a class of none of numba's families + # that takes the file, asked, is not reached, so its errors under the + # list are a caller's: the ValueError for a path with .zip in it and no + # part ending in it, and the IsADirectoryError for a directory named + # .zip above a source on disk. The remedy told both as the .zip + # locator's, the locators before it having passed the file over, where + # one of them took it. + import types + import numba + import numbox.core.configurations as configurations + from numba.core.caching import _CacheLocator + + where = tmp_path / "our-cache" + + class OurLocator(_CacheLocator): + def __init__(self, py_func, py_file): + self._py_file = py_file + + def get_cache_path(self): + return str(where) + + def get_source_stamp(self): + return 0 + + def get_disambiguator(self): + return "x" + + @classmethod + def from_function(cls, py_func, py_file): + return cls(py_func, py_file) + + ours = types.ModuleType("our_locators") + ours.OurLocator = OurLocator + monkeypatch.setitem(sys.modules, "our_locators", ours) + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "our_locators.OurLocator,ZipCacheLocator", + raising=False) + if error == "no archive in the path": + py_file = tmp_path / "not.zipped" / "module.py" + py_file.parent.mkdir() + py_file.write_text("x = 1\n") + configurations.check_cache_location(str(py_file), configurations.LONGEST_CACHE_FILE_NAME) + remedy = configurations.cache_remedy(str(py_file), ValueError("No zip file found in path"), "silence") + assert "passed the file over" not in remedy and ".zip locator" not in remedy, remedy + assert remedy == ( + "numba looks only where NUMBA_CACHE_LOCATOR_CLASSES, our_locators.OurLocator,ZipCacheLocator, says: make " + "one of those locations writable, or silence"), remedy + else: + container = tmp_path / "data.zip" + py_file = container / "package" / "module.py" + py_file.parent.mkdir(parents=True) + py_file.write_text("x = 1\n") + configurations.check_cache_location(str(py_file), configurations.LONGEST_CACHE_FILE_NAME) + remedy = configurations.cache_remedy(str(py_file), OSError(errno.EISDIR, "Is a directory", str(container)), + "silence") + assert "passed this file over" not in remedy and ".zip locator" not in remedy, remedy + assert remedy == ( + f"that location, {container}, where no file can be written (Is a directory): make room there, or make it " + "writable; or silence"), remedy + assert where.is_dir() + + +@pytest.mark.parametrize("beside", ["its source beside it", "alone"]) +def test_a_keyerror_for_a_pyc_compiled_to_name_another_path_in_the_archive_is_told_the_pyc(tmp_path, beside): + # numba 0.68 and later read a .zip member's source by the name the + # module's code gives, and zipfile raises for a name the archive does not + # hold; the remedy read every such error as a module shipped as .pyc + # alone and asked for source files, where the archive held the module's + # source beside a .pyc compiled to name another path in it, compileall's + # -d, which zipimport runs: the .pyc is what to recompile or leave out. + import numbox.core.configurations as configurations + + archive = tmp_path / "bundle.zip" + source = tmp_path / "b.py" + source.write_text("def f():\n pass\n") + compiled = py_compile.compile(str(source), cfile=str(tmp_path / "b.pyc"), + dfile=os.path.join(str(archive), "src", "pkg", "b.py"), doraise=True, + invalidation_mode=py_compile.PycInvalidationMode.UNCHECKED_HASH) + with zipfile.ZipFile(archive, "w") as zipped: + zipped.writestr("pkg/__init__.py", "") + if beside == "its source beside it": + zipped.write(source, "pkg/b.py") + zipped.write(compiled, "pkg/b.pyc") + py_file = str(archive / "src" / "pkg" / "b.py") + assert configurations._compiled_from(str(archive), "pkg", "b") == py_file + failure = KeyError("There is no item named 'src/pkg/b.py' in the archive") + remedy = configurations.cache_remedy(py_file, failure, "silence") + if beside == "its source beside it": + assert remedy == ( + "numba 0.68 and later read a .zip member's source from the archive by the name the module's code gives, " + "src/pkg/b.py, which pkg/b.pyc was compiled to name, and this one holds none so named, while it holds the " + "module's source, pkg/b.py: to cache, compile the .pyc to name its path in the archive, or leave it out, " + "so that Python compiles the source and numba reads it under that name; or silence"), remedy + else: + assert remedy.startswith( + "numba 0.68 and later read a .zip member's source from the archive, and this one holds none under " + "src/pkg/b.py, the module shipped as .pyc alone: to cache, "), remedy + + +@pytest.mark.skipif(numba_version < 61, reason="numba's .zip locator arrived in 0.61") +def test_an_archive_numba_cannot_read_is_told_to_make_it_readable(tmp_path, monkeypatch): + # numba reads the archive for the member's stamp, and from 0.68 opens it + # for the member's source; one it cannot read raises naming the archive, + # and the remedy told the archive as a location that refused a file, to + # make room in or make writable, which reads nothing. + import numba + import numbox.core.configurations as configurations + + class UserCacheUnderTmp: + def __init__(self, appname, appauthor): + self.user_cache_dir = str(tmp_path / "user-cache" / "numba") + + monkeypatch.setattr(configurations, "AppDirs", UserCacheUnderTmp) + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + archive = tmp_path / "bundle.zip" + with zipfile.ZipFile(archive, "w") as zipped: + zipped.writestr("package/module.py", "def f():\n pass\n") + py_file = str(archive / "package" / "module.py") + failure = PermissionError(errno.EACCES, "Permission denied", str(archive)) + remedy = configurations.cache_remedy(py_file, failure, "silence") + assert "writable" not in remedy and "make room" not in remedy, remedy + assert remedy == ( + "NUMBA_CACHE_DIR has no effect here, because the source is not a file on disk: numba reads the archive, " + f"{archive}, for the member's stamp, and from 0.68 for the member's source, and cannot (Permission denied): " + "make it readable, or silence"), remedy + + +def test_an_error_naming_a_members_moved_archive_under_a_temporary_ipython_directory_is_the_archives( + tmp_path, monkeypatch): + # The error numba's first save raises under an IPython giving a temporary + # directory per call names a numba_cache under that directory; one naming + # the archive a member's code names, where that has since moved, is the + # stamp's, told as the archive's, and the remedy for the temporary + # directory took it, for a member in an ipykernel directory, and said + # IPYTHONDIR. In the flow itself inspect reads no source from an archive + # that is gone, and that error is told first; a caller's error can name + # the archive. + import tempfile + import types + import numba + import numbox.core.configurations as configurations + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + paths = types.ModuleType("IPython.paths") + paths.get_ipython_cache_dir = lambda: tempfile.mkdtemp(dir=tmp_path) + monkeypatch.setitem(sys.modules, "IPython", types.ModuleType("IPython")) + monkeypatch.setitem(sys.modules, "IPython.paths", paths) + archive = tmp_path / "bundle.zip" + py_file = str(archive / "ipykernel_123" / "cell.py") + failure = FileNotFoundError(errno.ENOENT, "No such file or directory", str(archive)) + remedy = configurations.cache_remedy(py_file, failure, "silence") + assert "IPYTHONDIR" not in remedy, remedy + assert remedy.startswith(f"the module's code names {archive}, which is not there: its .pyc was compiled to name " + "that path, so "), remedy + + +@pytest.mark.skipif(numba_version < 61, reason="numba's .zip locator arrived in 0.61") +@pytest.mark.parametrize("name", ["APP.ZIP", "bundle.egg", "my.zipdir/bundle.egg"]) +def test_an_archive_numbas_zip_locator_does_not_read_is_told_to_be_named_zip(tmp_path, monkeypatch, name): + # numba's .zip locator takes a path with ".zip" in it, lower-case, so an + # archive named APP.ZIP, or an .egg, which Python imports from all the + # same, leaves numba no locator; the remedy told the reader to import + # from a .zip holding the source files, which they did, and never named + # the extension numba does not read. An .egg under a directory with + # ".zip" in its name is taken by that locator, which finds no part ending + # in .zip and raises its ValueError; the remedy held the rename back for + # the ".zip" in the path, and told the same. + import numba + import numbox.core.configurations as configurations + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + archive = tmp_path / name + archive.parent.mkdir(parents=True, exist_ok=True) + with zipfile.ZipFile(archive, "w") as zipped: + zipped.writestr("pkg/__init__.py", "") + zipped.writestr("pkg/mod.py", "def f():\n pass\n") + py_file = str(archive / "pkg" / "mod.py") + error, text = (ValueError, "No zip file found") if ".zip" in name else (RuntimeError, "no locator available") + with pytest.raises(error, match=text) as raised: + configurations.check_cache_location(py_file, configurations.LONGEST_CACHE_FILE_NAME) + assert configurations.is_a_cache_error(raised.value) + remedy = configurations.cache_remedy(py_file, raised.value, "silence") + assert "import it from a .zip" not in remedy, remedy + assert remedy == ( + "NUMBA_CACHE_DIR has no effect here, because the source is not a file on disk: to cache, install numbox with " + f"its source files on disk, unpacked from any archive, or rename the archive, {archive}, to end in .zip, " + "lower-case, which numba's .zip locator looks for in the path, with its source files in it, so that numba " + "0.61 and later cache it in the user's cache directory; or silence"), remedy + + +def test_the_archive_above_a_file_is_found_with_os_name_set_to_windows(tmp_path, monkeypatch): + # The archive above a file was found through pathlib.Path, which makes a + # path of the platform os.name says: set to Windows's, as the tests for + # numba's separator bug set it, Python before 3.12 refuses to make one on + # Linux, and the remedy for such a member, finding the archive for the + # rename it offers, died on it. The path is read by the separator in use. + import numbox.core.configurations as configurations + archive = tmp_path / "bundle.zip" + with zipfile.ZipFile(archive, "w") as zipped: + zipped.writestr("pkg/mod.py", "x = 1\n") + py_file = str(archive / "pkg" / "mod.py") + monkeypatch.setattr(os, "name", "nt") + assert configurations._archive_above(py_file) == (str(archive), "pkg/mod.py") + assert configurations._archive_above(str(tmp_path / "elsewhere" / "mod.py")) is None + + +def test_a_relative_members_moved_archive_under_a_temporary_ipython_directory_is_the_archives(tmp_path, monkeypatch): + # The file numba names can be relative, run from the archive's directory, + # and the error naming the archive above it relative too; the remedy for + # an IPython giving a temporary directory per call compared the file as + # given against the archive's name made absolute, matched nothing, and + # said IPYTHONDIR, where the archive's remedy is the one. + import tempfile + import types + import numba + import numbox.core.configurations as configurations + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + paths = types.ModuleType("IPython.paths") + paths.get_ipython_cache_dir = lambda: tempfile.mkdtemp(dir=tmp_path) + monkeypatch.setitem(sys.modules, "IPython", types.ModuleType("IPython")) + monkeypatch.setitem(sys.modules, "IPython.paths", paths) + monkeypatch.chdir(tmp_path) + py_file = os.path.join("bundle.zip", "ipykernel_123", "cell.py") + failure = FileNotFoundError(errno.ENOENT, "No such file or directory", "bundle.zip") + remedy = configurations.cache_remedy(py_file, failure, "silence") + assert "IPYTHONDIR" not in remedy, remedy + assert remedy.startswith("the module's code names bundle.zip, which is not there: its .pyc was compiled to name " + "that path, so "), remedy + + +@pytest.mark.skipif(numba_version < 61, reason="numba's .zip locator arrived in 0.61") +def test_a_relative_members_archive_named_absolute_by_the_error_is_told_as_moved(tmp_path, monkeypatch): + # The file numba names can be relative, run from the archive's directory, + # while the error names the archive absolute, as the stamp's does; the + # archive's remedy compared the file as given against the name, matched + # nothing, and told the archive that is gone as a location refusing a + # file, to make room in or make writable. + import numba + import numbox.core.configurations as configurations + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + monkeypatch.chdir(tmp_path) + py_file = os.path.join("build", "numbox.zip", "numbox", "module.py") + archive = str(tmp_path / "build" / "numbox.zip") + failure = FileNotFoundError(errno.ENOENT, "No such file or directory", archive) + remedy = configurations.cache_remedy(py_file, failure, "silence") + assert "make room" not in remedy and "writable" not in remedy, remedy + assert remedy.startswith(f"the module's code names {archive}, which is not there: its .pyc was compiled to name " + "that path, so "), remedy + + +@pytest.mark.skipif(numba_version < 62, reason="NUMBA_CACHE_LOCATOR_CLASSES arrived in numba 0.62") +def test_a_name_too_long_names_the_locator_the_in_tree_one_is_to_be_listed_before(tmp_path, monkeypatch): + # The in-tree locator caches beside the source under no directory named + # after the source's, so a name too long for the user's cache directory + # is told to list it before the locator that took the file; the remedy + # named numba's .zip locator for any, where a class of none of numba's + # families, listed before the in-tree one, took the file and the .zip + # locator was not listed at all. + import types + import numba + import numbox.core.configurations as configurations + from numba.core.caching import _CacheLocator + + class UserCacheUnderTmp: + def __init__(self, appname, appauthor): + self.user_cache_dir = str(tmp_path / "user-cache" / "numba") + + class OurLocator(_CacheLocator): + def __init__(self, py_func, py_file): + self._py_file = py_file + + def get_cache_path(self): + return str(tmp_path / "our-cache") + + def get_source_stamp(self): + return 0 + + def get_disambiguator(self): + return "x" + + @classmethod + def from_function(cls, py_func, py_file): + return cls(py_func, py_file) + + ours = types.ModuleType("our_locators") + ours.OurLocator = OurLocator + monkeypatch.setitem(sys.modules, "our_locators", ours) + monkeypatch.setattr(configurations, "AppDirs", UserCacheUnderTmp) + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", + "UserWideCacheLocator,our_locators.OurLocator,InTreeCacheLocator", raising=False) + monkeypatch.setattr(configurations, "_name_limit", lambda location: 20) + py_file = tmp_path / "site" / "package" / "module.py" + py_file.parent.mkdir(parents=True) + py_file.write_text("x = 1\n") + subpath = _CacheLocator.get_suitable_cache_subpath(str(py_file)) + named = str(tmp_path / "user-cache" / "numba" / subpath) + failure = OSError(errno.ENAMETOOLONG, "File name too long", named) + remedy = configurations.cache_remedy(str(py_file), failure, "silence") + assert "ZipCacheLocator" not in remedy, remedy + # The name is counted as the file system counts it, in UTF-16 units on + # Windows and macOS, bytes elsewhere; the test said bytes everywhere. + counted_in = "characters" if os.name == "nt" or sys.platform == "darwin" else "bytes" + assert remedy == ( + f"numba names the directory it caches this file in after the source's directory, {subpath}, " + f"{len(subpath.encode())} {counted_in}, which is longer than a name can be on that file system (20): put the " + "file in a directory of a shorter name, or list InTreeCacheLocator before OurLocator, which caches beside the " + "source, in its __pycache__, under no such name; or silence"), remedy + + +@pytest.mark.skipif(os.name == "nt", reason="Windows decodes any byte of a path") +@pytest.mark.skipif(sys.platform == "darwin", reason="APFS takes no name that is not UTF-8") +@pytest.mark.parametrize("error", ["a full __pycache__", "numba's no-locator error"]) +def test_a_source_under_a_directory_that_is_not_utf8_is_not_offered_numba_cache_dir(tmp_path, monkeypatch, error): + # numba names the directory it caches a file in under NUMBA_CACHE_DIR by + # hashing the source's directory as UTF-8, and raises UnicodeEncodeError + # for one that is not, no cache error; the remedy offered the variable + # for such a source, and set as advised, the import died where the + # warning had stood. + import numba + import numbox.core.configurations as configurations + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + directory = tmp_path / os.fsdecode(b"caf\xe9") / "package" + directory.mkdir(parents=True) + py_file = str(directory / "module.py") + with open(py_file, "w", encoding="utf-8") as handle: + handle.write("x = 1\n") + with pytest.raises(UnicodeEncodeError): + py_file.encode() + if error == "a full __pycache__": + failure = OSError(errno.ENOSPC, "No space left on device", str(directory / "__pycache__")) + else: + failure = RuntimeError(f"cannot cache function '_cache_probe': no locator available for file '{py_file}'") + remedy = configurations.cache_remedy(py_file, failure, "silence") + assert "NUMBA_CACHE_DIR set to" not in remedy and "Set NUMBA_CACHE_DIR" not in remedy, remedy + if error == "a full __pycache__": + assert remedy.endswith( + "make room there, or make it writable, and NUMBA_CACHE_DIR cures nothing, numba hashing the source's " + "directory as UTF-8 for its directory there, which this one is not; or silence"), remedy + else: + # numba's no-locator error here is the in-tree locator passing an + # unwritable __pycache__ over before the user-wide one raised, so + # that directory writable is a cure too, hashing no path. + assert remedy == ( + "numba hashes the source's directory as UTF-8 for the directory it caches this file in under " + f"NUMBA_CACHE_DIR or the user's cache directory, and {directory} is not: put the file under a UTF-8 " + f"path, or make the __pycache__ beside the source, {directory / '__pycache__'}, writable, where " + "InTreeCacheLocator caches it hashing no path; or silence"), remedy + + +def test_a_zip_member_in_an_ipykernel_directory_that_is_not_utf8_is_told_so(tmp_path, monkeypatch): + # numba's IPython locator reads a member's source through zipimport, + # which decodes it as UTF-8 and raises for one that is not, as numba then + # does at decoration; the probe's loader read it the same way, and the + # UnicodeDecodeError came out of the check as its own, no cache error, so + # the import died where the warning should have stood. inspect's error + # tells such a member, and the remedy reads why. + import types + import numba + import numbox.core.configurations as configurations + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + ipython_dir = tmp_path / "ipython" + paths = types.ModuleType("IPython.paths") + paths.get_ipython_cache_dir = lambda: str(ipython_dir) + monkeypatch.setitem(sys.modules, "IPython", types.ModuleType("IPython")) + monkeypatch.setitem(sys.modules, "IPython.paths", paths) + archive = tmp_path / "bundle.zip" + with zipfile.ZipFile(archive, "w") as zipped: + zipped.writestr("ipykernel_123/cell.py", b"x = 'caf\xe9'\n") + py_file = str(archive / "ipykernel_123" / "cell.py") + with pytest.raises(OSError, match="could not get source code") as raised: + configurations.check_cache_location(py_file, configurations.LONGEST_CACHE_FILE_NAME) + assert configurations.is_a_cache_error(raised.value) + remedy = configurations.cache_remedy(py_file, raised.value, "silence") + assert remedy == ( + f"numba's IPython locator reads the member's source as UTF-8, which {py_file} is not (invalid continuation " + "byte at byte 8): ship it as UTF-8; or silence"), remedy + + +@pytest.mark.skipif(numba_version < 62, reason="NUMBA_CACHE_LOCATOR_CLASSES arrived in numba 0.62") +def test_a_listed_locator_whose_module_raises_on_import_is_no_locator_for_the_remedy(tmp_path, monkeypatch): + # An entry of NUMBA_CACHE_LOCATOR_CLASSES whose module raises on import, + # anything, not importlib's error alone, numba raises as its own at + # decoration, and the check before any remedy; the remedy, reached for + # an error of the cache's, imported the module for the entry and raised + # the module's error in place of the remedy. + import numba + import numbox.core.configurations as configurations + (tmp_path / "badmod.py").write_text("raise RuntimeError('boom on import')\n") + monkeypatch.syspath_prepend(str(tmp_path)) + monkeypatch.delitem(sys.modules, "badmod", raising=False) + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "badmod.SomeClass,InTreeCacheLocator", raising=False) + py_file = tmp_path / "site" / "package" / "module.py" + py_file.parent.mkdir(parents=True) + py_file.write_text("x = 1\n") + failure = OSError(errno.EACCES, "Permission denied", str(py_file.parent / "__pycache__")) + remedy = configurations.cache_remedy(str(py_file), failure, "silence") + assert "boom" not in remedy and remedy.endswith("; or silence"), remedy + assert configurations._locators("badmod.SomeClass,InTreeCacheLocator") == [ + configurations._numba_locator("InTreeCacheLocator")] + + +@pytest.mark.skipif(numba_version < 61, reason="numba's .zip locator arrived in 0.61") +def test_an_empty_path_object_as_an_errors_name_is_the_working_directory_not_an_archive_above(tmp_path, monkeypatch): + # A path object built from an empty string names the working directory, + # which the remedy tells as a location that is none of numba's; for a + # member under that directory it stood as a path above the file, and a + # FileNotFoundError naming it was told as the member's moved archive. + import numba + import numbox.core.configurations as configurations + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + monkeypatch.chdir(tmp_path) + archive = tmp_path / "bundle.zip" + with zipfile.ZipFile(archive, "w") as zipped: + zipped.writestr("pkg/mod.py", "x = 1\n") + py_file = str(archive / "pkg" / "mod.py") + failure = FileNotFoundError(errno.ENOENT, "No such file or directory", Path("")) + remedy = configurations.cache_remedy(py_file, failure, "silence") + assert "which is not there" not in remedy and "compiled to name" not in remedy, remedy + assert remedy == ( + "NUMBA_CACHE_DIR has no effect here, because the source is not a file on disk: no file can be written at " + "that location, . (No such file or directory), so make room there, or make it writable, or silence"), remedy + + +@pytest.mark.parametrize("placement", ["on disk", "a .zip member"]) +def test_an_error_naming_no_numba_cache_under_a_temporary_ipython_directory_is_not_told_ipythondir( + tmp_path, monkeypatch, placement): + # The error numba's first save raises under an IPython giving a temporary + # directory per call names the numba_cache it could not save in, as the + # check's does; the remedy for that case took any error naming a location + # that is none of numba's, and told IPYTHONDIR for a cell file on disk + # that the in-tree locator took, IPython never reached, on a caller's + # error naming an unrelated place. + import tempfile + import types + import numba + import numbox.core.configurations as configurations + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + paths = types.ModuleType("IPython.paths") + paths.get_ipython_cache_dir = lambda: tempfile.mkdtemp(dir=tmp_path) + monkeypatch.setitem(sys.modules, "IPython", types.ModuleType("IPython")) + monkeypatch.setitem(sys.modules, "IPython.paths", paths) + if placement == "on disk": + py_file = tmp_path / "ipykernel_123" / "cell.py" + py_file.parent.mkdir() + py_file.write_text("def cell():\n return 1\n") + configurations.check_cache_location(str(py_file), configurations.LONGEST_CACHE_FILE_NAME) + assert (py_file.parent / "__pycache__").is_dir() + else: + archive = tmp_path / "bundle.zip" + with zipfile.ZipFile(archive, "w") as zipped: + zipped.writestr("ipykernel_123/cell.py", "def cell():\n return 1\n") + py_file = archive / "ipykernel_123" / "cell.py" + elsewhere = str(tmp_path / "elsewhere" / "x") + remedy = configurations.cache_remedy(str(py_file), OSError(errno.EACCES, "Permission denied", elsewhere), "silence") + assert "IPYTHONDIR" not in remedy and "temporary" not in remedy, remedy + assert f"that location, {elsewhere}" in remedy and "(Permission denied)" in remedy, remedy + + +def test_the_windows_separator_skip_fails_the_test_on_any_other_platform(monkeypatch): + # The end-to-end .zip tests skip where the import met numba issue 10889, + # numba 0.68.0's on Windows alone; keyed on the remedy's text in stderr, + # the skip hid that remedy wrongly given on Linux, to a module shipped + # as .pyc alone for one, as a skip. The skip holds on Windows alone. + stderr = f"RuntimeWarning: numba cannot cache numbox here (...). {windows_path_sep_bug}, x, which no archive holds" + monkeypatch.setattr(os, "name", "posix") + with pytest.raises(AssertionError, match="the Windows separator remedy on posix"): + _skip_for_windows_separator_bug(stderr) + monkeypatch.setattr(os, "name", "nt") + with pytest.raises(pytest.skip.Exception): + _skip_for_windows_separator_bug(stderr) + _skip_for_windows_separator_bug("RuntimeWarning: numba cannot cache numbox here, another remedy") + + +def test_an_ipython_that_gives_its_directory_once_and_then_raises_is_not_read_as_temporary(tmp_path, monkeypatch): + # The remedy asks IPython for its cache directory twice to tell a new + # temporary directory from a steady one; an IPython that answers once and + # raises on the second call gave no second directory, and the remedy, + # reading the raise as a new answer, blamed a temporary directory. + import types + import numba + import numbox.core.configurations as configurations + + class UserCacheUnderTmp: + def __init__(self, appname, appauthor): + self.user_cache_dir = str(tmp_path / "user-cache" / "numba") + + monkeypatch.setattr(configurations, "AppDirs", UserCacheUnderTmp) + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + answers = [str(tmp_path / "ipython")] + + def once(): + if answers: + return answers.pop() + raise RuntimeError("IPython misbehaved") + + paths = types.ModuleType("IPython.paths") + paths.get_ipython_cache_dir = once + monkeypatch.setitem(sys.modules, "IPython", types.ModuleType("IPython")) + monkeypatch.setitem(sys.modules, "IPython.paths", paths) + archive = tmp_path / "bundle.zip" + with zipfile.ZipFile(archive, "w") as zipped: + zipped.writestr("ipykernel_123/cell.py", "def f():\n pass\n") + py_file = str(archive / "ipykernel_123" / "cell.py") + named = tmp_path / "ipython" / "numba_cache" + remedy = configurations.cache_remedy(py_file, OSError(errno.EACCES, "Permission denied", str(named)), "silence") + assert "temporary" not in remedy and "IPYTHONDIR" not in remedy, remedy + assert remedy == ( + f"numba caches this file in {named}, where no file can be written (Permission denied): make room there, or " + "make it writable; or silence"), remedy + + +@pytest.mark.parametrize("placement", ["on disk", "a .zip member"]) +def test_a_callers_error_with_no_errno_and_no_name_is_not_read_as_inspects(tmp_path, monkeypatch, placement): + # inspect's error for a source it cannot read has no errno and no name, + # and so can a caller's, OSError("Disk quota exceeded on /home"); the + # remedy read every such error for a file numba's IPython locator takes + # as inspect's and told the reader to make the file readable. inspect's + # are known by their words. + import types + import numba + import numbox.core.configurations as configurations + + class UserCacheUnderTmp: + def __init__(self, appname, appauthor): + self.user_cache_dir = str(tmp_path / "user-cache" / "numba") + + monkeypatch.setattr(configurations, "AppDirs", UserCacheUnderTmp) + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + paths = types.ModuleType("IPython.paths") + paths.get_ipython_cache_dir = lambda: str(tmp_path / "ipython") + monkeypatch.setitem(sys.modules, "IPython", types.ModuleType("IPython")) + monkeypatch.setitem(sys.modules, "IPython.paths", paths) + if placement == "on disk": + py_file = tmp_path / "ipykernel_123" / "cell.py" + py_file.parent.mkdir() + py_file.write_text("def f():\n pass\n") + else: + archive = tmp_path / "bundle.zip" + with zipfile.ZipFile(archive, "w") as zipped: + zipped.writestr("ipykernel_123/cell.py", "def f():\n pass\n") + py_file = archive / "ipykernel_123" / "cell.py" + failure = OSError("Disk quota exceeded on /home") + assert failure.errno is None and failure.filename is None + remedy = configurations.cache_remedy(str(py_file), failure, "silence") + assert "readable" not in remedy and "linecache" not in remedy, remedy + assert "the location numba took" in remedy and "Disk quota exceeded on /home" in remedy, remedy + + +def test_the_check_reads_the_lines_linecache_holds_for_a_file_not_on_disk(tmp_path, monkeypatch): + # A traceback through a .zip member, or inspect, leaves the member's lines + # in the process-wide linecache under its name, and inspect reads those + # for the probe before it asks the module's loader; the check gave the + # probe's source to inspect under that name instead for the time numba + # picked the locator, and owed what was there back. + import linecache + import types + import numba + import numbox.core.configurations as configurations + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + ipython_dir = tmp_path / "ipython" + paths = types.ModuleType("IPython.paths") + paths.get_ipython_cache_dir = lambda: str(ipython_dir) + monkeypatch.setitem(sys.modules, "IPython", types.ModuleType("IPython")) + monkeypatch.setitem(sys.modules, "IPython.paths", paths) + archive = tmp_path / "bundle.zip" + with zipfile.ZipFile(archive, "w") as zipped: + zipped.writestr("ipykernel_123/cell.py", "def cell():\n return 1\n") + py_file = str(archive / "ipykernel_123" / "cell.py") + held = (24, None, ["def cell():\n", " return 1\n"], py_file) + monkeypatch.setitem(linecache.cache, py_file, held) + seen = [] + real = configurations.CompileResultCacheImpl + + def recording(probe): + seen.append(linecache.getlines(py_file)) + return real(probe) + + monkeypatch.setattr(configurations, "CompileResultCacheImpl", recording) + configurations.check_cache_location(py_file, configurations.LONGEST_CACHE_FILE_NAME) + assert seen == [["def cell():\n", " return 1\n"]], seen + assert linecache.cache[py_file] is held + assert (ipython_dir / "numba_cache").is_dir() + + +@pytest.mark.parametrize("locators, made", [ + pytest.param("", "__pycache__", id="numba's order, the in-tree locator takes it"), + pytest.param("IPythonCacheLocator,InTreeCacheLocator", "numba_cache", id="the IPython locator first", + marks=pytest.mark.skipif(numba_version < 62, reason="NUMBA_CACHE_LOCATOR_CLASSES arrived in 0.62")), +]) +def test_a_cell_file_on_disk_is_asked_without_touching_linecache(tmp_path, monkeypatch, locators, made): + # inspect reads a file on disk itself, and asks the probe's module's + # loader, which reads a .zip member from the archive, for no file on disk. + # The probe's source, given to inspect for every file once, stood in for the + # real file's lines in the process-wide linecache while numba picked the + # locator, and a locator reading another function's source from the file + # meanwhile, as numba's IPython locator reads the function's own, read the + # probe's two lines instead. numba's own order tries the in-tree locator + # before the IPython one, which takes a cell file on disk only listed + # ahead of it. + import linecache + import types + import numba + import numbox.core.configurations as configurations + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", locators, raising=False) + ipython_dir = tmp_path / "ipython" + paths = types.ModuleType("IPython.paths") + paths.get_ipython_cache_dir = lambda: str(ipython_dir) + monkeypatch.setitem(sys.modules, "IPython", types.ModuleType("IPython")) + monkeypatch.setitem(sys.modules, "IPython.paths", paths) + py_file = tmp_path / "ipykernel_123" / "cell.py" + py_file.parent.mkdir() + py_file.write_text("def cell():\n return 1\n") + lines = linecache.getlines(str(py_file)) + assert lines == ["def cell():\n", " return 1\n"] + seen = [] + real = configurations.CompileResultCacheImpl + + def recording(probe): + seen.append(linecache.getlines(str(py_file))) + return real(probe) + + monkeypatch.setattr(configurations, "CompileResultCacheImpl", recording) + configurations.check_cache_location(str(py_file), configurations.LONGEST_CACHE_FILE_NAME) + assert seen == [lines] and linecache.getlines(str(py_file)) == lines, seen + location = py_file.parent / "__pycache__" if made == "__pycache__" else ipython_dir / "numba_cache" + assert location.is_dir() + + +def test_two_checks_of_a_file_not_on_disk_overlap_and_each_reads_the_members_source(tmp_path, monkeypatch): + # The probe's source was given under a lock, one check at a time, so that + # each check put back what it had found in the process-wide linecache, + # and a second check waited at the lock while the first held it. Each + # probe reads the member's source through a module of its own now, and + # leaves the member's lines, so there is nothing to put back and nothing + # to wait for. + import inspect + import linecache + import threading + import types + import numba + import numbox.core.configurations as configurations + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + ipython_dir = tmp_path / "ipython" + paths = types.ModuleType("IPython.paths") + paths.get_ipython_cache_dir = lambda: str(ipython_dir) + monkeypatch.setitem(sys.modules, "IPython", types.ModuleType("IPython")) + monkeypatch.setitem(sys.modules, "IPython.paths", paths) + with zipfile.ZipFile(tmp_path / "bundle.zip", "w") as zipped: + zipped.writestr("ipykernel_123/cell.py", "def cell():\n pass\n") + py_file = str(tmp_path / "bundle.zip" / "ipykernel_123" / "cell.py") + linecache.cache.pop(py_file, None) + real = configurations.CompileResultCacheImpl + inside = {"first": threading.Event(), "second": threading.Event()} + hold = threading.Event() + seen = {} + + def holding(probe): + name = threading.current_thread().name + seen[name] = inspect.getsource(probe) + inside[name].set() + if name == "first": + hold.wait(5) + return real(probe) + + monkeypatch.setattr(configurations, "CompileResultCacheImpl", holding) + checks = [threading.Thread(target=configurations.check_cache_location, args=(py_file,), name=name) + for name in ("first", "second")] + checks[0].start() + assert inside["first"].wait(5) + checks[1].start() + # The second check goes in while the first is still there. + assert inside["second"].wait(5) + hold.set() + for check in checks: + check.join(5) + assert seen == {"first": "def cell():\n pass\n", "second": "def cell():\n pass\n"}, seen + assert linecache.getlines(py_file) == ["def cell():\n", " pass\n"] + assert not [name for name in sys.modules if name.startswith("_numbox_cache_probe_")] + linecache.cache.pop(py_file, None) + + +@pytest.mark.parametrize("refused, told", [ + pytest.param(("cell-1.py312.nbi.tmp.0123456789abcdef", errno.ENAMETOOLONG, "File name too long"), + "the path is too long for the file system: IPython's cache directory, {ipython_dir}, at a shorter path", + id="too long, a file in it"), + pytest.param(("", errno.EACCES, "Permission denied"), + "numba caches this file in {numba_cache}, where no file can be written (Permission denied): make room " + "there, or make it writable", id="unwritable, the location"), +]) +def test_a_zip_member_in_an_ipykernel_directory_refused_ipythons_location_is_told_ipythons_cache_directory( + tmp_path, monkeypatch, refused, told): + # numba's IPython locator takes a member of a .zip in an ipykernel directory + # before the .zip locator is reached, and caches it in numba_cache under + # IPython's cache directory, which the error names, or a file numba writes + # in it. The remedy, taking every OSError for a source not on disk as the + # .zip's, told that location as the user's cache directory, with what moves + # that directory, where it is IPython's and the remedy is the one a cell + # file on disk gets there, with no NUMBA_CACHE_DIR to offer for a source + # not on disk. + import types + import numba + import numbox.core.configurations as configurations + + class UserCacheUnderTmp: + def __init__(self, appname, appauthor): + self.user_cache_dir = str(tmp_path / "user-cache" / "numba") + + monkeypatch.setattr(configurations, "AppDirs", UserCacheUnderTmp) + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + ipython_dir = tmp_path / "ipython" + paths = types.ModuleType("IPython.paths") + paths.get_ipython_cache_dir = lambda: str(ipython_dir) + monkeypatch.setitem(sys.modules, "IPython", types.ModuleType("IPython")) + monkeypatch.setitem(sys.modules, "IPython.paths", paths) + py_file = str(tmp_path / "bundle.zip" / "ipykernel_123" / "cell.py") + name, number, strerror = refused + named = ipython_dir / "numba_cache" / name if name else ipython_dir / "numba_cache" + remedy = configurations.cache_remedy(py_file, OSError(number, strerror, str(named)), "silence") + told = told.format(ipython_dir=ipython_dir, numba_cache=ipython_dir / "numba_cache") + assert remedy == f"{told}; or silence", remedy + + +def test_a_zip_members_error_is_answered_without_asking_ipython_for_its_cache_directory(tmp_path, monkeypatch): + # numba's IPython locator takes a file in an ipykernel directory alone, so + # a .zip member anywhere else is never cached in IPython's cache directory; + # the remedy asked IPython for that directory all the same, to compare the + # error's path against it, and IPython, with its own directory unwritable, + # under a read-only home, warned on every import of the package and left a + # temporary directory behind. The directory is asked for where numba's + # IPython locator takes the file and is listed. + import types + import numba + import numbox.core.configurations as configurations + from numba.core.caching import _CacheLocator + + class UserCacheUnderTmp: + def __init__(self, appname, appauthor): + self.user_cache_dir = str(tmp_path / "user-cache" / "numba") + + def never(): + raise AssertionError("IPython was asked for its cache directory") + + monkeypatch.setattr(configurations, "AppDirs", UserCacheUnderTmp) + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + paths = types.ModuleType("IPython.paths") + paths.get_ipython_cache_dir = never + monkeypatch.setitem(sys.modules, "IPython", types.ModuleType("IPython")) + monkeypatch.setitem(sys.modules, "IPython.paths", paths) + py_file = str(tmp_path / "bundle.zip" / "package" / "module.py") + location = tmp_path / "user-cache" / "numba" / _CacheLocator.get_suitable_cache_subpath(py_file) + # The location is there and refuses, as a directory made can; one not + # there is told the directory it cannot be made under as well. + location.mkdir(parents=True) + remedy = configurations.cache_remedy(py_file, OSError(errno.EACCES, "Permission denied", str(location)), "silence") + if numba_version < 61: + # numba's .zip locator arrived in 0.61; before it, no locator takes a + # .zip member, and the error gets the remedy numba's no-locator one does. + assert remedy == f"{ARCHIVE_REMEDY}; or silence", remedy + return + assert remedy == ( + "numba caches a .zip, or a frozen application, in the user's cache directory, and NUMBA_CACHE_DIR has no " + f"effect here, because the source is not a file on disk: no file can be written in that directory, {location} " + "(Permission denied), so make room there, or make it writable, or silence"), remedy + + +@pytest.mark.parametrize("platform, moved_through", [ + ("linux", ", through XDG_CACHE_HOME or HOME"), ("darwin", ", through HOME"), ("win32", ""), +]) +def test_the_user_cache_directory_is_told_what_moves_it_on_the_platform(tmp_path, monkeypatch, platform, moved_through): + # numba's appdirs puts the user's cache directory under XDG_CACHE_HOME or + # HOME on Linux, under HOME alone on macOS, and asks the system on Windows, + # so "through XDG_CACHE_HOME or HOME" sent a macOS reader to a variable that + # moves nothing, for a source on disk and for a .zip alike. + import numba + import numbox.core.configurations as configurations + from numba.core.caching import _CacheLocator + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + py_file = tmp_path / "package" / "module.py" + py_file.parent.mkdir() + py_file.write_text("") + user_cache_dir = configurations.AppDirs(appname="numba", appauthor=False).user_cache_dir + location = os.path.join(user_cache_dir, _CacheLocator.get_suitable_cache_subpath(str(py_file))) + monkeypatch.setattr(sys, "platform", platform) + on_disk = configurations.cache_remedy( + str(py_file), OSError(errno.ENAMETOOLONG, "File name too long", location), "silence") + assert f"the user's cache directory, {user_cache_dir}, at a shorter path{moved_through}, or NUMBA_CACHE_DIR" in ( + on_disk), on_disk + if numba_version < 61: + # numba's .zip locator arrived in 0.61; before it, a .zip member has + # no location of numba's under the user's cache directory. + return + zip_member = str(tmp_path / "package.zip" / "package" / "module.py") + zip_location = os.path.join(user_cache_dir, _CacheLocator.get_suitable_cache_subpath(zip_member)) + zipped = configurations.cache_remedy( + zip_member, OSError(errno.ENAMETOOLONG, "File name too long", zip_location), "silence") + assert f"put that directory, {zip_location}, at a shorter path{moved_through}, or silence" in zipped, zipped + + +@pytest.mark.parametrize("locators, offered", [ + ("InTreeCacheLocator,UserWideCacheLocator", False), + ("numba.core.caching.UserProvidedCacheLocator, numba.core.caching.InTreeCacheLocator", True), + ("IPythonCacheLocator,UserProvidedCacheLocator,InTreeCacheLocator,UserWideCacheLocator", True), + ("InTreeCacheLocator,UserProvidedCacheLocator,UserWideCacheLocator", False), + pytest.param( + "UserProvidedCacheLocator,InTreeCacheLocatorFsAgnostic,UserWideCacheLocator", True, + marks=pytest.mark.skipif(numba_version < 62, reason="InTreeCacheLocatorFsAgnostic arrived in numba 0.62")), + pytest.param( + "InTreeCacheLocatorFsAgnostic", False, + marks=pytest.mark.skipif(numba_version < 62, reason="InTreeCacheLocatorFsAgnostic arrived in numba 0.62")), +]) +def test_numba_cache_dir_is_offered_only_where_numba_tries_it_before_the_location_it_took( + tmp_path, monkeypatch, locators, offered): + # numba 0.62 and later take NUMBA_CACHE_LOCATOR_CLASSES as the locators and + # their order, each entry a class of numba's caching module by its name or + # its dotted path. NUMBA_CACHE_DIR is an alternative to the __pycache__ + # numba took only where numba tries the variable before that: a set one + # was passed over, an unset one would be taken. The IPython and .zip + # locators take no plain file on disk, so one of them ahead changes + # nothing, where the first entry's name alone said another locator was + # first. numba's InTreeCacheLocatorFsAgnostic caches in the __pycache__ as + # its parent does, and keyed by name alone it was no in-tree locator, so + # its too-long __pycache__ got "that location" and no cure. + import numba + from numbox.core.configurations import cache_remedy + monkeypatch.setattr(numba.config, "CACHE_DIR", str(tmp_path / "cache")) + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", locators, raising=False) + py_file = tmp_path / "package" / "module.py" + py_file.parent.mkdir() + py_file.write_text("") + failure = OSError(errno.ENAMETOOLONG, "File name too long", str(py_file.parent / "__pycache__")) + remedy = cache_remedy(str(py_file), failure, "silence", package="numbduck") + instead = (f", or NUMBA_CACHE_DIR, which is set to {tmp_path / 'cache'} and numba could not use, made a writable " + "directory at a short path") if offered else "" + assert remedy == f"the path is too long for the file system: numbduck installed at a shorter path{instead}; or silence", ( + remedy) + + +# Each case: whether NUMBA_CACHE_DIR is set, NUMBA_CACHE_LOCATOR_CLASSES, and +# the remedy before "or silence". +NO_LOCATOR_CASES = { + "unset": (False, "", "Set NUMBA_CACHE_DIR to a writable directory"), + "set, passed over": ( + True, "", + "NUMBA_CACHE_DIR is set to {cache_dir}, which numba could not use: set it to a writable directory at a short " + "path"), + "without the user-provided locator": ( + False, "InTreeCacheLocator,UserWideCacheLocator", + "numba looks only where NUMBA_CACHE_LOCATOR_CLASSES, {locators}, says: make one of those locations writable"), + "unset, the user-provided locator after another": ( + False, "InTreeCacheLocator,UserProvidedCacheLocator", "Set NUMBA_CACHE_DIR to a writable directory"), + "set, passed over, the user-provided locator after another": ( + True, "InTreeCacheLocator,UserProvidedCacheLocator", + "NUMBA_CACHE_DIR is set to {cache_dir}, which numba could not use: set it to a writable directory at a short " + "path"), + "only locators that take no plain file on disk": ( + False, "IPythonCacheLocator,ZipCacheLocator", + "numba looks only where NUMBA_CACHE_LOCATOR_CLASSES, {locators}, says, and none of those locators takes a " + "source file on disk: list UserProvidedCacheLocator, with NUMBA_CACHE_DIR set, InTreeCacheLocator or " + "UserWideCacheLocator"), +} + + +@pytest.mark.parametrize("setting", list(NO_LOCATOR_CASES)) +def test_no_locator_for_a_source_on_disk_is_told_what_numba_would_take(tmp_path, monkeypatch, setting): + # numba raises "no locator available" where it passed every location over. + # "Set NUMBA_CACHE_DIR to a writable directory" was the answer whatever the + # setting: to a NUMBA_CACHE_DIR too deep for numba to make its directory in, + # where a writable one as deep cures nothing, and to a locator list numba + # 0.62 and later read from NUMBA_CACHE_LOCATOR_CLASSES without the + # user-provided locator, where numba never reads the variable. With that + # locator anywhere in the list numba reads the variable once the locators + # before it have passed, so its place in the list changes nothing here. A + # list of numba's IPython and .zip locators alone has no locator for a + # plain file on disk, and "make one of those locations writable" asked for + # what no listing can give. + import numba + from numbox.core.configurations import cache_remedy + py_file = tmp_path / "package" / "module.py" + py_file.parent.mkdir() + py_file.write_text("") + cache_dir = str(tmp_path / ("d" * 200)) + cache_dir_set, locators, opening = NO_LOCATOR_CASES[setting] + monkeypatch.setattr(numba.config, "CACHE_DIR", cache_dir if cache_dir_set else "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", locators, raising=False) + failure = RuntimeError(f"cannot cache function '_cache_probe': no locator available for file '{py_file}'") + remedy = cache_remedy(str(py_file), failure, "silence") + assert remedy == f"{opening.format(cache_dir=cache_dir, locators=locators)}, or silence", remedy + + +@pytest.mark.skipif(numba_version < 62, reason="NUMBA_CACHE_LOCATOR_CLASSES arrived in numba 0.62") +def test_a_location_too_long_under_the_zip_locator_listed_first_is_the_users_cache_directorys(tmp_path, monkeypatch): + # numba's .zip locator takes any path with ".zip" in it before every + # locator after it, and caches one with a part ending in ".zip", a + # directory of that name included, under the user's cache directory. + # Listed first it leaves + # NUMBA_CACHE_DIR unread for such a path, where the remedy, holding that + # locator to take no file on disk, left it out of the order and offered + # the variable. + import numba + import numbox.core.configurations as configurations + from numba.core.caching import _CacheLocator + + class UserCacheUnderTmp: + def __init__(self, appname, appauthor): + self.user_cache_dir = str(tmp_path / "user-cache" / "numba") + + monkeypatch.setattr(configurations, "AppDirs", UserCacheUnderTmp) + monkeypatch.setattr(numba.config, "CACHE_DIR", str(tmp_path / "cache")) + monkeypatch.setattr( + numba.config, "CACHE_LOCATOR_CLASSES", + "ZipCacheLocator,UserProvidedCacheLocator,InTreeCacheLocator,UserWideCacheLocator", raising=False) + py_file = tmp_path / "bundle.zip" / "package" / "module.py" + py_file.parent.mkdir(parents=True) + py_file.write_text("") + user_cache_dir = tmp_path / "user-cache" / "numba" + location = user_cache_dir / _CacheLocator.get_suitable_cache_subpath(str(py_file)) + failure = OSError(errno.ENAMETOOLONG, "File name too long", str(location)) + remedy = configurations.cache_remedy(str(py_file), failure, "silence") + assert remedy == ( + f"the path is too long for the file system: the user's cache directory, {user_cache_dir}, at a shorter path" + f"{configurations._moved_through()}; or silence"), remedy + + +from_061 = pytest.mark.skipif(numba_version < 61, reason="numba's .zip locator arrived in 0.61") +from_062 = pytest.mark.skipif(numba_version < 62, reason="NUMBA_CACHE_LOCATOR_CLASSES arrived in numba 0.62") + + +# Each case: whether NUMBA_CACHE_DIR is set, NUMBA_CACHE_LOCATOR_CLASSES, and +# the remedy before "or silence". +@pytest.mark.parametrize("cache_dir_set, locators, expected", [ + (False, "", "Set NUMBA_CACHE_DIR to a writable directory"), + pytest.param( + False, "ZipCacheLocator,InTreeCacheLocator,UserWideCacheLocator", + 'numba\'s .zip locator, which NUMBA_CACHE_LOCATOR_CLASSES puts before every locator for a source file on disk, ' + 'takes this file for the ".zip" in its path and finds no archive there: list UserProvidedCacheLocator, with ' + "NUMBA_CACHE_DIR set, InTreeCacheLocator or UserWideCacheLocator before it", + marks=from_062), + pytest.param( + True, "InTreeCacheLocator,ZipCacheLocator,UserProvidedCacheLocator", + 'numba\'s .zip locator takes this file for the ".zip" in its path and finds no archive there, after every ' + "locator NUMBA_CACHE_LOCATOR_CLASSES, {locators}, puts before it passed the file over: make one of those " + "locations writable, or list UserProvidedCacheLocator, with NUMBA_CACHE_DIR set, InTreeCacheLocator or " + "UserWideCacheLocator before it", + marks=from_062), + pytest.param( + True, "UserProvidedCacheLocator,ZipCacheLocator,InTreeCacheLocator", + "NUMBA_CACHE_DIR is set to {cache_dir}, which numba could not use: set it to a writable directory at a short " + "path", + marks=from_062), + pytest.param( + False, "UserProvidedCacheLocator,ZipCacheLocator,InTreeCacheLocator", + "Set NUMBA_CACHE_DIR to a writable directory", marks=from_062), +]) +def test_a_source_on_disk_with_zip_in_its_path_and_no_archive_is_told_by_the_zip_locators_place( + tmp_path, monkeypatch, cache_dir_set, locators, expected): + # numba's .zip locator takes a path with ".zip" in it and raises ValueError + # where no part of the path ends in .zip. numba tries the locators before + # it and none after, so the error means the ones before it passed the file + # over: in numba's own order that is every locator for a file on disk, and + # the no-locator remedy applies; listed first it takes the file first, and + # the remedy offered NUMBA_CACHE_DIR, which numba never reaches; listed + # after one locator for a file on disk and before the user-provided one, + # it raised before numba read NUMBA_CACHE_DIR, and the remedy, answering + # the error only where the .zip locator came first, said the variable, set + # and writable, was one numba could not use. The user-provided locator + # takes a file only with NUMBA_CACHE_DIR set, so listing it is asked for + # with the variable set, as the no-locator remedy asks; without that, the + # reader who listed it first saw the same error again. + import numba + from numbox.core.configurations import cache_remedy + cache_dir = str(tmp_path / "cache") + monkeypatch.setattr(numba.config, "CACHE_DIR", cache_dir if cache_dir_set else "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", locators, raising=False) + py_file = tmp_path / "not.zipped" / "module.py" + py_file.parent.mkdir() + py_file.write_text("") + remedy = cache_remedy(str(py_file), ValueError("No zip file found in path"), "silence") + assert remedy == f"{expected.format(cache_dir=cache_dir, locators=locators)}, or silence", remedy + + +@from_062 +@pytest.mark.parametrize("failure", [ + pytest.param(OSError(errno.ENAMETOOLONG, "File name too long"), id="too long, unnamed"), + pytest.param(OSError(errno.EACCES, "Permission denied"), id="unwritable, unnamed"), + pytest.param(OSError(errno.ENAMETOOLONG, "File name too long", "/elsewhere/cache"), id="too long, named"), +]) +def test_an_error_under_a_locator_list_with_no_locator_that_takes_the_file_is_told_the_list( + tmp_path, monkeypatch, failure): + # A list with no locator that takes a source file on disk leaves numba no + # location for it, and numba's error is the no-locator one, which is told + # the list. An OSError a caller passes on under such a list was answered + # as if numba had taken a location: unnamed, "the location numba took" + # with no location to list after it, and named, that location at a + # shorter path, though the list keeps numba from caching the file at all. + import numba + from numbox.core.configurations import cache_remedy + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "IPythonCacheLocator,ZipCacheLocator", raising=False) + py_file = tmp_path / "site" / "package" / "module.py" + py_file.parent.mkdir(parents=True) + py_file.write_text("") + assert cache_remedy(str(py_file), failure, "silence") == ( + "numba looks only where NUMBA_CACHE_LOCATOR_CLASSES, IPythonCacheLocator,ZipCacheLocator, says, and none of " + "those locators takes a source file on disk: list UserProvidedCacheLocator, with NUMBA_CACHE_DIR set, " + "InTreeCacheLocator or UserWideCacheLocator, or silence"), cache_remedy(str(py_file), failure, "silence") + + +@from_062 +@pytest.mark.parametrize("failure", [ + pytest.param(OSError(errno.ENAMETOOLONG, "File name too long"), id="too long, unnamed"), + pytest.param(OSError(errno.EACCES, "Permission denied"), id="unwritable, unnamed"), + pytest.param(OSError(errno.ENAMETOOLONG, "File name too long", "/elsewhere/cache"), id="too long, named"), +]) +def test_an_error_under_the_user_provided_locator_alone_with_numba_cache_dir_unset_is_told_to_set_it( + tmp_path, monkeypatch, failure): + # numba's user-provided locator takes a file only with NUMBA_CACHE_DIR set, + # so a list of that locator alone leaves numba no locator while the + # variable is unset, and numba's error is the no-locator one, told to set + # the variable. An OSError a caller passes on under that list counted the + # locator as taking the file, and was told "the location numba took" with + # no location after it, or the location it named at a shorter path, where + # numba took none. + import numba + from numbox.core.configurations import cache_remedy + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "UserProvidedCacheLocator", raising=False) + py_file = tmp_path / "site" / "package" / "module.py" + py_file.parent.mkdir(parents=True) + py_file.write_text("") + no_locator = RuntimeError(f"cannot cache function '_cache_probe': no locator available for file '{py_file}'") + told = cache_remedy(str(py_file), no_locator, "silence") + assert told == "Set NUMBA_CACHE_DIR to a writable directory, or silence", told + assert cache_remedy(str(py_file), failure, "silence") == told, cache_remedy(str(py_file), failure, "silence") + + +ON_DISK = ("install numbox with its source files on disk, unpacked from any archive") +LIST_ON_DISK = (", and list UserProvidedCacheLocator, with NUMBA_CACHE_DIR set, InTreeCacheLocator or UserWideCacheLocator " + "in NUMBA_CACHE_LOCATOR_CLASSES, {locators}, which has none of them") +A_ZIP = ("import it from a .zip holding its source files, which numba 0.61 and later cache in the user's cache directory") +LIST_A_ZIP = " through ZipCacheLocator, once NUMBA_CACHE_LOCATOR_CLASSES, {locators}, lists it" +SET_THE_VARIABLE = (", with NUMBA_CACHE_DIR set, which UserProvidedCacheLocator, the one locator for a file on disk in " + "NUMBA_CACHE_LOCATOR_CLASSES, {locators}, takes nothing without") + + +@from_062 +@pytest.mark.parametrize("locators, cache_dir_set, on_disk, a_zip", [ + pytest.param("", False, ON_DISK, A_ZIP, id="numba's order"), + pytest.param("UserProvidedCacheLocator,InTreeCacheLocator,UserWideCacheLocator", False, ON_DISK, + A_ZIP + LIST_A_ZIP, id="no .zip locator"), + pytest.param("ZipCacheLocator", False, ON_DISK + LIST_ON_DISK, A_ZIP, id="no locator for a file on disk"), + pytest.param("IPythonCacheLocator", False, ON_DISK + LIST_ON_DISK, A_ZIP + LIST_A_ZIP, id="neither"), + pytest.param("UserProvidedCacheLocator,ZipCacheLocator", False, ON_DISK + SET_THE_VARIABLE, A_ZIP, + id="the user-provided locator alone for a file on disk, NUMBA_CACHE_DIR unset"), + pytest.param("UserProvidedCacheLocator,ZipCacheLocator", True, ON_DISK, A_ZIP, + id="the user-provided locator alone for a file on disk, NUMBA_CACHE_DIR set"), + pytest.param("InTreeCacheLocator,ZipCacheLocator", False, ON_DISK, A_ZIP, + id="the in-tree locator alone for a file on disk"), + pytest.param("UserWideCacheLocator,ZipCacheLocator", False, ON_DISK, A_ZIP, + id="the user-wide locator alone for a file on disk"), +]) +def test_an_import_from_an_egg_under_a_locator_list_is_offered_each_way_with_the_locator_it_needs( + tmp_path, monkeypatch, locators, cache_dir_set, on_disk, a_zip): + # The remedy for an archive numba does not cache offers the source files + # on disk, which the three locators for a file on disk take, or a .zip + # holding them, which the .zip locator alone takes. It offered both under + # any list, so under one without the .zip locator it offered a .zip numba + # then had no locator for, as under one without a locator for a file on + # disk it offered an install numba would not cache either; and the + # user-provided locator, alone of the three in the list, takes nothing + # without NUMBA_CACHE_DIR, so unset it is asked for with the install, + # where the in-tree or the user-wide one alone takes a file on disk as + # it is, and the install is offered as it is. + import numba + from numbox.core.configurations import cache_remedy + monkeypatch.setattr(numba.config, "CACHE_DIR", str(tmp_path / "cache") if cache_dir_set else "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", locators, raising=False) + py_file = tmp_path / "numbox-0.0.0-py3.12.egg" / "numbox" / "module.py" + failure = RuntimeError(f"cannot cache function '_cache_probe': no locator available for file '{py_file}'") + remedy = cache_remedy(str(py_file), failure, "silence") + assert remedy == ( + "NUMBA_CACHE_DIR has no effect here, because the source is not a file on disk: to cache, " + f"{on_disk.format(locators=locators)}, or {a_zip.format(locators=locators)}; or silence"), remedy + + +@pytest.mark.skipif(numba_version < 62, reason="NUMBA_CACHE_LOCATOR_CLASSES arrived in numba 0.62") +@pytest.mark.parametrize("locators, expected", [ + ("", "NUMBA_CACHE_DIR has no effect here, because the source is not a file on disk: to cache, install numbox with " + "its source files on disk, unpacked from any archive, or import it from a .zip holding its source files, " + "which numba 0.61 and later cache in the user's cache directory"), + ("UserProvidedCacheLocator,InTreeCacheLocator,UserWideCacheLocator", + "numba caches a .zip through ZipCacheLocator alone, and NUMBA_CACHE_LOCATOR_CLASSES, {locators}, leaves it out: " + "list it"), +]) +def test_a_zip_import_under_a_locator_list_without_the_zip_locator_is_told_to_list_it( + tmp_path, monkeypatch, locators, expected): + # numba caches a .zip through its .zip locator alone, so a list that leaves + # that locator out gives numba no locator for a source in a .zip, and the + # error is the no-locator one. The archive remedy, which reads the list for + # a source on disk only, told the reader to import from a .zip holding the + # source files, which they had done, and never named the list. + import numba + from numbox.core.configurations import cache_remedy + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", locators, raising=False) + py_file = str(tmp_path / "bundle.zip" / "numbox" / "module.py") + failure = RuntimeError(f"cannot cache function '_cache_probe': no locator available for file '{py_file}'") + remedy = cache_remedy(py_file, failure, "silence") + assert remedy == f"{expected.format(locators=locators)}; or silence", remedy + + +@pytest.mark.skipif(numba_version < 62, reason="NUMBA_CACHE_LOCATOR_CLASSES arrived in numba 0.62") +@pytest.mark.parametrize("locators, expected", [ + ("", "numba caches a .zip, or a frozen application, in the user's cache directory, and NUMBA_CACHE_DIR has no " + "effect here, because the source is not a file on disk: numba could not use that directory, {user_cache_dir}: " + "make it writable, or put it at a shorter path{moved_through}"), + ("UserProvidedCacheLocator,InTreeCacheLocator", + "numba caches a frozen application through UserWideCacheLocator alone, and NUMBA_CACHE_LOCATOR_CLASSES, " + "{locators}, leaves it out: list it"), +]) +def test_a_frozen_application_under_a_locator_list_without_the_user_wide_locator_is_told_to_list_it( + tmp_path, monkeypatch, locators, expected): + # numba's user-wide locator alone takes a frozen application, whose source + # is not on disk, so a list that leaves it out gives numba no locator and + # the error is the no-locator one. The frozen remedy, reading no list, + # asked for the user's cache directory to be made writable, which it was + # and which numba never tried. + import numba + import numbox.core.configurations as configurations + + class UserCacheUnderTmp: + def __init__(self, appname, appauthor): + self.user_cache_dir = str(tmp_path / "user-cache" / "numba") + + monkeypatch.setattr(configurations, "AppDirs", UserCacheUnderTmp) + monkeypatch.setattr(sys, "frozen", True, raising=False) + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", locators, raising=False) + py_file = str(tmp_path / "frozen" / "numbox" / "module.py") + failure = RuntimeError(f"cannot cache function '_cache_probe': no locator available for file '{py_file}'") + remedy = configurations.cache_remedy(py_file, failure, "silence") + expected = expected.format(locators=locators, user_cache_dir=tmp_path / "user-cache" / "numba", + moved_through=configurations._moved_through()) + assert remedy == f"{expected}, or silence", remedy + + +NONE_OF_NUMBAS_OFF_DISK = { + "too long": ((errno.ENAMETOOLONG, "File name too long"), + "the path is too long for the file system, so put that location, {named}, at a shorter path"), + "unwritable": ((errno.EACCES, "Permission denied"), + "no file can be written at that location, {named} (Permission denied), so make room there, or make it " + "writable"), +} + + +@pytest.mark.parametrize("locators, in_numba_cache, refused", [ + pytest.param("ZipCacheLocator,IPythonCacheLocator", "cell-1.py312.nbi", "too long", marks=from_062, + id="the .zip locator first, a file in numba_cache, too long"), + pytest.param("ZipCacheLocator,IPythonCacheLocator", "", "unwritable", marks=from_062, + id="the .zip locator first, numba_cache, unwritable"), + pytest.param("", "subdir/cell-1.py312.nbi", "too long", id="numba's order, a directory numba never makes in it"), +]) +def test_an_error_naming_ipythons_location_numbas_ipython_locator_did_not_take_is_told_the_location_as_named( + tmp_path, monkeypatch, locators, in_numba_cache, refused): + # numba's .zip locator takes a file without trying its location, so a + # list with it before the IPython one leaves that one unreached for a + # .zip member in an ipykernel directory, and numba caches the member in + # the user's cache directory; the remedy answered an error a caller passes + # on naming IPython's numba_cache with IPython's cache directory all the + # same, a directory numba never used. And numba's IPython locator writes + # its files in numba_cache itself, with no directory under it, so a path + # under one is none of numba's, and was told as the .zip's directory to + # put at a shorter path through what moves the user's cache directory, + # which moves nothing of it. A location that is none of numba's is told + # as the error names it, as for a source on disk, and with nothing of + # where numba caches the file, which it is none of. + import types + import numba + import numbox.core.configurations as configurations + + class UserCacheUnderTmp: + def __init__(self, appname, appauthor): + self.user_cache_dir = str(tmp_path / "user-cache" / "numba") + + monkeypatch.setattr(configurations, "AppDirs", UserCacheUnderTmp) + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", locators, raising=False) + ipython_dir = tmp_path / "ipython" + paths = types.ModuleType("IPython.paths") + paths.get_ipython_cache_dir = lambda: str(ipython_dir) + monkeypatch.setitem(sys.modules, "IPython", types.ModuleType("IPython")) + monkeypatch.setitem(sys.modules, "IPython.paths", paths) + py_file = str(tmp_path / "bundle.zip" / "ipykernel_123" / "cell.py") + named = ipython_dir / "numba_cache" / in_numba_cache if in_numba_cache else ipython_dir / "numba_cache" + error, told = NONE_OF_NUMBAS_OFF_DISK[refused] + remedy = configurations.cache_remedy(py_file, OSError(*error, str(named)), "silence") + assert remedy == ( + "NUMBA_CACHE_DIR has no effect here, because the source is not a file on disk: " + f"{told.format(named=named)}, or silence"), remedy + + +@from_062 +def test_a_zip_members_error_naming_the_zips_location_after_ipythons_locator_passed_it_over_is_told_the_location( + tmp_path, monkeypatch): + # numba's IPython locator passes a location it cannot make or write over, + # and the .zip locator listed after it then takes the member and caches + # it in the user's cache directory: an error naming that location is the + # .zip's, under numba's own order and under one that lists the IPython + # locator first alike. + import types + import numba + import numbox.core.configurations as configurations + from numba.core.caching import _CacheLocator + + class UserCacheUnderTmp: + def __init__(self, appname, appauthor): + self.user_cache_dir = str(tmp_path / "user-cache" / "numba") + + monkeypatch.setattr(configurations, "AppDirs", UserCacheUnderTmp) + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "IPythonCacheLocator,ZipCacheLocator", raising=False) + ipython_dir = tmp_path / "ipython" + paths = types.ModuleType("IPython.paths") + paths.get_ipython_cache_dir = lambda: str(ipython_dir) + monkeypatch.setitem(sys.modules, "IPython", types.ModuleType("IPython")) + monkeypatch.setitem(sys.modules, "IPython.paths", paths) + py_file = str(tmp_path / "bundle.zip" / "ipykernel_123" / "cell.py") + location = tmp_path / "user-cache" / "numba" / _CacheLocator.get_suitable_cache_subpath(py_file) + remedy = configurations.cache_remedy( + py_file, OSError(errno.EACCES, "Permission denied", str(location / "cell-1.py312.nbi.tmp.0123456789abcdef")), + "silence") + assert remedy == ( + "numba caches a .zip, or a frozen application, in the user's cache directory, and NUMBA_CACHE_DIR has no " + f"effect here, because the source is not a file on disk: no file can be written in that directory, {location} " + "(Permission denied), so make room there, or make it writable, or silence"), remedy + + +@pytest.mark.parametrize("locators, member", [ + pytest.param("", "bundle.zip/package/module.py", id="numba's order, a .zip member in no ipykernel directory", + marks=from_061), + pytest.param("ZipCacheLocator", "bundle.zip/ipykernel_123/cell.py", id="a list without the IPython locator", + marks=from_062), + pytest.param("ZipCacheLocator,IPythonCacheLocator", "bundle.zip/ipykernel_123/cell.py", + id="the .zip locator listed first, which takes the member before the IPython one is reached", + marks=from_062), +]) +@on_windows_under_numba_0_68_0 +def test_a_file_not_on_disk_numbas_ipython_locator_does_not_read_is_asked_without_touching_linecache( + tmp_path, monkeypatch, locators, member): + # numba's IPython locator reads the function's source when it takes the + # file, through the probe's module's loader for a .zip member, which + # leaves the member's lines in the process-wide linecache; the probe's own + # source stood there instead, for every file not on disk, while numba + # picked the locator, and a reader in another thread, formatting a + # traceback through that file meanwhile, got the probe's two lines for it, + # for a .zip member in no ipykernel directory too, which that locator + # never takes. The linecache is untouched for a file no locator of that + # family numba reaches takes. + import linecache + import types + import numba + import numba.core.caching + import numbox.core.configurations as configurations + + class UserCacheUnderTmp: + def __init__(self, appname, appauthor): + self.user_cache_dir = str(tmp_path / "user-cache" / "numba") + + monkeypatch.setattr(numba.core.caching, "AppDirs", UserCacheUnderTmp) + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", locators, raising=False) + ipython_dir = tmp_path / "ipython" + paths = types.ModuleType("IPython.paths") + paths.get_ipython_cache_dir = lambda: str(ipython_dir) + monkeypatch.setitem(sys.modules, "IPython", types.ModuleType("IPython")) + monkeypatch.setitem(sys.modules, "IPython.paths", paths) + archive = tmp_path / "bundle.zip" + with zipfile.ZipFile(archive, "w") as zipped: + zipped.writestr(member.split("/", 1)[1], "def f():\n pass\n") + py_file = str(tmp_path / member) + seen = [] + real = configurations.CompileResultCacheImpl + + def recording(probe): + seen.append((py_file in linecache.cache, linecache.getlines(py_file))) + return real(probe) + + monkeypatch.setattr(configurations, "CompileResultCacheImpl", recording) + configurations.check_cache_location(py_file, configurations.LONGEST_CACHE_FILE_NAME) + assert seen == [(False, [])], seen + assert py_file not in linecache.cache + assert (tmp_path / "user-cache" / "numba").is_dir() + assert not ipython_dir.exists() + + +@from_062 +@pytest.mark.parametrize("refused, told", [ + pytest.param(("__pycache__", errno.ENAMETOOLONG, "File name too long"), + "the path is too long for the file system: that location, {named}, at a shorter path", + id="too long, beside the source"), + pytest.param(("__pycache__", errno.EACCES, "Permission denied"), + "that location, {named}, where no file can be written (Permission denied): make room there, or make it " + "writable", id="unwritable, beside the source"), + pytest.param(("", errno.ENAMETOOLONG, "File name too long"), + "the path is too long for the file system: the user's cache directory, {user_cache_dir}, at a shorter " + "path{moved_through}", id="too long, the .zip locator's location"), +]) +def test_a_file_on_disk_under_a_zip_directory_with_the_zip_locator_first_is_answered_for_the_locators_numba_reaches( + tmp_path, monkeypatch, refused, told): + # numba's .zip locator takes a path with .zip in it, a file on disk under + # a directory named so included, without trying its location, so numba + # reaches no locator listed after it. The remedy matched the error's path + # against the locations of those too, and an error naming the __pycache__ + # beside the source, under a list with the in-tree locator after the .zip + # one, was told the package installed at a shorter path, for a location + # numba never used. The path is matched against the locations of the + # locators numba reaches alone, and any other is told as the error names + # it. + import numba + import numbox.core.configurations as configurations + from numba.core.caching import _CacheLocator + + class UserCacheUnderTmp: + def __init__(self, appname, appauthor): + self.user_cache_dir = str(tmp_path / "user-cache" / "numba") + + monkeypatch.setattr(configurations, "AppDirs", UserCacheUnderTmp) + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "ZipCacheLocator,InTreeCacheLocator", raising=False) + py_file = tmp_path / "d.zip" / "package" / "module.py" + py_file.parent.mkdir(parents=True) + py_file.write_text("") + user_cache_dir = tmp_path / "user-cache" / "numba" + location = user_cache_dir / _CacheLocator.get_suitable_cache_subpath(str(py_file)) + which, number, strerror = refused + named = py_file.parent / which if which else location + remedy = configurations.cache_remedy(str(py_file), OSError(number, strerror, str(named)), "silence") + told = told.format(named=named, user_cache_dir=user_cache_dir, moved_through=configurations._moved_through()) + assert remedy == f"{told}; or silence", remedy + + +def test_an_empty_path_object_names_the_working_directory_as_pathlib_reads_it(tmp_path, monkeypatch): + # An error that names no file, or an empty name, is told the locations + # numba could have taken; a path object has no empty name, pathlib + # reading an empty string as the working directory, ".", so one built + # from it names that directory, which is none of numba's, and is told as + # named, as the string "." is. + import numba + import numbox.core.configurations as configurations + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + py_file = tmp_path / "site" / "package" / "module.py" + py_file.parent.mkdir(parents=True) + py_file.write_text("") + + def remedy(name): + return configurations.cache_remedy( + str(py_file), OSError(errno.ENAMETOOLONG, "File name too long", name), "silence") + + assert Path("") == Path(".") and os.fspath(Path("")) == "." + assert remedy(Path("")) == remedy(".") == ( + "the path is too long for the file system: that location, ., at a shorter path, or NUMBA_CACHE_DIR set to a " + "short path; or silence") + assert remedy("").startswith("the path is too long for the file system: the location numba took, one of ") + assert remedy(b"") == remedy("") + + +@from_062 +def test_a_locator_entry_naming_no_class_is_left_to_numba_and_its_error_raised_as_it_was(tmp_path, monkeypatch): + # numba resolves a dotted entry of NUMBA_CACHE_LOCATOR_CLASSES to whatever + # the module holds under the name and fails, for want of from_function, + # at a function it decorates where it reaches that entry; the check + # read the entry as numba does and asked issubclass of it first, which + # raised a TypeError naming neither the entry nor the variable, ahead of + # numba's error. An entry that is no class is no locator here, numba's + # own error is raised as it was, and the remedy reads the list without it. + import types + import numba + import numbox.core.configurations as configurations + ours = types.ModuleType("our_locators") + + def not_a_locator_class(): + pass + + ours.not_a_locator_class = not_a_locator_class + monkeypatch.setitem(sys.modules, "our_locators", ours) + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "our_locators.not_a_locator_class,InTreeCacheLocator", + raising=False) + archive = tmp_path / "bundle.zip" + with zipfile.ZipFile(archive, "w") as zipped: + zipped.writestr("ipykernel_123/cell.py", "def f():\n pass\n") + py_file = str(archive / "ipykernel_123" / "cell.py") + with pytest.raises(AttributeError, match="from_function") as raised: + configurations.check_cache_location(py_file, configurations.LONGEST_CACHE_FILE_NAME) + assert not configurations.is_a_cache_error(raised.value) + on_disk = tmp_path / "site" / "package" / "module.py" + on_disk.parent.mkdir(parents=True) + on_disk.write_text("") + failure = OSError(errno.EACCES, "Permission denied", str(on_disk.parent / "__pycache__")) + with_the_entry = configurations.cache_remedy(str(on_disk), failure, "silence") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "InTreeCacheLocator", raising=False) + assert with_the_entry == configurations.cache_remedy(str(on_disk), failure, "silence") + assert "numba caches this file in" in with_the_entry, with_the_entry + + +@from_062 +@pytest.mark.parametrize("refused", ["CacheLocator", "SourceFileBackedLocatorMixin"]) +def test_a_listed_name_numba_refuses_is_left_out_as_numba_leaves_it(tmp_path, monkeypatch, refused): + # numba 0.62 and later resolve a plain entry of NUMBA_CACHE_LOCATOR_CLASSES + # as a name of the caching module and refuse any other before a location + # is tried; the remedy, reading a name with the underscore numba put + # before its locators until 0.62 too, read the two names of numba's base + # classes as those classes and answered for a list numba refuses, with + # one of their locations to make writable. The underscore is tried under + # 0.60 and 0.61 alone, which read no list. + import numba + import numbox.core.configurations as configurations + from numba.core import caching + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + assert configurations._locators(f"{refused},InTreeCacheLocator") == [caching.InTreeCacheLocator] + py_file = tmp_path / "site" / "package" / "module.py" + py_file.parent.mkdir(parents=True) + py_file.write_text("") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", refused, raising=False) + with pytest.raises(RuntimeError, match="Unknown cache locator class") as raised: + configurations.check_cache_location(str(py_file), configurations.LONGEST_CACHE_FILE_NAME) + assert not configurations.is_a_cache_error(raised.value) + failure = OSError(errno.EACCES, "Permission denied", str(py_file.parent / "__pycache__")) + remedy = configurations.cache_remedy(str(py_file), failure, "silence") + assert remedy == ( + f"numba looks only where NUMBA_CACHE_LOCATOR_CLASSES, {refused}, says, and none of those locators takes a " + "source file on disk: list UserProvidedCacheLocator, with NUMBA_CACHE_DIR set, InTreeCacheLocator or " + "UserWideCacheLocator, or silence"), remedy + + +@from_062 +def test_a_locator_entry_with_an_empty_module_path_is_left_out_and_numbas_error_raised_as_it_was(tmp_path, monkeypatch): + # numba imports the module of a dotted entry and lets the ValueError for + # an empty module path, ".InTreeCacheLocator", through at the first + # function it decorates; the remedy, reading the list, raised that + # ValueError itself instead of answering an error a caller passes on + # under such a list. + import numba + import numbox.core.configurations as configurations + from numba.core import caching + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + assert configurations._locators(".InTreeCacheLocator,InTreeCacheLocator") == [caching.InTreeCacheLocator] + py_file = tmp_path / "site" / "package" / "module.py" + py_file.parent.mkdir(parents=True) + py_file.write_text("") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", ".InTreeCacheLocator,InTreeCacheLocator", raising=False) + with pytest.raises(ValueError, match="Empty module name") as raised: + configurations.check_cache_location(str(py_file), configurations.LONGEST_CACHE_FILE_NAME) + assert not configurations.is_a_cache_error(raised.value) + failure = OSError(errno.EACCES, "Permission denied", str(py_file.parent / "__pycache__")) + remedy = configurations.cache_remedy(str(py_file), failure, "silence") + assert remedy.startswith(f"numba caches this file in {py_file.parent / '__pycache__'},"), remedy + + +@from_062 +@pytest.mark.parametrize("placement", ["a cell file on disk", "a .zip member in an ipykernel directory"]) +def test_an_ipython_that_raises_for_its_cache_directory_leaves_numba_no_location_of_ipythons( + tmp_path, monkeypatch, placement): + # numba's IPython locator asks IPython for its cache directory when it + # makes its location, and passes the file over on OSError there as it does + # a location it cannot make; IPython raises one where it can make the + # directory nowhere. The remedy, asking IPython the same for the location + # to match the error's path against, raised that error instead of + # answering, for numba's no-locator error and for the .zip's alike. + import types + import numba + import numbox.core.configurations as configurations + from numba.core.caching import _CacheLocator + + class UserCacheUnderTmp: + def __init__(self, appname, appauthor): + self.user_cache_dir = str(tmp_path / "user-cache" / "numba") + + def nowhere(): + raise PermissionError(errno.EACCES, "Permission denied", str(tmp_path / "home")) + + monkeypatch.setattr(configurations, "AppDirs", UserCacheUnderTmp) + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + paths = types.ModuleType("IPython.paths") + paths.get_ipython_cache_dir = nowhere + monkeypatch.setitem(sys.modules, "IPython", types.ModuleType("IPython")) + monkeypatch.setitem(sys.modules, "IPython.paths", paths) + assert configurations._ipython_numba_cache() is None + if placement == "a cell file on disk": + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "IPythonCacheLocator", raising=False) + py_file = tmp_path / "ipykernel_123" / "cell.py" + py_file.parent.mkdir() + py_file.write_text("def cell():\n pass\n") + with pytest.raises(RuntimeError, match="no locator available") as raised: + configurations.check_cache_location(str(py_file), configurations.LONGEST_CACHE_FILE_NAME) + assert configurations.is_a_cache_error(raised.value) + remedy = configurations.cache_remedy(str(py_file), raised.value, "silence") + assert remedy == ( + "numba looks only where NUMBA_CACHE_LOCATOR_CLASSES, IPythonCacheLocator, says: make one of those " + "locations writable, or silence"), remedy + else: + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + py_file = tmp_path / "bundle.zip" / "ipykernel_123" / "cell.py" + location = tmp_path / "user-cache" / "numba" / _CacheLocator.get_suitable_cache_subpath(str(py_file)) + # The location is there and refuses, as a directory made can; one not + # there is told the directory it cannot be made under as well. + location.mkdir(parents=True) + failure = OSError(errno.EACCES, "Permission denied", str(location)) + remedy = configurations.cache_remedy(str(py_file), failure, "silence") + assert remedy == ( + "numba caches a .zip, or a frozen application, in the user's cache directory, and NUMBA_CACHE_DIR has " + "no effect here, because the source is not a file on disk: no file can be written in that directory, " + f"{location} (Permission denied), so make room there, or make it writable, or silence"), remedy + + +NO_LOCATOR_TAKES_IT_OFF_DISK = { + "a .zip member, too long": ( + "IPythonCacheLocator", "bundle.zip/package/module.py", (errno.ENAMETOOLONG, "File name too long"), + "numba caches a .zip through ZipCacheLocator alone, and NUMBA_CACHE_LOCATOR_CLASSES, IPythonCacheLocator, " + "leaves it out: list it; or silence"), + "a .zip member, unwritable": ( + "IPythonCacheLocator,InTreeCacheLocator", "bundle.zip/package/module.py", (errno.EACCES, "Permission denied"), + "numba caches a .zip through ZipCacheLocator alone, and NUMBA_CACHE_LOCATOR_CLASSES, " + "IPythonCacheLocator,InTreeCacheLocator, leaves it out: list it; or silence"), + "a frozen application, unwritable": ( + "InTreeCacheLocator", "frozen/numbox/module.py", (errno.EACCES, "Permission denied"), + "numba caches a frozen application through UserWideCacheLocator alone, and NUMBA_CACHE_LOCATOR_CLASSES, " + "InTreeCacheLocator, leaves it out: list it, or silence"), +} + + +@from_062 +@pytest.mark.parametrize("case", list(NO_LOCATOR_TAKES_IT_OFF_DISK)) +@pytest.mark.parametrize("names", ["the location", "nothing"]) +def test_an_error_for_a_source_not_on_disk_under_a_list_with_no_locator_that_takes_it_is_told_the_list( + tmp_path, monkeypatch, case, names): + # numba's .zip locator alone takes a .zip member in no ipykernel directory, + # and its user-wide one alone a frozen application's file, so a list + # without the one leaves numba no locator for the file, and its error is + # the no-locator one, told to list it. An OSError a caller passes on under + # such a list, naming a location or none, was answered as a .zip's or a + # frozen application's: that numba caches the file in the user's cache + # directory, and that location, or that directory, to make writable or put + # at a shorter path, while numba took none. It is told the list, as + # numba's no-locator error is, as for a source on disk. + import numba + import numbox.core.configurations as configurations + from numba.core.caching import _CacheLocator + + class UserCacheUnderTmp: + def __init__(self, appname, appauthor): + self.user_cache_dir = str(tmp_path / "user-cache" / "numba") + + locators, member, error, told = NO_LOCATOR_TAKES_IT_OFF_DISK[case] + monkeypatch.setattr(configurations, "AppDirs", UserCacheUnderTmp) + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", locators, raising=False) + monkeypatch.setattr(sys, "frozen", case.startswith("a frozen"), raising=False) + py_file = str(tmp_path / member) + location = tmp_path / "user-cache" / "numba" / _CacheLocator.get_suitable_cache_subpath(py_file) + failure = OSError(*error, str(location)) if names == "the location" else OSError(*error) + remedy = configurations.cache_remedy(py_file, failure, "silence") + assert remedy == told, remedy + no_locator = RuntimeError(f"cannot cache function '_cache_probe': no locator available for file '{py_file}'") + assert configurations.cache_remedy(py_file, no_locator, "silence") == told + + +NO_NAME_OFF_DISK = { + "a .zip member in an ipykernel directory, numba's order, a full disk": ( + "", "bundle.zip/ipykernel_123/cell.py", (errno.ENOSPC, "No space left on device"), + "NUMBA_CACHE_DIR has no effect here, because the source is not a file on disk: no file can be written in the " + "location numba took, one of {numba_cache} or {location} (No space left on device), so make room there, or " + "make it writable"), + "a .zip member in an ipykernel directory, numba's order, too long": ( + "", "bundle.zip/ipykernel_123/cell.py", (errno.ENAMETOOLONG, "File name too long"), + "NUMBA_CACHE_DIR has no effect here, because the source is not a file on disk: the path is too long for the " + "file system, so put the location numba took, one of {numba_cache} or {location}, at a shorter path"), + "a .zip member in an ipykernel directory, the .zip locator first, a full disk": ( + "ZipCacheLocator,IPythonCacheLocator", "bundle.zip/ipykernel_123/cell.py", + (errno.ENOSPC, "No space left on device"), + "numba caches a .zip, or a frozen application, in the user's cache directory, and NUMBA_CACHE_DIR has no " + "effect here, because the source is not a file on disk: no file can be written in the location numba took, " + "{location} (No space left on device), so make room there, or make it writable"), + "a .zip member, a full disk": ( + "", "bundle.zip/package/module.py", (errno.ENOSPC, "No space left on device"), + "numba caches a .zip, or a frozen application, in the user's cache directory, and NUMBA_CACHE_DIR has no " + "effect here, because the source is not a file on disk: no file can be written in the location numba took, " + "{location} (No space left on device), so make room there, or make it writable"), + "a .zip member, too long": ( + "", "bundle.zip/package/module.py", (errno.ENAMETOOLONG, "File name too long"), + "numba caches a .zip, or a frozen application, in the user's cache directory, and NUMBA_CACHE_DIR has no " + "effect here, because the source is not a file on disk: the path is too long for the file system, so put the " + "location numba took, {location}, at a shorter path{moved_through}"), + "a frozen application, a full disk": ( + "", "frozen/numbox/module.py", (errno.ENOSPC, "No space left on device"), + "numba caches a .zip, or a frozen application, in the user's cache directory, and NUMBA_CACHE_DIR has no " + "effect here, because the source is not a file on disk: no file can be written in the location numba took, " + "{location} (No space left on device), so make room there, or make it writable"), +} + + +@pytest.mark.parametrize("case", [ + pytest.param(case, marks=from_062 if "locator first" in case else from_061 if ".zip" in case else [], id=case) + for case in NO_NAME_OFF_DISK +]) +def test_an_error_naming_no_file_for_a_source_not_on_disk_is_told_the_locations_numba_could_have_taken( + tmp_path, monkeypatch, case): + # numba's first save raises an OSError that names no file where a write + # fails, as on a full disk, and a caller can pass one on. Which location + # numba took cannot be told from it, so the remedy for a source on disk + # lists the ones numba could have taken; the remedy for a source not on + # disk told the user's cache directory, which for a .zip member in an + # ipykernel directory under numba's order, cached through its IPython + # locator in numba_cache under IPython's cache directory, is a directory + # numba never used, and for a .zip or a frozen application is above the + # directory numba took. It is told the locations numba could have taken, + # in numba's order and each once, and where IPython's is among them + # nothing of the user's cache directory as where numba caches the file. + import types + import numba + import numbox.core.configurations as configurations + from numba.core.caching import _CacheLocator + + class UserCacheUnderTmp: + def __init__(self, appname, appauthor): + self.user_cache_dir = str(tmp_path / "user-cache" / "numba") + + locators, member, error, told = NO_NAME_OFF_DISK[case] + monkeypatch.setattr(configurations, "AppDirs", UserCacheUnderTmp) + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", locators, raising=False) + monkeypatch.setattr(sys, "frozen", case.startswith("a frozen"), raising=False) + ipython_dir = tmp_path / "ipython" + paths = types.ModuleType("IPython.paths") + paths.get_ipython_cache_dir = lambda: str(ipython_dir) + monkeypatch.setitem(sys.modules, "IPython", types.ModuleType("IPython")) + monkeypatch.setitem(sys.modules, "IPython.paths", paths) + py_file = str(tmp_path / member) + location = tmp_path / "user-cache" / "numba" / _CacheLocator.get_suitable_cache_subpath(py_file) + remedy = configurations.cache_remedy(py_file, OSError(*error), "silence") + told = told.format(numba_cache=ipython_dir / "numba_cache", location=location, + moved_through=configurations._moved_through()) + assert remedy == f"{told}, or silence", remedy + + +@from_061 +@pytest.mark.parametrize("refused", list(NONE_OF_NUMBAS_OFF_DISK)) +def test_an_error_naming_ipythons_location_with_ipython_not_importable_is_told_the_location_as_none_of_numbas( + tmp_path, monkeypatch, refused): + # numba's IPython locator takes a .zip member in an ipykernel directory + # and imports IPython for its location, catching OSError alone, so with + # IPython not importable numba raises ImportError at decoration, no cache + # error, and caches the file nowhere; an error a caller passes on naming + # IPython's location is told it as none of numba's. The remedy opened with + # where numba caches a .zip, or a frozen application, the user's cache + # directory, which numba did not cache this file in. + import numba + import numbox.core.configurations as configurations + + class UserCacheUnderTmp: + def __init__(self, appname, appauthor): + self.user_cache_dir = str(tmp_path / "user-cache" / "numba") + + monkeypatch.setattr(configurations, "AppDirs", UserCacheUnderTmp) + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + monkeypatch.setitem(sys.modules, "IPython", None) + monkeypatch.setitem(sys.modules, "IPython.paths", None) + monkeypatch.setitem(sys.modules, "IPython.utils.path", None) + with zipfile.ZipFile(tmp_path / "bundle.zip", "w") as zipped: + zipped.writestr("ipykernel_123/cell.py", "def cell():\n pass\n") + py_file = str(tmp_path / "bundle.zip" / "ipykernel_123" / "cell.py") + with pytest.raises(ImportError) as raised: + configurations.check_cache_location(py_file, configurations.LONGEST_CACHE_FILE_NAME) + assert not configurations.is_a_cache_error(raised.value) + named = tmp_path / "ipython" / "numba_cache" + error, told = NONE_OF_NUMBAS_OFF_DISK[refused] + remedy = configurations.cache_remedy(py_file, OSError(*error, str(named)), "silence") + assert remedy == ( + "NUMBA_CACHE_DIR has no effect here, because the source is not a file on disk: " + f"{told.format(named=named)}, or silence"), remedy + + +@pytest.mark.parametrize("refused, told", [ + ("too long", "the path is too long for the file system: IPython's cache directory, {ipython_dir}, at a shorter path"), + ("unwritable", "numba caches this file in {numba_cache}, where no file can be written (Permission denied): make " + "room there, or make it writable"), +]) +def test_a_cell_numbas_ipython_locator_takes_by_its_name_is_asked_and_told_as_a_cell_file_is( + tmp_path, monkeypatch, refused, told): + # numba's IPython locator takes a cell, "", by its name, as it + # takes a file in an ipykernel directory, reads its source from linecache, + # where IPython holds it while it runs the cell, and caches it in + # numba_cache under IPython's cache directory: the check leaves linecache + # to the locator, which reads the cell as numba does, and the remedy is + # the one a cell file on disk gets. Nothing held the remedy to the cell: + # it told IPython's location as none of numba's. + import linecache + import types + import numba + import numbox.core.configurations as configurations + + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + ipython_dir = tmp_path / "ipython" + paths = types.ModuleType("IPython.paths") + paths.get_ipython_cache_dir = lambda: str(ipython_dir) + monkeypatch.setitem(sys.modules, "IPython", types.ModuleType("IPython")) + monkeypatch.setitem(sys.modules, "IPython.paths", paths) + cell = "" + source = "def f():\n pass\n" + monkeypatch.setitem(linecache.cache, cell, (len(source), None, source.splitlines(True), cell)) + configurations.check_cache_location(cell, configurations.LONGEST_CACHE_FILE_NAME) + assert (ipython_dir / "numba_cache").is_dir() + assert linecache.cache[cell][2] == source.splitlines(True) + error = (errno.ENAMETOOLONG, "File name too long") if refused == "too long" else (errno.EACCES, "Permission denied") + remedy = configurations.cache_remedy(cell, OSError(*error, str(ipython_dir / "numba_cache")), "silence") + told = told.format(ipython_dir=ipython_dir, numba_cache=ipython_dir / "numba_cache") + assert remedy == f"{told}; or silence", remedy + + +def test_a_frozen_application_whose_user_cache_directory_is_too_deep_is_told_a_shorter_path(tmp_path, monkeypatch): + # numba's user-wide locator passes a location it cannot make over, one too + # deep for the file system as an unwritable one, and numba's error for a + # frozen application is then its no-locator one, which gives no reason; + # the remedy asked for the user's cache directory to be made writable + # alone, which cures no path too long. + import numba + from numba.core import caching + import numbox.core.configurations as configurations + + class UserCacheTooDeep: + def __init__(self, appname, appauthor): + self.user_cache_dir = str(tmp_path / ("x" * 300) / "numba") + + monkeypatch.setattr(caching, "AppDirs", UserCacheTooDeep) + monkeypatch.setattr(configurations, "AppDirs", UserCacheTooDeep) + monkeypatch.setattr(sys, "frozen", True, raising=False) + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + py_file = str(tmp_path / "frozen" / "numbox" / "module.py") + with pytest.raises(RuntimeError, match="no locator available") as raised: + configurations.check_cache_location(py_file, configurations.LONGEST_CACHE_FILE_NAME) + assert configurations.is_a_cache_error(raised.value) + remedy = configurations.cache_remedy(py_file, raised.value, "silence") + assert remedy == ( + "numba caches a .zip, or a frozen application, in the user's cache directory, and NUMBA_CACHE_DIR has no " + "effect here, because the source is not a file on disk: numba could not use that directory, " + f"{tmp_path / ('x' * 300) / 'numba'}: make it writable, or put it at a shorter " + f"path{configurations._moved_through()}, or silence"), remedy + + +A_FROZEN_ZIP_PATH = { + "the .zip locator first": ( + "ZipCacheLocator,UserWideCacheLocator", + "numba's .zip locator, listed in NUMBA_CACHE_LOCATOR_CLASSES, {locators}, with no UserWideCacheLocator before " + 'it, takes this file for the ".zip" in its path and finds no archive there: list UserWideCacheLocator, the one ' + "locator for a frozen application, before it"), + "the .zip locator and no user-wide one": ( + "ZipCacheLocator,InTreeCacheLocator", + "numba's .zip locator, listed in NUMBA_CACHE_LOCATOR_CLASSES, {locators}, with no UserWideCacheLocator before " + 'it, takes this file for the ".zip" in its path and finds no archive there: list UserWideCacheLocator, the one ' + "locator for a frozen application, before it"), + "numba's order, the user-wide locator passed over": ( + "", + 'numba\'s .zip locator takes this file for the ".zip" in its path and finds no archive there, after ' + "UserWideCacheLocator, which numba tries before it, could not use the user's cache directory, " + "{user_cache_dir}: make it writable, or put it at a shorter path{moved_through}"), +} + + +@pytest.mark.parametrize("case", [ + pytest.param(case, marks=from_061 if case.startswith("numba's order") else from_062, id=case) + for case in A_FROZEN_ZIP_PATH +]) +def test_a_frozen_applications_path_with_zip_in_it_and_no_archive_is_answered_for_the_zip_locators_error( + tmp_path, monkeypatch, case): + # numba's .zip locator takes a path with ".zip" in it without trying its + # location and raises where no part of the path ends in .zip, so a frozen + # application's file at such a path gets that error where the list puts + # the .zip locator before the user-wide one, which alone takes the file, + # or once the user-wide one passed its location over. The remedy answered + # with the user's cache directory to make writable, a directory numba never + # tried under such a list; it is the user-wide locator to list before the + # .zip one, or the location that one passed over, as for a source on disk. + import numba + from numba.core import caching + import numbox.core.configurations as configurations + + class UserCacheTooDeep: + def __init__(self, appname, appauthor): + self.user_cache_dir = str(tmp_path / ("x" * 300) / "numba") + + locators, told = A_FROZEN_ZIP_PATH[case] + monkeypatch.setattr(caching, "AppDirs", UserCacheTooDeep) + monkeypatch.setattr(configurations, "AppDirs", UserCacheTooDeep) + monkeypatch.setattr(sys, "frozen", True, raising=False) + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", locators, raising=False) + py_file = str(tmp_path / "not.zipped" / "frozen" / "numbox" / "module.py") + with pytest.raises(ValueError, match="No zip file found") as raised: + configurations.check_cache_location(py_file, configurations.LONGEST_CACHE_FILE_NAME) + assert configurations.is_a_cache_error(raised.value) + remedy = configurations.cache_remedy(py_file, raised.value, "silence") + told = told.format(locators=locators, user_cache_dir=tmp_path / ("x" * 300) / "numba", + moved_through=configurations._moved_through()) + assert remedy == f"{told}, or silence", remedy + + +@pytest.mark.parametrize("cell_file, order", [ + pytest.param("empty", "the IPython locator alone", marks=[from_062, pytest.mark.skipif( + sys.version_info >= (3, 13), reason="linecache gives an empty file a blank line from Python 3.13")]), + pytest.param("unreadable", "the IPython locator alone", marks=[from_062, needs_a_directory_it_cannot_write]), + pytest.param("unreadable", "numba's order", marks=needs_a_directory_it_cannot_write), +]) +def test_the_check_raises_inspects_error_naming_no_file_for_a_cell_file_on_disk_it_reads_no_source_in( + tmp_path, monkeypatch, cell_file, order): + # numba's IPython locator reads the function's source, and for a file on + # disk inspect reads it from the file itself and raises OSError, naming no + # file and with no errno, where it reads no source there, a file it cannot + # read or, before Python 3.13, an empty one, which linecache gives a blank + # line from then on; the check raises that error before that locator + # tries its location, the locators numba tries before it having passed + # the file over, and is_a_cache_error admits it as it admits every + # OSError. The + # check's docstring said every OSError from it names the location numba + # picked, and the remedy told the location numba took, which it had not, + # as refusing a file. + import types + import numba + import numbox.core.configurations as configurations + + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + listed = "IPythonCacheLocator" if order == "the IPython locator alone" else "" + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", listed, raising=False) + if order == "numba's order": + # The in-tree and user-wide locators try their locations first and + # pass the file over, the cell's directory unwritable and the user's + # cache directory under a file. + from numba.core import caching + blocker = tmp_path / "not_a_dir" + blocker.write_text("") + + class UserCacheUnderAFile: + def __init__(self, appname, appauthor): + self.user_cache_dir = str(blocker / "user-cache" / "numba") + + monkeypatch.setattr(caching, "AppDirs", UserCacheUnderAFile) + monkeypatch.setattr(configurations, "AppDirs", UserCacheUnderAFile) + ipython_dir = tmp_path / "ipython" + paths = types.ModuleType("IPython.paths") + paths.get_ipython_cache_dir = lambda: str(ipython_dir) + monkeypatch.setitem(sys.modules, "IPython", types.ModuleType("IPython")) + monkeypatch.setitem(sys.modules, "IPython.paths", paths) + cell = tmp_path / "ipykernel_123" / "__init__.py" + cell.parent.mkdir() + cell.touch() + if cell_file == "unreadable": + cell.write_text("def f():\n pass\n") + cell.chmod(0) + if order == "numba's order": + cell.parent.chmod(0o555) + try: + with pytest.raises(OSError, match="could not get source code") as raised: + configurations.check_cache_location(str(cell), configurations.LONGEST_CACHE_FILE_NAME) + finally: + cell.parent.chmod(0o755) + cell.chmod(0o644) + assert raised.value.filename is None and raised.value.errno is None + assert configurations.is_a_cache_error(raised.value) + assert not (ipython_dir / "numba_cache").exists() and not (cell.parent / "__pycache__").exists() + remedy = configurations.cache_remedy(str(cell), raised.value, "silence") + assert remedy == ( + f"numba's IPython locator reads the function's source from its file before trying its location, and read " + f"none in {cell} (could not get source code): make the file readable, with the source in it; or silence"), remedy + # An error with no errno that names a file is a caller's, not inspect's, + # and is told the location it names. + named = OSError("could not get source code") + named.filename = str(tmp_path / "elsewhere") + remedy = configurations.cache_remedy(str(cell), named, "silence") + assert remedy.startswith(f"that location, {tmp_path / 'elsewhere'}, where no file can be written"), remedy + + +@from_062 +def test_a_source_not_on_disk_under_a_directory_named_for_a_zip_is_in_no_zip(tmp_path, monkeypatch): + # numba's .zip locator takes a path with ".zip" in it and finds the archive + # by a part of the path that ends in .zip, so a source under a directory + # named my.zipped is in no .zip: under a list without that locator, numba's + # no-locator error is told the source files on disk or a .zip holding them, + # with the locator each needs, and under numba's order the .zip locator + # takes the path and raises for the archive it finds no part of it to be, + # which is told the same. Nothing held the remedy to the part: with the + # substring enough, the no-locator error was told to list the .zip locator + # alone, which numba then raises through, and the suite passed. + import numba + import numbox.core.configurations as configurations + + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + py_file = str(tmp_path / "my.zipped" / "package.egg" / "numbox" / "module.py") + archive_remedy = ( + "NUMBA_CACHE_DIR has no effect here, because the source is not a file on disk: to cache, install numbox with " + "its source files on disk, unpacked from any archive, or import it from a .zip holding its source files, " + "which numba 0.61 and later cache in the user's cache directory") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "InTreeCacheLocator", raising=False) + with pytest.raises(RuntimeError, match="no locator available") as raised: + configurations.check_cache_location(py_file, configurations.LONGEST_CACHE_FILE_NAME) + remedy = configurations.cache_remedy(py_file, raised.value, "silence") + assert remedy == ( + f"{archive_remedy} through ZipCacheLocator, once NUMBA_CACHE_LOCATOR_CLASSES, InTreeCacheLocator, lists it; " + "or silence"), remedy + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + with pytest.raises(ValueError, match="No zip file found") as raised: + configurations.check_cache_location(py_file, configurations.LONGEST_CACHE_FILE_NAME) + assert configurations.is_a_cache_error(raised.value) + assert configurations.cache_remedy(py_file, raised.value, "silence") == f"{archive_remedy}; or silence" + + +@from_062 +@pytest.mark.parametrize("entry", ["..InTreeCacheLocator", ".a.InTreeCacheLocator"]) +def test_a_locator_entry_with_a_relative_module_path_is_left_out_and_numbas_error_raised_as_it_was( + tmp_path, monkeypatch, entry): + # numba imports the module of a dotted entry and lets importlib's TypeError + # for a relative module path, "..InTreeCacheLocator", through at the first + # function it decorates, as it lets the ValueError for an empty one; the + # remedy, reading the list, caught the ValueError and raised the TypeError + # itself instead of answering an error a caller passes on under such a list. + import numba + import numbox.core.configurations as configurations + from numba.core import caching + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + assert configurations._locators(f"{entry},InTreeCacheLocator") == [caching.InTreeCacheLocator] + py_file = tmp_path / "site" / "package" / "module.py" + py_file.parent.mkdir(parents=True) + py_file.write_text("") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", f"{entry},InTreeCacheLocator", raising=False) + with pytest.raises(TypeError, match="relative import") as raised: + configurations.check_cache_location(str(py_file), configurations.LONGEST_CACHE_FILE_NAME) + assert not configurations.is_a_cache_error(raised.value) + failure = OSError(errno.EACCES, "Permission denied", str(py_file.parent / "__pycache__")) + remedy = configurations.cache_remedy(str(py_file), failure, "silence") + assert remedy.startswith(f"numba caches this file in {py_file.parent / '__pycache__'},"), remedy + + +@from_062 +@pytest.mark.parametrize("entry", ["CacheImpl", "pathlib.PurePath"]) +def test_a_locator_entry_naming_a_class_with_no_from_function_is_left_out_as_one_naming_no_class_is( + tmp_path, monkeypatch, entry): + # numba fails on a listed class with no from_function, its own CacheImpl or + # any other, as it fails on an entry naming no class, for want of that + # method at a function it decorates where it reaches the entry; the + # remedy left out the entry naming no class and kept the class, as a + # locator of its own that takes any file on disk, so the remedy with the + # entry lost the NUMBA_CACHE_DIR offer, and the entry alone was told to + # make one of its locations writable, a list with no locator at all. + import numba + import numbox.core.configurations as configurations + from numba.core import caching + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + assert configurations._locators(f"{entry},InTreeCacheLocator") == [caching.InTreeCacheLocator] + py_file = tmp_path / "site" / "package" / "module.py" + py_file.parent.mkdir(parents=True) + py_file.write_text("") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", f"{entry},InTreeCacheLocator", raising=False) + with pytest.raises(AttributeError, match="from_function") as raised: + configurations.check_cache_location(str(py_file), configurations.LONGEST_CACHE_FILE_NAME) + assert not configurations.is_a_cache_error(raised.value) + failure = OSError(errno.ENAMETOOLONG, "File name too long", str(tmp_path / "elsewhere")) + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", f"{entry},UserProvidedCacheLocator,InTreeCacheLocator", + raising=False) + with_the_entry = configurations.cache_remedy(str(py_file), failure, "silence") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "UserProvidedCacheLocator,InTreeCacheLocator", + raising=False) + assert with_the_entry == configurations.cache_remedy(str(py_file), failure, "silence") + assert "NUMBA_CACHE_DIR set to a short path" in with_the_entry, with_the_entry + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", entry, raising=False) + alone = configurations.cache_remedy(str(py_file), failure, "silence") + assert alone == ( + f"numba looks only where NUMBA_CACHE_LOCATOR_CLASSES, {entry}, says, and none of those locators takes a " + "source file on disk: list UserProvidedCacheLocator, with NUMBA_CACHE_DIR set, InTreeCacheLocator or " + "UserWideCacheLocator, or silence"), alone + + +@from_061 +@pytest.mark.parametrize("refused, told", [ + ("a full disk", "no file can be written in the location numba took, {location} (No space left on device), so make " + "room there, or make it writable"), + ("too long", "the path is too long for the file system, so put the location numba took, {location}, at a shorter " + "path"), +]) +def test_an_error_naming_no_file_with_ipython_not_importable_says_nothing_of_the_user_cache_directory( + tmp_path, monkeypatch, refused, told): + # numba's IPython locator takes a .zip member in an ipykernel directory + # before its .zip locator can, so an error naming no file for the member + # may be IPython's location's, and the remedy says nothing of the user's + # cache directory as where numba caches the file; it did so only where + # IPython gave its directory, and with IPython not importable, which + # numba raises ImportError for at decoration, opened with that directory + # as where numba caches a .zip, or a frozen application, against its own + # comment. + import numba + import numbox.core.configurations as configurations + from numba.core.caching import _CacheLocator + + class UserCacheUnderTmp: + def __init__(self, appname, appauthor): + self.user_cache_dir = str(tmp_path / "user-cache" / "numba") + + monkeypatch.setattr(configurations, "AppDirs", UserCacheUnderTmp) + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + monkeypatch.setitem(sys.modules, "IPython", None) + monkeypatch.setitem(sys.modules, "IPython.paths", None) + monkeypatch.setitem(sys.modules, "IPython.utils.path", None) + py_file = str(tmp_path / "bundle.zip" / "ipykernel_123" / "cell.py") + location = tmp_path / "user-cache" / "numba" / _CacheLocator.get_suitable_cache_subpath(py_file) + error = ((errno.ENOSPC, "No space left on device") if refused == "a full disk" else + (errno.ENAMETOOLONG, "File name too long")) + remedy = configurations.cache_remedy(py_file, OSError(*error), "silence") + assert remedy == ( + "NUMBA_CACHE_DIR has no effect here, because the source is not a file on disk: " + f"{told.format(location=location)}, or silence"), remedy + + +@from_062 +@pytest.mark.parametrize("locators, told", [ + ("IPythonCacheLocator", + "numba looks only where NUMBA_CACHE_LOCATOR_CLASSES, IPythonCacheLocator, says: make one of those locations " + "writable, or silence"), + ("IPythonCacheLocator,InTreeCacheLocator", + "the path is too long for the file system: that location, {named}, at a shorter path; or silence"), +]) +def test_an_error_naming_ipythons_location_with_ipython_not_importable_is_told_the_list_where_no_locator_has_a_place( + tmp_path, monkeypatch, locators, told): + # numba's IPython locator imports IPython for its location and lets the + # ImportError through at decoration, so with IPython not importable numba + # caches a cell file nowhere and raises no cache error; an error a caller + # passes on naming IPython's location is told it as none of numba's where + # another listed locator has a place for the file, and the list where + # none has, as numba's no-locator error is told it. The docstring promised + # the location as named under any list. + import numba + import numbox.core.configurations as configurations + + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", locators, raising=False) + monkeypatch.setitem(sys.modules, "IPython", None) + monkeypatch.setitem(sys.modules, "IPython.paths", None) + monkeypatch.setitem(sys.modules, "IPython.utils.path", None) + py_file = tmp_path / "ipykernel_123" / "cell.py" + py_file.parent.mkdir() + py_file.write_text("def cell():\n pass\n") + with pytest.raises(ImportError) as raised: + configurations.check_cache_location(str(py_file), configurations.LONGEST_CACHE_FILE_NAME) + assert not configurations.is_a_cache_error(raised.value) + named = tmp_path / "ipython" / "numba_cache" / "cell-1.py312.nbi" + failure = OSError(errno.ENAMETOOLONG, "File name too long", str(named)) + remedy = configurations.cache_remedy(str(py_file), failure, "silence") + assert remedy == told.format(named=named), remedy + + +@from_062 +@pytest.mark.parametrize("placement, refused", [ + pytest.param("a source on disk", "too long", marks=pytest.mark.skipif( + os.name == "nt", reason="Windows refuses a 300-character directory name as invalid, not as too long")), + pytest.param("a source on disk", "unwritable", marks=needs_a_directory_it_cannot_write), + pytest.param("a .zip member", "unwritable", marks=needs_a_directory_it_cannot_write), + ("a source on disk", "naming no file"), ("a .zip member", "naming no file"), + ("a source on disk, the in-tree locator after", "naming no file"), +]) +def test_a_listed_class_of_none_of_numbas_families_is_told_the_location_the_error_names( + tmp_path, monkeypatch, placement, refused): + # NUMBA_CACHE_LOCATOR_CLASSES exists for a locator of the reader's own, a + # class of none of numba's families, with a from_function and a place of + # its own that the remedy cannot know; numba caches through it, and the + # check's error names its location. The remedy read the class as taking + # files on disk alone and as having no place, so a path too long on disk + # was told to make one of the list's locations writable, with the cause + # dropped, and a .zip member's refused file to list the .zip locator, with + # the location dropped; upstream main named the cause for the one and the + # location for the other. The class takes any file for the remedy, and the + # location the error names, which no locator of numba's gives the file, is + # told as named, with the cause; an error naming no file is told the + # location numba took as the class's own, with no path to name, among + # numba's locators' places where those are reached. + import types + import numba + import numbox.core.configurations as configurations + from numba.core.caching import _CacheLocator + + where = tmp_path / ("x" * 300) if refused == "too long" else tmp_path / "our-cache" + + class OurLocator(_CacheLocator): + def __init__(self, py_func, py_file): + self._py_file = py_file + self._cache_path = str(where / self.get_suitable_cache_subpath(py_file)) + + def get_cache_path(self): + return self._cache_path + + def get_source_stamp(self): + return 0 + + def get_disambiguator(self): + return "x" + + @classmethod + def from_function(cls, py_func, py_file): + return cls(py_func, py_file) + + ours = types.ModuleType("our_locators") + ours.OurLocator = OurLocator + monkeypatch.setitem(sys.modules, "our_locators", ours) + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + listed = "our_locators.OurLocator,InTreeCacheLocator" if placement.endswith("after") else "our_locators.OurLocator" + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", listed, raising=False) + assert configurations._locators("our_locators.OurLocator") == [OurLocator] + if placement.startswith("a source on disk"): + py_file = tmp_path / "site" / "package" / "module.py" + py_file.parent.mkdir(parents=True) + py_file.write_text("") + else: + py_file = tmp_path / "bundle.zip" / "package" / "module.py" + if refused == "naming no file": + remedy = configurations.cache_remedy(str(py_file), OSError(errno.ENOSPC, "No space left on device"), "silence") + if placement.startswith("a source on disk"): + # OurLocator takes the file, asked, so the in-tree locator listed + # after it is not reached: the remedy named its __pycache__ too. + assert remedy == ( + "the location numba took, OurLocator's own, where no file can be written (No space left on device): " + "make room there, or make it writable; or silence"), remedy + else: + assert remedy == ( + "NUMBA_CACHE_DIR has no effect here, because the source is not a file on disk: no file can be written " + "in the location numba took, OurLocator's own (No space left on device), so make room there, or make " + "it writable, or silence"), remedy + return + if refused == "unwritable": + where.mkdir() + where.chmod(0o555) + try: + with pytest.raises(OSError) as raised: + configurations.check_cache_location(str(py_file), configurations.LONGEST_CACHE_FILE_NAME) + finally: + if refused == "unwritable": + where.chmod(0o755) + failure = raised.value + assert configurations.is_a_cache_error(failure) + named = os.fsdecode(failure.filename) + assert named.startswith(str(where)), named + remedy = configurations.cache_remedy(str(py_file), failure, "silence") + if refused == "too long": + assert failure.errno == errno.ENAMETOOLONG + assert remedy == ( + f"the path is too long for the file system: that location, {named}, at a shorter path; or silence"), remedy + elif placement == "a source on disk": + assert remedy == ( + f"that location, {named}, where no file can be written (Permission denied): make room there, or make it " + "writable; or silence"), remedy + else: + assert remedy == ( + "NUMBA_CACHE_DIR has no effect here, because the source is not a file on disk: no file can be written at " + f"that location, {named} (Permission denied), so make room there, or make it writable, or silence"), remedy + + +@from_062 +def test_a_frozen_applications_no_locator_error_under_a_class_of_none_of_numbas_families_is_told_the_user_wide_locator( + tmp_path, monkeypatch): + # A locator of the reader's own listed alone can pass a frozen + # application's file over, and numba's error is then its no-locator one, + # which names nothing; the remedy, counting the class as having a place + # of its own, answered with the user's cache directory as one numba could + # not use, a directory numba never tried, where under a list of numba's + # own locators without the user-wide one it asks for that locator. + import types + import numba + import numbox.core.configurations as configurations + from numba.core.caching import _CacheLocator + + class OurLocator(_CacheLocator): + @classmethod + def from_function(cls, py_func, py_file): + return None + + ours = types.ModuleType("our_locators") + ours.OurLocator = OurLocator + monkeypatch.setitem(sys.modules, "our_locators", ours) + monkeypatch.setattr(sys, "frozen", True, raising=False) + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "our_locators.OurLocator", raising=False) + py_file = str(tmp_path / "frozen" / "numbox" / "module.py") + with pytest.raises(RuntimeError, match="no locator available") as raised: + configurations.check_cache_location(py_file, configurations.LONGEST_CACHE_FILE_NAME) + assert configurations.is_a_cache_error(raised.value) + remedy = configurations.cache_remedy(py_file, raised.value, "silence") + assert remedy == ( + "numba caches a frozen application through UserWideCacheLocator alone, and NUMBA_CACHE_LOCATOR_CLASSES, " + "our_locators.OurLocator, leaves it out: list it, or silence"), remedy + + +def test_an_error_naming_a_file_descriptor_names_no_file(tmp_path, monkeypatch): + # os.stat(fd) and the other calls that take a descriptor raise OSError with + # the integer as the file name, and is_a_cache_error admits every OSError; + # the remedy read every name as a path and raised TypeError on the integer + # instead of answering. A name of another kind than str, bytes or a path + # object is none, and the error is told the locations numba could have + # taken, as one naming no file is. + import numba + import numbox.core.configurations as configurations + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + py_file = tmp_path / "site" / "package" / "module.py" + py_file.parent.mkdir(parents=True) + py_file.write_text("") + with pytest.raises(OSError) as raised: + os.stat(987654) + failure = raised.value + assert failure.filename == 987654 and configurations.is_a_cache_error(failure) + remedy = configurations.cache_remedy(str(py_file), failure, "silence") + assert remedy == configurations.cache_remedy(str(py_file), OSError(failure.errno, failure.strerror), "silence") + assert remedy.startswith("the location numba took, one of "), remedy + assert "987654" not in remedy + + +@pytest.mark.parametrize("placement", [ + pytest.param("a cell", id="a cell, numba's order"), + pytest.param("a cell, a class of none of numba's families listed too", marks=from_062, + id="a cell, the IPython locator and a class of the reader's own"), + pytest.param("a cell, a class of none of numba's families listed first", marks=from_062, + id="a cell, a class of the reader's own and the IPython locator"), + pytest.param("a .zip member in an ipykernel directory", marks=from_062, id="a .zip member, the IPython locator alone"), +]) +@pytest.mark.parametrize("ipython", ["cannot make its directory", "raises for its directory"]) +def test_a_file_numbas_ipython_locator_alone_takes_and_passes_over_is_told_ipythons_location( + tmp_path, monkeypatch, placement, ipython): + # numba's IPython locator passes its location over where it cannot make or + # write in it, or where IPython raises for its directory, and a cell, which + # no other locator takes, then gets numba's no-locator error, as does a + # .zip member in an ipykernel directory under a list with no .zip locator + # after the IPython one; the remedy answered with the archive remedy, the + # source files on disk or a .zip holding them, which touch no cell, and + # answered so again for a cell under a list of the IPython locator and a + # class of none of numba's families, which passed the cell over too and + # is named as having. + import linecache + import types + import numba + import numbox.core.configurations as configurations + + blocker = tmp_path / "not_a_dir" + blocker.write_text("") + ipython_dir = str(blocker / "ipython") + + def nowhere(): + raise PermissionError(errno.EACCES, "Permission denied", str(tmp_path / "home")) + + paths = types.ModuleType("IPython.paths") + paths.get_ipython_cache_dir = (lambda: ipython_dir) if ipython == "cannot make its directory" else nowhere + monkeypatch.setitem(sys.modules, "IPython", types.ModuleType("IPython")) + monkeypatch.setitem(sys.modules, "IPython.paths", paths) + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + py_file = "" if placement.startswith("a cell") else str( + tmp_path / "bundle.zip" / "ipykernel_123" / "cell.py") + if placement.startswith("a cell"): + # IPython holds a cell's source in linecache while it runs it, and + # numba's IPython locator reads it there. + source = "def f():\n pass\n" + monkeypatch.setitem(linecache.cache, py_file, (len(source), None, source.splitlines(True), py_file)) + else: + # inspect reads a .zip member's source through the module's loader, + # which the probe has none of, so the check gives it the probe's + # where the archive holds the member's source. + with zipfile.ZipFile(tmp_path / "bundle.zip", "w") as zipped: + zipped.writestr("ipykernel_123/cell.py", "def cell():\n pass\n") + if placement == "a cell": + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + elif placement.startswith("a cell"): + from numba.core.caching import _CacheLocator + + class OurLocator(_CacheLocator): + @classmethod + def from_function(cls, py_func, py_file): + return None + + ours = types.ModuleType("our_locators") + ours.OurLocator = OurLocator + monkeypatch.setitem(sys.modules, "our_locators", ours) + listed = ("IPythonCacheLocator,our_locators.OurLocator" if placement.endswith("too") + else "our_locators.OurLocator,IPythonCacheLocator") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", listed, raising=False) + else: + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "IPythonCacheLocator", raising=False) + passed_over = ", and OurLocator, listed too, passed the file over" if "none of numba's" in placement else "" + with pytest.raises(RuntimeError, match="no locator available") as raised: + configurations.check_cache_location(py_file, configurations.LONGEST_CACHE_FILE_NAME) + assert configurations.is_a_cache_error(raised.value) + remedy = configurations.cache_remedy(py_file, raised.value, "silence") + if ipython == "cannot make its directory": + assert remedy == ( + f"numba caches this file in {os.path.join(ipython_dir, 'numba_cache')}, which it could not make or write " + f"in{passed_over}: " + f"make it writable, or put IPython's cache directory, {ipython_dir}, at a shorter path; or silence"), remedy + else: + assert remedy == ( + "numba caches this file in numba_cache under IPython's cache directory, which IPython raised for, under a " + f"home it cannot write{passed_over}: make that home writable; or silence"), remedy + + +@from_061 +def test_the_check_raises_the_stamps_error_naming_the_archive_a_moved_zips_members_code_names(tmp_path, monkeypatch): + # numba reads the source's stamp at decoration, the archive's for a .zip + # member, and the check reads it as numba does, so for a member whose code + # names an archive since moved it raises FileNotFoundError naming that + # archive, no location of numba's, before any location is tried; the + # check's docstring excepted inspect's error alone from naming the + # location numba picked. + import numba + import numbox.core.configurations as configurations + + class UserCacheUnderTmp: + def __init__(self, appname, appauthor): + self.user_cache_dir = str(tmp_path / "user-cache" / "numba") + + monkeypatch.setattr(configurations, "AppDirs", UserCacheUnderTmp) + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + archive = tmp_path / "build" / "numbox.zip" + py_file = str(archive / "numbox" / "module.py") + with pytest.raises(FileNotFoundError) as raised: + configurations.check_cache_location(py_file, configurations.LONGEST_CACHE_FILE_NAME) + assert raised.value.filename == str(archive) + assert configurations.is_a_cache_error(raised.value) + assert not (tmp_path / "user-cache").exists() + remedy = configurations.cache_remedy(py_file, raised.value, "silence") + # Members compiled to name the archive's path now cache under numba 0.67 + # and earlier; from 0.68 the archive must hold the sources too. + recompile = "" if numba_version >= 68 else "compile the archive's .pyc members to name its path now, or " + assert remedy == ( + f"the module's code names {archive}, which is not there: its .pyc was compiled to name that path, so " + f"{recompile}ship its source files; or silence"), remedy + + +A_NAME_TOO_LONG = { + "a .zip member, numba's order": [from_061, on_windows_under_numba_0_68_0], + "on disk, NUMBA_CACHE_DIR set, the user-provided locator alone": from_062, + "on disk, numba's order, the __pycache__ unwritable": needs_a_directory_it_cannot_write, + "on disk, an error naming the location under NUMBA_CACHE_DIR, the user-provided locator alone": from_062, + "on disk under a directory named for a .zip, a caller's no-locator error, the .zip locator then the user-wide one": + from_062, + "a frozen application": [], + "a frozen application, a name that just fits, the user's cache directory too deep": [], + "on disk, an error naming the location under NUMBA_CACHE_DIR as an invalid name, the user-provided locator alone": + from_062, + "a .zip member, an error naming the location as an invalid name": from_061, + "on disk, NUMBA_CACHE_DIR set, the user-provided locator alone, a name of CJK characters": [ + from_062, pytest.mark.skipif(os.name == "nt" or sys.platform == "darwin", + reason="Windows and macOS count a name in UTF-16 units, where it fits")], + "a .zip member, numba's order, a name of astral characters, counted as macOS counts": [ + from_061, on_windows_under_numba_0_68_0], +} + + +@pytest.mark.parametrize("case", [pytest.param(case, marks=marks, id=case) for case, marks in A_NAME_TOO_LONG.items()]) +def test_a_source_directory_whose_name_is_too_long_for_numbas_cache_directory_name_is_told_a_shorter_name( + tmp_path, monkeypatch, case): + # numba names the directory it caches a file in, under NUMBA_CACHE_DIR or + # the user's cache directory, after the source's directory, by its name + # and a hash of its path, forty-one bytes more, and a name longer than the + # file system takes can be made at no path: numba passes the location + # over, or the first save dies there for a .zip member, and the remedy + # asked for the user's cache directory at a shorter path, or a writable + # NUMBA_CACHE_DIR at a short path, which cure nothing there. It tells the + # name and its length, and asks for the file in a directory of a shorter + # name, with the __pycache__ beside a source on disk, which the in-tree + # locator names after nothing, made writable or that locator listed; a + # name that just fits is told as before. The name is counted as the + # file system counts it, in bytes, or in UTF-16 units on Windows and + # macOS, where a name of CJK characters that fits was counted in bytes + # and told too long; Windows refuses a name too long as an invalid one, + # errno EINVAL, and the check runs whatever the errno. A name of astral + # characters, each one code point, two UTF-16 units and four bytes, is + # too long in units where it fits in code points: the count by units + # was held by no test, and one by code points passed them all. + import numba + from numba.core import caching + import numbox.core.configurations as configurations + from numba.core.caching import _CacheLocator + + just_fits = "just fits" in case + if "astral" in case: + monkeypatch.setattr(sys, "platform", "darwin") + limit = os.statvfs(tmp_path).f_namemax if hasattr(os, "statvfs") else 255 + too_deep = tmp_path / ("x" * 300) / "numba" if "too deep" in case else None + + class UserCacheUnderTmp: + def __init__(self, appname, appauthor): + self.user_cache_dir = str(too_deep or tmp_path / "user-cache" / "numba") + + monkeypatch.setattr(caching, "AppDirs", UserCacheUnderTmp) + monkeypatch.setattr(configurations, "AppDirs", UserCacheUnderTmp) + monkeypatch.setattr(sys, "frozen", case.startswith("a frozen"), raising=False) + cache_dir = str(tmp_path / "cache") if "NUMBA_CACHE_DIR" in case else "" + if cache_dir: + os.mkdir(cache_dir) + monkeypatch.setattr(numba.config, "CACHE_DIR", cache_dir) + listed = "UserProvidedCacheLocator" if "user-provided locator alone" in case else "" + if "the .zip locator then the user-wide one" in case: + listed = "ZipCacheLocator,UserWideCacheLocator" + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", listed, raising=False) + directory = "d" * (limit - 41 if just_fits else limit - 35) + if "CJK" in case: + directory = "\u5b57" * ((limit - 41) // 3 + 1) + if "astral" in case: + directory = "\U0001f600" * ((limit - 41) // 2 + 1) + if case.startswith("a .zip"): + archive = tmp_path / "app.zip" + with zipfile.ZipFile(archive, "w") as zipped: + zipped.writestr(f"{directory}/module.py", "x = 1\n") + py_file = str(archive / directory / "module.py") + elif case.startswith("on disk"): + site = tmp_path / "site" / "a.zip" if "named for a .zip" in case else tmp_path / "site" + py_file = str(site / directory / "module.py") + os.makedirs(os.path.dirname(py_file)) + with open(py_file, "w", encoding="utf-8") as handle: + handle.write("x = 1\n") + else: + py_file = str(tmp_path / "frozen" / directory / "module.py") + subpath = _CacheLocator.get_suitable_cache_subpath(py_file) + counted_in = "characters" if os.name == "nt" or sys.platform == "darwin" else "bytes" + length = len(subpath.encode("utf-16-le")) // 2 if counted_in == "characters" else len(subpath.encode()) + assert length == limit if just_fits else length > limit + if "CJK" in case or "astral" in case: + assert len(subpath) < limit + clause = ( + f"numba names the directory it caches this file in after the source's directory, {subpath}, {length} " + f"{counted_in}, which is longer than a name can be on that file system ({limit})") + if "unwritable" in case: + Path(py_file).parent.chmod(0o555) + try: + if "an error naming" in case: + refused = ((errno.EINVAL, "Invalid argument") if "invalid name" in case + else (errno.ENAMETOOLONG, "File name too long")) + named = os.path.join(cache_dir or str(tmp_path / "user-cache" / "numba"), subpath) + failure = OSError(*refused, named) + elif "a caller's no-locator error" in case: + # numba's .zip locator takes the path without trying its + # location, so numba raises no no-locator error here, and a + # caller's reached a class listed after that locator, unreached, + # for its location and raised KeyError. + failure = RuntimeError(f"cannot cache function '_cache_probe': no locator available for file '{py_file}'") + else: + expected = OSError if case.startswith("a .zip") else RuntimeError + with pytest.raises(expected) as raised: + configurations.check_cache_location(py_file, configurations.LONGEST_CACHE_FILE_NAME) + failure = raised.value + assert configurations.is_a_cache_error(failure) + remedy = configurations.cache_remedy(py_file, failure, "silence") + finally: + if "unwritable" in case: + Path(py_file).parent.chmod(0o755) + if just_fits: + assert remedy == ( + "numba caches a .zip, or a frozen application, in the user's cache directory, and NUMBA_CACHE_DIR has no " + f"effect here, because the source is not a file on disk: numba could not use that directory, {too_deep}: " + f"make it writable, or put it at a shorter path{configurations._moved_through()}, or silence"), remedy + elif case.startswith("on disk, numba's order"): + assert failure.filename is None if isinstance(failure, OSError) else True + assert remedy == ( + f"{clause}: put the file in a directory of a shorter name, or make the __pycache__ beside the source, " + f"{os.path.join(os.path.dirname(py_file), '__pycache__')}, writable, where InTreeCacheLocator caches it " + "under no such name; " + "or silence"), remedy + elif case.startswith("on disk"): + # The user-provided locator alone, with the variable unset, took no + # location and measured no name; the variable it takes nothing + # without is named beside the shorter name it would then need. + variable = (", with NUMBA_CACHE_DIR set, which UserProvidedCacheLocator takes nothing without" + if listed == "UserProvidedCacheLocator" and not cache_dir else "") + assert remedy == ( + f"{clause}: put the file in a directory of a shorter name{variable}, or list InTreeCacheLocator, which " + "caches beside the source, in its __pycache__, under no such name; or silence"), remedy + else: + assert remedy == ( + "numba caches a .zip, or a frozen application, in the user's cache directory, and NUMBA_CACHE_DIR has no " + f"effect here, because the source is not a file on disk: {clause}, so put the file in a directory of a " + "shorter name, or silence"), remedy + + +@pytest.mark.skipif(os.name == "nt", reason="Windows decodes any byte of a path") +@pytest.mark.parametrize("placement", [ + "an .egg member, numba's order", + pytest.param("on disk, the in-tree locator alone, its __pycache__ unwritable", marks=[ + from_062, needs_a_directory_it_cannot_write, + pytest.mark.skipif(sys.platform == "darwin", reason="APFS takes no name that is not UTF-8")]), +]) +def test_a_source_whose_path_holds_bytes_that_are_not_utf8_is_answered_where_numba_hashes_no_path( + tmp_path, monkeypatch, placement): + # numba names the directory it caches a file in under NUMBA_CACHE_DIR or + # the user's cache directory with a SHA-1 of the source's directory path + # encoded as UTF-8, and raises UnicodeEncodeError for a path holding bytes + # that are not, decoded with surrogates, where a locator that names its + # directory so reaches the file; the remedy computed that name for every + # file and raised the same where numba hashed no path, an .egg member no + # locator takes and a source on disk under the in-tree locator alone, and + # the import died where upstream main warned. + import numba + import numbox.core.configurations as configurations + + directory = tmp_path / os.fsdecode(b"caf\xe9") + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + if placement.startswith("an .egg"): + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + py_file = str(directory / "numbox-0.0.0-py3.12.egg" / "numbox" / "module.py") + failure = RuntimeError(f"cannot cache function 'f': no locator available for file '{py_file}'") + expected = f"{ARCHIVE_REMEDY}; or silence" + else: + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "InTreeCacheLocator", raising=False) + (directory / "package").mkdir(parents=True) + py_file = str(directory / "package" / "module.py") + with open(py_file, "w", encoding="utf-8") as handle: + handle.write("x = 1\n") + (directory / "package").chmod(0o555) + try: + with pytest.raises(RuntimeError, match="no locator available") as raised: + configurations.check_cache_location(py_file, configurations.LONGEST_CACHE_FILE_NAME) + finally: + (directory / "package").chmod(0o755) + failure = raised.value + expected = ("numba looks only where NUMBA_CACHE_LOCATOR_CLASSES, InTreeCacheLocator, says: make one of those " + "locations writable, or silence") + with pytest.raises(UnicodeEncodeError): + py_file.encode() + assert configurations.is_a_cache_error(failure) + assert configurations.cache_remedy(py_file, failure, "silence") == expected + + +@pytest.mark.parametrize("ipython", ["giving its directory", "giving a temporary directory per call"]) +def test_a_cell_whose_source_linecache_does_not_hold_raises_inspects_error_as_numba_does(tmp_path, monkeypatch, ipython): + # numba's IPython locator reads a cell's source from linecache, where + # IPython holds it while it runs the cell, and inspect raises, with no + # errno and no name, for a cell linecache does not hold, before the + # locator tries its location; the check gave the probe's source to every + # file the locator takes by name and returned for such a cell, where numba + # raises at decoration, and the remedy told the location numba took as + # refusing a file, or, under an IPython giving temporary directories, + # blamed the directory the locator never tried. + import linecache + import tempfile + import types + import numba + import numbox.core.configurations as configurations + + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + ipython_dir = tmp_path / "ipython" + paths = types.ModuleType("IPython.paths") + temporary = ipython != "giving its directory" + paths.get_ipython_cache_dir = (lambda: tempfile.mkdtemp(dir=tmp_path)) if temporary else (lambda: str(ipython_dir)) + monkeypatch.setitem(sys.modules, "IPython", types.ModuleType("IPython")) + monkeypatch.setitem(sys.modules, "IPython.paths", paths) + cell = "" + linecache.cache.pop(cell, None) + with pytest.raises(OSError, match="could not get source code") as raised: + configurations.check_cache_location(cell, configurations.LONGEST_CACHE_FILE_NAME) + assert raised.value.errno is None and raised.value.filename is None + assert temporary or not ipython_dir.exists() + assert configurations.is_a_cache_error(raised.value) + remedy = configurations.cache_remedy(cell, raised.value, "silence") + assert remedy == ( + f"numba's IPython locator reads a cell's source from linecache, which holds none for {cell} (could not get " + "source code), and numba caches a cell only while IPython holds its source; or silence"), remedy + + +@pytest.mark.parametrize("ipython", ["giving its directory", "giving a temporary directory per call"]) +@pytest.mark.parametrize("name", ["bundle.zip", "bundle.egg"]) +def test_a_zip_member_in_an_ipykernel_directory_compiled_to_pyc_alone_raises_inspects_error_as_numba_does( + tmp_path, monkeypatch, name, ipython): + # numba's IPython locator reads the function's source through the module's + # loader for a .zip member, and zipimport gives none for a member compiled + # to .pyc alone, so numba raises inspect's error at decoration; the check + # gave the probe's source to every member a locator of that family takes + # and returned, where upstream main's check raised as numba does, and the + # remedy told the location numba took as refusing a file where the + # archive's remedy, its source files to ship, is the one, and, under an + # IPython giving temporary directories, blamed the directory the locator + # never tried. + import linecache + import tempfile + import types + import numba + import numbox.core.configurations as configurations + + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + ipython_dir = tmp_path / "ipython" + paths = types.ModuleType("IPython.paths") + temporary = ipython != "giving its directory" + paths.get_ipython_cache_dir = (lambda: tempfile.mkdtemp(dir=tmp_path)) if temporary else (lambda: str(ipython_dir)) + monkeypatch.setitem(sys.modules, "IPython", types.ModuleType("IPython")) + monkeypatch.setitem(sys.modules, "IPython.paths", paths) + source = tmp_path / "cell.py" + source.write_text("def cell():\n pass\n") + archive = tmp_path / name + compiled = py_compile.compile(str(source), cfile=str(tmp_path / "cell.pyc"), + dfile=str(archive / "ipykernel_123" / "cell.py"), doraise=True) + with zipfile.ZipFile(archive, "w") as zipped: + zipped.write(compiled, "ipykernel_123/cell.pyc") + py_file = str(archive / "ipykernel_123" / "cell.py") + linecache.cache.pop(py_file, None) + with pytest.raises(OSError, match="could not get source code") as raised: + configurations.check_cache_location(py_file, configurations.LONGEST_CACHE_FILE_NAME) + assert raised.value.errno is None and raised.value.filename is None + # linecache kept the probe's loader under the member's name, to ask again + # for a file whose source it found none of; the check takes it out. + assert py_file not in linecache.cache + assert temporary or not ipython_dir.exists() + assert configurations.is_a_cache_error(raised.value) + remedy = configurations.cache_remedy(py_file, raised.value, "silence") + if name.endswith(".egg") and numba_version >= 61: + # An archive numba's .zip locator never takes is told to be renamed + # to end in .zip, with its source files in it; before 0.61 numba has + # no .zip locator, and no archive caches. + assert remedy == ( + "NUMBA_CACHE_DIR has no effect here, because the source is not a file on disk: to cache, install numbox " + f"with its source files on disk, unpacked from any archive, or rename the archive, {archive}, to end in " + ".zip, lower-case, which numba's .zip locator looks for in the path, with its source files in it, so that " + "numba 0.61 and later cache it in the user's cache directory; or silence"), remedy + else: + assert remedy == f"{ARCHIVE_REMEDY}; or silence", remedy + + +A_ZIP_REFUSED = { + "unwritable": ((errno.EACCES, "Permission denied"), + "no file can be written in that directory, {location} (Permission denied), so make room there, or " + "make it writable"), + "too long": ((errno.ENAMETOOLONG, "File name too long"), + "the path is too long for the file system, so put that directory, {location}, at a shorter " + "path{moved_through}"), +} + + +@from_062 +@pytest.mark.parametrize("refused", list(A_ZIP_REFUSED)) +def test_a_zip_import_refused_its_location_under_a_list_without_the_user_wide_locator_is_told_the_location( + tmp_path, monkeypatch, refused): + # The remedy that asks for the user-wide locator to be listed is for + # numba's no-locator error, which a frozen application gets under a list + # without that locator. A .zip's error names the location numba took + # through its .zip locator, which such a list can hold, and the remedy is + # that location made writable or put at a shorter path, as under numba's + # own order; held to any error under such a list, the frozen remedy would + # tell a .zip's reader to list a locator numba takes no .zip through. + import numba + import numbox.core.configurations as configurations + from numba.core.caching import _CacheLocator + + class UserCacheUnderTmp: + def __init__(self, appname, appauthor): + self.user_cache_dir = str(tmp_path / "user-cache" / "numba") + + monkeypatch.setattr(configurations, "AppDirs", UserCacheUnderTmp) + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "InTreeCacheLocator,ZipCacheLocator", raising=False) + py_file = str(tmp_path / "bundle.zip" / "package" / "module.py") + location = tmp_path / "user-cache" / "numba" / _CacheLocator.get_suitable_cache_subpath(py_file) + error, told = A_ZIP_REFUSED[refused] + # The location is there and refuses, as a directory made can; one not + # there is told the directory it cannot be made under as well. + location.mkdir(parents=True) + remedy = configurations.cache_remedy(py_file, OSError(*error, str(location)), "silence") + told = told.format(location=location, moved_through=configurations._moved_through()) + assert remedy == ( + "numba caches a .zip, or a frozen application, in the user's cache directory, and NUMBA_CACHE_DIR has no " + f"effect here, because the source is not a file on disk: {told}, or silence"), remedy + + +@from_061 +@pytest.mark.parametrize("refused", list(A_ZIP_REFUSED)) +def test_a_zip_error_naming_a_file_numba_writes_in_its_location_is_told_the_location(tmp_path, monkeypatch, refused): + # The package's own check names the location numba took for a .zip, a + # directory of its own under the user's cache directory; numba's first + # save names the file it was writing there, under a temporary name, and a + # caller can pass that error on. The remedy quoted the file as the + # directory to make writable or to put at a shorter path, which the + # remedy for a source on disk, matching the file's directory too, did not. + import numba + import numbox.core.configurations as configurations + from numba.core.caching import _CacheLocator + + class UserCacheUnderTmp: + def __init__(self, appname, appauthor): + self.user_cache_dir = str(tmp_path / "user-cache" / "numba") + + monkeypatch.setattr(configurations, "AppDirs", UserCacheUnderTmp) + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + py_file = str(tmp_path / "bundle.zip" / "package" / "module.py") + location = tmp_path / "user-cache" / "numba" / _CacheLocator.get_suitable_cache_subpath(py_file) + named = location / "module-1.py312.nbi.tmp.0123456789abcdef" + error, told = A_ZIP_REFUSED[refused] + remedy = configurations.cache_remedy(py_file, OSError(*error, str(named)), "silence") + told = told.format(location=location, moved_through=configurations._moved_through()) + assert remedy == ( + "numba caches a .zip, or a frozen application, in the user's cache directory, and NUMBA_CACHE_DIR has no " + f"effect here, because the source is not a file on disk: {told}, or silence"), remedy + + +@pytest.mark.parametrize("refused", list(A_ZIP_REFUSED)) +def test_a_frozen_applications_error_naming_a_file_numba_writes_in_its_location_is_told_the_location( + tmp_path, monkeypatch, refused): + # numba's user-wide locator takes a frozen application's file, which is + # not on disk, besides any file on disk, and caches it in a directory of + # its own under the user's cache directory, as the .zip locator does a + # .zip; numba's first save names the file it was writing there, and the + # remedy is the directory, to make writable or put at a shorter path, as + # for a .zip. Nothing held the remedy to that locator taking the file: + # with the frozen application left to the locators for a file on disk, no + # locator took it, and the error was answered with the list, the archive + # remedy, instead of the location numba took. + import numba + import numbox.core.configurations as configurations + from numba.core.caching import _CacheLocator + + class UserCacheUnderTmp: + def __init__(self, appname, appauthor): + self.user_cache_dir = str(tmp_path / "user-cache" / "numba") + + monkeypatch.setattr(configurations, "AppDirs", UserCacheUnderTmp) + monkeypatch.setattr(sys, "frozen", True, raising=False) + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + py_file = str(tmp_path / "frozen" / "numbox" / "module.py") + location = tmp_path / "user-cache" / "numba" / _CacheLocator.get_suitable_cache_subpath(py_file) + named = location / "module-1.py312.nbi.tmp.0123456789abcdef" + error, told = A_ZIP_REFUSED[refused] + remedy = configurations.cache_remedy(py_file, OSError(*error, str(named)), "silence") + told = told.format(location=location, moved_through=configurations._moved_through()) + assert remedy == ( + "numba caches a .zip, or a frozen application, in the user's cache directory, and NUMBA_CACHE_DIR has no " + f"effect here, because the source is not a file on disk: {told}, or silence"), remedy + + +def test_a_file_not_found_error_that_names_no_file_for_a_source_not_on_disk_gets_the_archive_remedy(tmp_path, monkeypatch): + # numba's errors name the file they are about, and so does the package's + # own check, but a caller can pass on a FileNotFoundError that names none. + # The check for a moved archive, which asks whether the source lies under + # the file the error names, added os.sep to that name before asking, and + # raised TypeError on None instead of returning a remedy. + import numba + import numbox.core.configurations as configurations + + class UserCacheUnderTmp: + def __init__(self, appname, appauthor): + self.user_cache_dir = str(tmp_path / "user-cache" / "numba") + + monkeypatch.setattr(configurations, "AppDirs", UserCacheUnderTmp) + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + py_file = str(tmp_path / "site" / "package" / "module.py") + remedy = configurations.cache_remedy(py_file, FileNotFoundError(errno.ENOENT, "No such file or directory"), "silence") + # No locator takes a source neither on disk, in a .zip nor a frozen + # application's, so the error gets the remedy numba's no-locator one does. + assert remedy == f"{ARCHIVE_REMEDY}; or silence", remedy + + +def test_a_zip_member_whose_source_the_archive_does_not_hold_is_told_the_source_files_naming_the_member( + tmp_path, monkeypatch): + # numba 0.68 and later read a member's source from the archive for the + # stamp, and zipfile's KeyError names the member it did not find, a module + # shipped as .pyc alone: the error was not the cache's, so the import died + # at the first decorated function, and the archive remedy told the source + # files without the member. + import numba + import numbox.core.configurations as configurations + + class UserCacheUnderTmp: + def __init__(self, appname, appauthor): + self.user_cache_dir = str(tmp_path / "user-cache" / "numba") + + monkeypatch.setattr(configurations, "AppDirs", UserCacheUnderTmp) + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + py_file = str(tmp_path / "numbox.zip" / "numbox" / "core" / "bindings" / "libm.py") + failure = KeyError("There is no item named 'numbox/core/bindings/libm.py' in the archive") + assert configurations.is_a_cache_error(failure) == (numba_version >= 68) + remedy = configurations.cache_remedy(py_file, failure, "silence") + assert remedy == ( + "numba 0.68 and later read a .zip member's source from the archive, and this one holds none under " + "numbox/core/bindings/libm.py, the module shipped as .pyc alone: to cache, install numbox with its source " + "files on disk, unpacked from any archive, or import it from a .zip holding its source files, which numba " + "0.61 and later cache in the user's cache directory; or silence"), remedy + + +def test_a_zip_members_name_built_with_windows_separator_is_told_numba_0_68_1(tmp_path, monkeypatch): + # numba 0.68.0 builds the member's name with the platform's separator, so + # on Windows no archive holds it (numba issue 10889, fixed in 0.68.1): the + # archive's user was told to add source files that were there. + import numba + import numbox.core.configurations as configurations + + class UserCacheUnderTmp: + def __init__(self, appname, appauthor): + self.user_cache_dir = str(tmp_path / "user-cache" / "numba") + + monkeypatch.setattr(configurations, "AppDirs", UserCacheUnderTmp) + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + monkeypatch.setattr(os, "name", "nt") + archive = tmp_path / "numbox.zip" + with zipfile.ZipFile(archive, "w") as zipped: + zipped.writestr("numbox/core/bindings/libm.py", "") + py_file = str(archive / "numbox" / "core" / "bindings" / "libm.py") + # zipfile quotes the name as Python does, each backslash doubled. + failure = KeyError("There is no item named 'numbox\\\\core\\\\bindings\\\\libm.py' in the archive") + assert configurations.is_a_cache_error(failure) == (numba_version >= 68) + remedy = configurations.cache_remedy(py_file, failure, "silence") + assert remedy == ( + "numba 0.68.0 reads a .zip member's source by a name built with Windows's separator, " + "numbox\\core\\bindings\\libm.py, which no archive holds (numba issue " + "https://github.com/numba/numba/issues/10889, fixed for numba 0.68.1): to cache, install numbox with its " + "source files on disk, unpacked from any archive, which no separator touches, or a numba release that " + "carries the fix, which reads the member as the archive names it; or silence"), remedy + + +def test_a_pyc_alone_in_a_zip_on_windows_is_told_the_source_files_not_numba_0_68_1(tmp_path, monkeypatch): + # The separator is in the name numba 0.68.0 builds for every member on + # Windows, a module shipped as .pyc alone among them, and that one numba + # 0.68.1 cannot cache either, the archive holding no source under the name + # it reads; the remedy read the backslash alone and promised that release. + import numba + import numbox.core.configurations as configurations + + class UserCacheUnderTmp: + def __init__(self, appname, appauthor): + self.user_cache_dir = str(tmp_path / "user-cache" / "numba") + + monkeypatch.setattr(configurations, "AppDirs", UserCacheUnderTmp) + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + monkeypatch.setattr(os, "name", "nt") + archive = tmp_path / "numbox.zip" + with zipfile.ZipFile(archive, "w") as zipped: + zipped.writestr("numbox/core/bindings/libm.pyc", b"") + py_file = str(archive / "numbox" / "core" / "bindings" / "libm.py") + failure = KeyError("There is no item named 'numbox\\\\core\\\\bindings\\\\libm.py' in the archive") + remedy = configurations.cache_remedy(py_file, failure, "silence") + assert remedy == ( + "numba 0.68 and later read a .zip member's source from the archive, and this one holds none under " + "numbox/core/bindings/libm.py, the module shipped as .pyc alone: to cache, install numbox with its source " + "files on disk, unpacked from any archive, or import it from a .zip holding its source files, which numba " + "0.61 and later cache in the user's cache directory; or silence"), remedy + + +@pytest.mark.skipif(numba_version < 61, reason="numba's .zip locator arrived in 0.61") +def test_a_directory_named_zip_above_the_archive_is_told_as_the_directory_from_numba_0_68(tmp_path, monkeypatch): + # numba takes the first part of the path ending in .zip for the archive, a + # directory so named above it included, and from 0.68 opens it to read the + # member's source, which the directory refuses; the remedy told the reader + # to make room in the directory, or make it writable. + import numba + import numbox.core.configurations as configurations + + class UserCacheUnderTmp: + def __init__(self, appname, appauthor): + self.user_cache_dir = str(tmp_path / "user-cache" / "numba") + + monkeypatch.setattr(configurations, "AppDirs", UserCacheUnderTmp) + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + container = tmp_path / "container.zip" + container.mkdir() + py_file = str(container / "numbox.zip" / "numbox" / "core" / "configurations.py") + failure = IsADirectoryError(errno.EISDIR, "Is a directory", str(container)) + assert configurations.is_a_cache_error(failure) + remedy = configurations.cache_remedy(py_file, failure, "silence") + assert remedy == ( + "NUMBA_CACHE_DIR has no effect here, because the source is not a file on disk: numba takes the first part of " + f"the path ending in .zip, {container}, for the archive, and it is a directory, which numba 0.68 and later " + "open to read the member's source: put the archive under a path with no such directory above it, or silence" + ), remedy + + +SUBCLASSED_LOCATORS = ( + "from numba.core.caching import UserProvidedCacheLocator, ZipCacheLocator\n" + "class OurUserProvided(UserProvidedCacheLocator):\n pass\n" + "class OurZip(ZipCacheLocator):\n pass\n" +) + +# Each case: NUMBA_CACHE_LOCATOR_CLASSES, whether NUMBA_CACHE_DIR is set, the +# source's directory, which location the error names ("none" for numba's +# no-locator error), and the remedy before "; or silence" or ", or silence". +SUBCLASS_CASES = { + "the user-provided subclass took NUMBA_CACHE_DIR's location": ( + "our_locators.OurUserProvided,InTreeCacheLocator,UserWideCacheLocator", True, "site", "NUMBA_CACHE_DIR", + "the path is too long for the file system: a shorter NUMBA_CACHE_DIR"), + "the .zip subclass took the user's cache directory's location": ( + "our_locators.OurZip,UserProvidedCacheLocator,InTreeCacheLocator,UserWideCacheLocator", True, "bundle.zip", + "user", "the path is too long for the file system: the user's cache directory, {user_cache_dir}, at a " + "shorter path{moved_through}"), + "the user-provided subclass before the in-tree locator that took the location": ( + "our_locators.OurUserProvided,InTreeCacheLocator", True, "site", "beside", + "the path is too long for the file system: numbox installed at a shorter path, or NUMBA_CACHE_DIR, which " + "is set to {cache_dir} and numba could not use, made a writable directory at a short path"), + "no locator, the user-provided subclass passed over": ( + "our_locators.OurUserProvided,InTreeCacheLocator", True, "site", "none", + "NUMBA_CACHE_DIR is set to {cache_dir}, which numba could not use: set it to a writable directory at a short " + "path"), +} + + +@pytest.mark.skipif(numba_version < 62, reason="NUMBA_CACHE_LOCATOR_CLASSES arrived in numba 0.62") +@pytest.mark.parametrize("case", list(SUBCLASS_CASES)) +def test_a_subclass_of_one_of_numbas_locators_is_told_as_its_parent_is(tmp_path, monkeypatch, case): + # NUMBA_CACHE_LOCATOR_CLASSES takes a dotted path to any class, and a + # subclass of one of numba's locators caches where its parent does; numba + # takes it as it takes the parent. The remedy knew a subclass of the + # in-tree locator for one, and matched the user-provided and .zip + # locators by identity, so their subclasses got "that location, X" with no + # cure, and the variable the subclass reads counted as unread. + import numba + import numbox.core.configurations as configurations + from numba.core.caching import _CacheLocator + + class UserCacheUnderTmp: + def __init__(self, appname, appauthor): + self.user_cache_dir = str(tmp_path / "user-cache" / "numba") + + (tmp_path / "our_locators.py").write_text(SUBCLASSED_LOCATORS) + monkeypatch.syspath_prepend(str(tmp_path)) + monkeypatch.delitem(sys.modules, "our_locators", raising=False) + monkeypatch.setattr(configurations, "AppDirs", UserCacheUnderTmp) + locators, cache_dir_set, source_dir, which, expected = SUBCLASS_CASES[case] + cache_dir = tmp_path / "cache" + user_cache_dir = tmp_path / "user-cache" / "numba" + monkeypatch.setattr(numba.config, "CACHE_DIR", str(cache_dir) if cache_dir_set else "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", locators, raising=False) + py_file = tmp_path / source_dir / "package" / "module.py" + py_file.parent.mkdir(parents=True) + py_file.write_text("") + subpath = _CacheLocator.get_suitable_cache_subpath(str(py_file)) + if which == "none": + failure = RuntimeError(f"cannot cache function '_cache_probe': no locator available for file '{py_file}'") + ending = ", or silence" + else: + location = { + "NUMBA_CACHE_DIR": cache_dir / subpath, "beside": py_file.parent / "__pycache__", "user": user_cache_dir / subpath, + }[which] + failure = OSError(errno.ENAMETOOLONG, "File name too long", str(location)) + ending = "; or silence" + remedy = configurations.cache_remedy(str(py_file), failure, "silence") + expected = expected.format(cache_dir=cache_dir, user_cache_dir=user_cache_dir, + moved_through=configurations._moved_through()) + assert remedy == f"{expected}{ending}", remedy + + +@pytest.mark.parametrize("given", ["Path", "bytes"]) +@pytest.mark.parametrize("placement", ["no locator", "too long", "archive"]) +def test_a_source_given_as_a_path_object_or_bytes_gets_the_remedy_its_string_gets( + tmp_path, monkeypatch, placement, given): + # check_cache_location takes any path-like through os.fspath, and the + # remedy took one too until it asked '".zip" in py_file' of it, which a + # Path cannot answer: TypeError for every source on disk under numba's own + # order, and AttributeError at 'py_file.startswith' for a source not on disk. + # A bytes path, which check_cache_location takes as well, raised TypeError + # at the first 'startswith' asked of it with a str, os.fspath passing bytes + # through as bytes. + import numba + import numbox.core.configurations as configurations + + class UserCacheUnderTmp: + def __init__(self, appname, appauthor): + self.user_cache_dir = str(tmp_path / "user-cache" / "numba") + + monkeypatch.setattr(configurations, "AppDirs", UserCacheUnderTmp) + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + if placement == "archive": + py_file = tmp_path / "bundle.zip" / "package" / "module.py" + failure = FileNotFoundError(errno.ENOENT, "No such file or directory", str(tmp_path / "bundle.zip")) + else: + py_file = tmp_path / "site" / "package" / "module.py" + py_file.parent.mkdir(parents=True) + py_file.write_text("") + if placement == "too long": + failure = OSError(errno.ENAMETOOLONG, "File name too long", str(py_file.parent / "__pycache__")) + else: + failure = RuntimeError(f"cannot cache function '_cache_probe': no locator available for file '{py_file}'") + as_given = py_file if given == "Path" else os.fsencode(py_file) + as_a_string = configurations.cache_remedy(str(py_file), failure, "silence") + assert configurations.cache_remedy(as_given, failure, "silence") == as_a_string + + +@pytest.mark.parametrize("given", ["Path", "bytes"]) +@pytest.mark.parametrize("placement", ["too long", "unwritable", "a .zip's location", "a moved archive"]) +def test_an_error_naming_its_file_as_a_path_object_or_bytes_gets_the_remedy_a_string_name_gets( + tmp_path, monkeypatch, placement, given): + # An error names the file it refused as the call that raised it was given + # it, bytes for a call given bytes, or as a caller built it, a path object + # too, and the remedy read the name as a string: bytes matched none of + # the locations, which the remedy writes as strings, and were told as a + # location that is none of numba's, and for a source not on disk the + # check for a moved archive, adding os.sep to the name, raised TypeError + # on bytes and on a path object alike. + import numba + import numbox.core.configurations as configurations + from numba.core.caching import _CacheLocator + + class UserCacheUnderTmp: + def __init__(self, appname, appauthor): + self.user_cache_dir = str(tmp_path / "user-cache" / "numba") + + monkeypatch.setattr(configurations, "AppDirs", UserCacheUnderTmp) + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + refused = (errno.EACCES, "Permission denied") + if placement in ("a .zip's location", "a moved archive"): + py_file = tmp_path / "bundle.zip" / "package" / "module.py" + if placement == "a moved archive": + error, named = (errno.ENOENT, "No such file or directory"), tmp_path / "bundle.zip" + else: + subpath = _CacheLocator.get_suitable_cache_subpath(str(py_file)) + error, named = refused, tmp_path / "user-cache" / "numba" / subpath + else: + py_file = tmp_path / "site" / "package" / "module.py" + py_file.parent.mkdir(parents=True) + py_file.write_text("") + error = (errno.ENAMETOOLONG, "File name too long") if placement == "too long" else refused + named = py_file.parent / "__pycache__" + as_given = named if given == "Path" else os.fsencode(named) + as_a_string = configurations.cache_remedy(str(py_file), OSError(*error, str(named)), "silence") + assert configurations.cache_remedy(str(py_file), OSError(*error, as_given), "silence") == as_a_string + + +@pytest.mark.parametrize("placement", ["too long", "archive"]) +def test_the_remedies_that_name_a_package_name_the_one_given(tmp_path, monkeypatch, placement): + # A package built on numbox puts the question for its own files and takes + # the remedy from cache_remedy; the two remedies that tell the reader to + # install a package again named numbox whichever package had asked. + import numba + from numbox.core.configurations import cache_remedy + if placement == "too long": + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + py_file = tmp_path / "ducklib.py" + py_file.write_text("") + failure = OSError(errno.ENAMETOOLONG, "File name too long", str(tmp_path / "__pycache__")) + named = "{} installed at a shorter path" + else: + py_file = tmp_path / "numbduck-0.0.0-py3-none-any.whl" / "numbduck" / "ducklib.py" + failure = RuntimeError(f"cannot cache function 'f': no locator available for file '{py_file}'") + named = "install {} with its source files on disk" + remedy = cache_remedy(str(py_file), failure, "silence", package="numbduck") + assert named.format("numbduck") in remedy and "numbox" not in remedy, remedy + assert named.format("numbox") in cache_remedy(str(py_file), failure, "silence") + + +def _function_names(tree): + """Each function's qualified name as numba names its cache files, angle brackets dropped, with its line.""" + def walk(node, scope): + for child in ast.iter_child_nodes(node): + if isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef)): + yield ".".join(scope + [child.name]), child.lineno + yield from walk(child, scope + [child.name, "locals"]) + elif isinstance(child, ast.ClassDef): + yield from walk(child, scope + [child.name]) + else: + yield from walk(child, scope) + return walk(tree, []) + + +def test_numbas_longest_file_name_for_the_package_is_within_the_probes_bound(): + # The probe reserves LONGEST_CACHE_FILE_NAME bytes in a location for the + # files numba writes there: the module's stem, the function's qualified + # name, a line number, the interpreter tag with its abiflags and an index + # number, written under a 21-byte temporary name. Every function of the + # package must fit, decorated or not, and so must the builder's generated + # kernel, anchored to its file under a name of 64 hex characters. + from numbox.core.configurations import LONGEST_CACHE_FILE_NAME, _OVERLOAD_DIGITS + tag = f".py{sys.version_info[0]}{sys.version_info[1]}t.{'9' * _OVERLOAD_DIGITS}.nbc.tmp.0123456789abcdef" + names = [f"{path.stem}.{qualname}-{lineno}{tag}" + for path in sorted((REPO / "numbox").rglob("*.py")) + for qualname, lineno in _function_names(ast.parse(path.read_text(encoding="utf-8")))] + names.append(f"builder._make_{'0' * 64}-9999{tag}") + longest = max(names, key=len) + assert len(names) > 500 + assert len(longest) <= LONGEST_CACHE_FILE_NAME, longest + + +ANOTHER_ERROR_AT_THE_CACHE_SET_UP = ( + "import numba.core.caching as caching\n" + "def refuse(self, py_func):\n" + " raise RuntimeError('a locator of another kind refused')\n" + "caching.CompileResultCacheImpl.__init__ = refuse\n" + "import numbox.core.configurations\n" +) + + +def test_an_error_that_is_not_the_caches_is_raised_as_it_was(tmp_path): + # The fallback answers two errors of numba's cache set-up, no locator and + # a directory that cannot be written. A RuntimeError of another kind at + # the same step, a locator of the user's own refusing, say, is not its to + # turn into an uncached import with a remedy that does not apply. + env = dict(os.environ, NUMBA_CACHE_DIR=str(tmp_path / "cache")) + env.pop("NUMBOX_JIT_OPTIONS", None) + run = subprocess.run([sys.executable, "-W", "always", "-c", ANOTHER_ERROR_AT_THE_CACHE_SET_UP], + capture_output=True, text=True, env=env, cwd=str(tmp_path)) + assert run.returncode != 0 and "a locator of another kind refused" in run.stderr, run.stderr + assert "compiles without a cache" not in run.stderr + + +# The code numbox generates at run time, each child importing the helpers of +# the test module that exercises it. +MAKE_A_STRUCTREF = ( + "from numba.core.types import StructRef, float32\n" + "from numba.experimental.structref import register\n" + "from numbox.utils.highlevel import make_structref\n" + "@register\n" + "class TypeClass(StructRef):\n" + " pass\n" + "Struct = make_structref('Struct', {'value': float32}, TypeClass)\n" + "assert Struct(2.5).value == 2.5\n" + "print('made')\n" +) + +REGISTER_AN_AGGREGATE = ( + "from test.core.test_sqlite_udf_helpers import (_open_memory, _make_table, _read1_int64, sum_state_type,\n" + " sum_init, sum_step, sum_finalize)\n" + "from numbox.core.bindings.sqlite.udf_helpers import register_aggregate\n" + "from numbox.core.bindings.sqlite.conn import sqlite3_close\n" + "db = _open_memory()\n" + "_make_table(db, [1, 2, 3, 4, 5])\n" + "register_aggregate(db, 'my_sum', 1, sum_state_type, sum_init, sum_step, sum_finalize)\n" + "value, _ = _read1_int64(db, 'SELECT __cap(my_sum(v)) FROM t')\n" + "sqlite3_close(db)\n" + "assert value == 15, value\n" + "print('summed')\n" +) + +COMPILE_A_KERNEL = ( + "from numbox.core.variable.variable import Graph\n" + "from numbox.core.variable.compile_kernel import compile_kernel\n" + "def f(x):\n" + " return x + 1.0\n" + "graph = Graph({'calc': [{'name': 'y', 'inputs': {'x': 'ext'}, 'formula': f}]}, ['ext'])\n" + "kernel = compile_kernel(graph, 'calc.y')\n" + "assert kernel.execute({'ext': {'x': 1.0}})['calc.y'] == 2.0\n" + "print('executed')\n" +) + +BUILD_A_DERIVE = ( + "from numpy import isclose\n" + "from numbox.core.work.builder import Derived, End, make_graph\n" + "x = End(name='x', init_value=3.14)\n" + "def twice(x):\n" + " return 2 * x\n" + "y = Derived(name='y', init_value=0.0, derive=twice, sources=(x,))\n" + "access = make_graph(y)\n" + "access.y.calculate()\n" + "assert isclose(access.y.data, 6.28), access.y.data\n" + "print('derived')\n" +) + +REGISTER_A_TVF = ( + "import numpy as np\n" + "from test.core.test_sqlite_tvf import _open, _select_int, _series, _OUT\n" + "from numbox.core.bindings.sqlite.tvf import register_tvf\n" + "from numbox.core.bindings.sqlite.conn import sqlite3_close\n" + "db = _open()\n" + "handle = register_tvf(db.value, 'series', (np.int64, np.int64), _OUT, _series)\n" + "rc, rows = _select_int(db, 'SELECT n FROM series(2, 5)')\n" + "assert rc == 0 and [row[0] for row in rows] == [2, 3, 4], (rc, rows)\n" + "sqlite3_close(db.value)\n" + "print('selected')\n" +) + + +@needs_a_directory_it_cannot_write +@pytest.mark.parametrize("child, word", [ + (MAKE_A_STRUCTREF, "made"), (REGISTER_AN_AGGREGATE, "summed"), (REGISTER_A_TVF, "selected"), +], ids=["make_structref", "register_aggregate", "register_tvf"]) +def test_generated_code_compiles_uncached_where_its_anchor_cannot_be_written(tmp_path, child, word): + # The code numbox generates at run time is anchored to a file under + # NUMBA_CACHE_DIR or the user's cache directory, and the anchor was written + # whatever the cache option said. So with the tree writable, where the + # package's own functions cache, and the user's cache directory not, + # make_structref and the sqlite registrations died at the anchor's + # directory. The anchor is written to be cached from, and where it cannot + # be, the code compiles without a cache after a warning; NUMBA_CACHE_DIR at + # a writable directory cures it and holds the anchor. + home = tmp_path / "home" + home.mkdir() + home.chmod(0o555) + try: + env = dict(os.environ, PYTHONPATH=str(REPO), HOME=str(home), XDG_CACHE_HOME=str(home / "cache")) + env.pop("NUMBA_CACHE_DIR", None) + env.pop("NUMBOX_JIT_OPTIONS", None) + run = subprocess.run([sys.executable, "-W", "always", "-c", child], capture_output=True, text=True, + env=env, cwd=str(tmp_path)) + assert run.returncode == 0 and word in run.stdout, run.stderr + assert "compiles without a cache" in run.stderr and "Set NUMBA_CACHE_DIR" in run.stderr, run.stderr + cured = subprocess.run([sys.executable, "-W", "error::RuntimeWarning", "-c", child], capture_output=True, + text=True, env=dict(env, NUMBA_CACHE_DIR=str(tmp_path / "cache")), cwd=str(tmp_path)) + assert cured.returncode == 0 and word in cured.stdout, cured.stderr + assert list((tmp_path / "cache").rglob("*.py")), "no anchor under NUMBA_CACHE_DIR" + finally: + home.chmod(0o755) + + +@needs_a_directory_it_cannot_write +@pytest.mark.parametrize("child, word, warns", [ + (MAKE_A_STRUCTREF, "made", True), (REGISTER_AN_AGGREGATE, "summed", True), (REGISTER_A_TVF, "selected", True), + (COMPILE_A_KERNEL, "executed", True), (BUILD_A_DERIVE, "derived", False), +], ids=["make_structref", "register_aggregate", "register_tvf", "compile_kernel", "derive"]) +def test_a_warm_anchor_in_a_directory_that_stopped_being_writable(tmp_path, child, word, warns): + # The anchor is on disk from an earlier run, in a NUMBA_CACHE_DIR that can + # no longer be written. numba then caches the code in the user's cache + # directory, and a check on the anchor's own directory would have turned + # caching off where numba had a location. With the user's cache directory + # unwritable too there is none, and a check that only wrote the anchor + # would have let the first decorated function die at numba's set-up. The + # builder's derive falls back without a word, as it did for an anchor it + # could not write. + home = tmp_path / "home" home.mkdir() cache = tmp_path / "cache" env = dict(os.environ, PYTHONPATH=str(REPO), HOME=str(home), XDG_CACHE_HOME=str(home / "cache"), @@ -1125,7 +5803,7 @@ def test_a_struct_name_of_any_length_caches(tmp_path, name): @pytest.mark.skipif(sys.platform != "linux", reason="a path of 4096 bytes and a name of 255 are Linux's limits") @pytest.mark.parametrize("directory, remedy", [ ("HOME", "too long for the file system: NUMBA_CACHE_DIR at a shorter path"), - ("NUMBA_CACHE_DIR", "the path is too long for the file system: a shorter NUMBA_CACHE_DIR, or none"), + ("NUMBA_CACHE_DIR", "the path is too long for the file system: a shorter NUMBA_CACHE_DIR; or NUMBOX_JIT_OPTIONS"), ]) def test_an_anchor_path_too_long_for_the_file_system_compiles_uncached_and_the_warning_says_so( tmp_path, directory, remedy): @@ -1135,10 +5813,20 @@ def test_an_anchor_path_too_long_for_the_file_system_compiles_uncached_and_the_w # NUMBA_CACHE_DIR" where none was set. There is no cache here, and the # warning names the length rather than offering a writable directory, which # this one is; NUMBA_CACHE_DIR at a short path cures it. A NUMBA_CACHE_DIR - # as deep is too deep for the package's own files, whose location under it - # appends their directory's path, so the package answers first, with its - # remedy, and the struct compiles under its answer without a word of its own. - deep = _directory_of_length(tmp_path, 4096 - 50) + # is made deep enough for the package's own files instead, whose location + # under it, a directory named for theirs by its name and a hash of its + # path, ends 20 bytes short of the limit: numba's own check passes there, + # with a named temporary file too where the file system makes no unnamed + # one, and the package's does not, so the package answers first, with its + # remedy, and the struct compiles under its answer without a word of its + # own. At the home's depth numba's check failed the location on such a file + # system and moved on to the tree, and the anchor answered instead. + if directory == "NUMBA_CACHE_DIR": + from numba.core.caching import _CacheLocator + location = _CacheLocator.get_suitable_cache_subpath(str(REPO / "numbox" / "core" / "configurations.py")) + deep = _directory_of_length(tmp_path, 4096 - 20 - 1 - len(location)) + else: + deep = _directory_of_length(tmp_path, 4096 - 50) env = dict(os.environ, PYTHONPATH=str(REPO)) for name in ("NUMBOX_JIT_OPTIONS", "NUMBA_CACHE_DIR", "XDG_CACHE_HOME"): env.pop(name, None) @@ -1148,3 +5836,1016 @@ def test_an_anchor_path_too_long_for_the_file_system_compiles_uncached_and_the_w assert run.returncode == 0 and "made" in run.stdout, run.stderr assert run.stderr.count("compiles without a cache") == 1, run.stderr assert remedy in run.stderr, run.stderr + + +A_CACHE_FILE_NAME_TOO_LONG = { + "on disk, numba's order": [], + "on disk, numba's order, an invalid name": [], + "on disk, NUMBA_CACHE_DIR set": [], + "on disk, the in-tree locator alone": from_062, + "a .zip member in an ipykernel directory": [], + "a .zip member, numba's order": from_061, + "on disk, numba's order, a name a file system in use takes": [], + "on disk, NUMBA_CACHE_DIR set, a name a file system in use takes": [], + "on disk, the in-tree locator alone, a name a file system in use takes": from_062, + "a .zip member in an ipykernel directory, a name a file system in use takes": [], + "a .zip member, numba's order, a name a file system in use takes": from_061, +} + + +@pytest.mark.parametrize( + "case", [pytest.param(case, marks=marks, id=case) for case, marks in A_CACHE_FILE_NAME_TOO_LONG.items()]) +def test_a_cache_file_whose_name_is_too_long_for_the_file_system_is_told_the_name_which_no_shorter_path_cures( + tmp_path, monkeypatch, case): + # numba names the files it caches a function in after the source's name + # and the function's qualified name, and a name longer than the file + # system takes can be made at no path: numba's first read of the index + # dies on it, and the remedy told the path as too long, asking for the + # package, the user's cache directory, IPython's or NUMBA_CACHE_DIR at a + # shorter path, which move nothing of the name. It tells the name and its + # length, counted as the file system counts it, that no shorter path + # cures it, and the cure, a shorter name for the function or its file, + # the package's to give, with the variable, where numba tries it before + # the locator that took the file and a file system in use takes the name, + # 255 being their limit, pointed at a directory on one, and the + # variable's own location named where it took the file; the remedy + # offered such a file system for a name none takes, and never the + # shorter name. Windows refuses a name too long as an invalid one, errno + # EINVAL, and the check runs whatever the errno. + import types + import numba + from numba.core import caching + import numbox.core.configurations as configurations + from numba.core.caching import _CacheLocator + + class UserCacheUnderTmp: + def __init__(self, appname, appauthor): + self.user_cache_dir = str(tmp_path / "user-cache" / "numba") + + monkeypatch.setattr(caching, "AppDirs", UserCacheUnderTmp) + monkeypatch.setattr(configurations, "AppDirs", UserCacheUnderTmp) + cache_dir = str(tmp_path / "cache") if "NUMBA_CACHE_DIR set" in case else "" + if cache_dir: + os.mkdir(cache_dir) + monkeypatch.setattr(numba.config, "CACHE_DIR", cache_dir) + listed = "InTreeCacheLocator" if "the in-tree locator alone" in case else "" + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", listed, raising=False) + ipython_dir = tmp_path / "ipython" + paths = types.ModuleType("IPython.paths") + paths.get_ipython_cache_dir = lambda: str(ipython_dir) + monkeypatch.setitem(sys.modules, "IPython", types.ModuleType("IPython")) + monkeypatch.setitem(sys.modules, "IPython.paths", paths) + if case.startswith("on disk"): + py_file = str(tmp_path / "site" / "module.py") + os.makedirs(os.path.dirname(py_file)) + with open(py_file, "w", encoding="utf-8") as handle: + handle.write("x = 1\n") + elif "ipykernel" in case: + py_file = str(tmp_path / "bundle.zip" / "ipykernel_123" / "cell.py") + else: + py_file = str(tmp_path / "bundle.zip" / "package" / "module.py") + subpath = _CacheLocator.get_suitable_cache_subpath(py_file) + if "ipykernel" in case: + location = ipython_dir / "numba_cache" + elif case.startswith("on disk") and not cache_dir: + location = Path(py_file).parent / "__pycache__" + else: + location = Path(cache_dir or str(tmp_path / "user-cache" / "numba")) / subpath + if "a file system in use takes" in case: + # A limit the directory numba names after the source's, its name and + # forty-one bytes, fits, and the file's name does not. + monkeypatch.setattr(configurations, "_name_limit", lambda location: 60) + limit = configurations._name_limit(str(location)) + name = f"{Path(py_file).stem}.{'f' * (limit + 5)}-1.py{sys.version_info[0]}{sys.version_info[1]}.nbi" + counted_in = "characters" if os.name == "nt" or sys.platform == "darwin" else "bytes" + length = len(name.encode("utf-16-le")) // 2 if counted_in == "characters" else len(os.fsencode(name)) + assert length > limit + assert (length <= 255) == ("a file system in use takes" in case) + refused = ((errno.EINVAL, "Invalid argument") if "invalid name" in case + else (errno.ENAMETOOLONG, "File name too long")) + failure = OSError(*refused, str(location / name)) + assert configurations.is_a_cache_error(failure) + remedy = configurations.cache_remedy(py_file, failure, "silence") + clause = ( + f"numba names the cache files for a function after the source's name and the function's qualified name, " + f"{name}, {length} {counted_in}, which is longer than a name can be on that file system ({limit})") + fits = "directory on a file system that takes a name that long" + cure = f"{clause}, and no shorter path cures it: a shorter name for the function or its file, which is numbox's to give" + if length > 255: + offer = "" + elif case.startswith("on disk, numba's order"): + offer = f", or NUMBA_CACHE_DIR set to a {fits}" + elif cache_dir: + offer = f", or NUMBA_CACHE_DIR, which is set to {cache_dir}, pointed at a {fits}" + else: + offer = "" + if case.startswith("on disk") or "ipykernel" in case: + assert remedy == f"{cure}{offer}; or silence", remedy + else: + assert remedy == ( + "numba caches a .zip, or a frozen application, in the user's cache directory, and NUMBA_CACHE_DIR has no " + f"effect here, because the source is not a file on disk: {cure}, or silence"), remedy + + +def test_a_function_whose_cache_file_name_is_too_long_for_the_file_system_is_told_the_name_by_numbas_own_error( + tmp_path, monkeypatch): + # numba's own error for a cache file whose name is too long, which its + # first read of the index raises at the first call, names the file, and + # the remedy told the path as too long and asked for the package installed + # at a shorter path, or NUMBA_CACHE_DIR set to a short one, which cure + # nothing of the name. Windows refuses the name as an invalid one, errno + # EINVAL, and the remedy reads the name whatever the errno. + import importlib + import numba + import numbox.core.configurations as configurations + + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + limit = configurations._name_limit(str(tmp_path)) + function = "f" * (limit + 5) + source = tmp_path / "module_named_long.py" + source.write_text( + f"from numba import njit\n\n\n@njit(cache=True)\ndef {function}(x):\n return x + 1\n", encoding="utf-8") + monkeypatch.syspath_prepend(str(tmp_path)) + monkeypatch.delitem(sys.modules, "module_named_long", raising=False) + module = importlib.import_module("module_named_long") + with pytest.raises(OSError) as raised: + getattr(module, function)(1) + failure = raised.value + assert configurations.is_a_cache_error(failure) + assert Path(failure.filename).parent == tmp_path / "__pycache__" + name = Path(failure.filename).name + assert name.startswith(f"module_named_long.{function}-") + counted_in = "characters" if os.name == "nt" or sys.platform == "darwin" else "bytes" + length = len(name.encode("utf-16-le")) // 2 if counted_in == "characters" else len(os.fsencode(name)) + assert length > limit + offer = ", or NUMBA_CACHE_DIR set to a directory on a file system that takes a name that long" if length <= 255 else "" + assert configurations.cache_remedy(str(source), failure, "silence") == ( + f"numba names the cache files for a function after the source's name and the function's qualified name, " + f"{name}, {length} {counted_in}, which is longer than a name can be on that file system ({limit}), and no " + f"shorter path cures it: a shorter name for the function or its file, which is numbox's to give{offer}; or " + "silence") + + +def test_a_cell_file_on_disk_the_in_tree_locator_takes_is_not_told_ipythondir_for_an_error_naming_another_numba_cache( + tmp_path, monkeypatch): + # numba's in-tree locator takes a cell file on disk in an ipykernel + # directory whose __pycache__ it can write, and IPython's locator, listed + # after it, is never reached; the remedy read IPython's as reached for any + # file its locator takes by name, and told a caller's error naming another + # numba_cache, a location that is none of numba's, as IPython giving a + # temporary directory per call, with IPYTHONDIR for it. + import tempfile + import types + import numba + import numbox.core.configurations as configurations + + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + paths = types.ModuleType("IPython.paths") + paths.get_ipython_cache_dir = lambda: tempfile.mkdtemp(dir=tmp_path) + monkeypatch.setitem(sys.modules, "IPython", types.ModuleType("IPython")) + monkeypatch.setitem(sys.modules, "IPython.paths", paths) + kernel = tmp_path / "ipykernel_123" + kernel.mkdir() + py_file = str(kernel / "cell.py") + with open(py_file, "w", encoding="utf-8") as handle: + handle.write("def cell():\n return 1\n") + configurations.check_cache_location(py_file, 0) + assert (kernel / "__pycache__").is_dir() + unrelated = str(tmp_path / "elsewhere" / "numba_cache") + remedy = configurations.cache_remedy(py_file, OSError(errno.EACCES, "Permission denied", unrelated), "silence") + assert remedy == ( + f"that location, {unrelated}, where no file can be written (Permission denied): make room there, or make it " + "writable, or NUMBA_CACHE_DIR set to a writable directory; or silence"), remedy + + +@pytest.mark.skipif(os.name == "nt", reason="Windows decodes any byte of a path") +@pytest.mark.skipif(sys.platform == "darwin", reason="APFS takes no name that is not UTF-8") +@needs_a_directory_it_cannot_write +def test_a_source_under_a_directory_that_is_not_utf8_whose_pycache_is_unwritable_is_a_cache_error_told_a_utf8_path( + tmp_path, monkeypatch): + # numba hashes the source's directory path as UTF-8 for the name of its + # directory under NUMBA_CACHE_DIR or the user's cache directory, and + # raises UnicodeEncodeError for bytes that are not, decoded with + # surrogates, at decoration, where a locator that names its directory so + # reaches the file: the user-wide one, once the in-tree locator passed an + # unwritable __pycache__ over. The check raises it as numba does, and + # is_a_cache_error did not admit it, a ValueError that is not the .zip + # locator's, so a package installed under such a directory died at import + # on it, where with caching off it imports, and the remedy for it stood + # unreached. A UnicodeEncodeError of another reason is no cache error. + import numba + from numba.core import caching + import numbox.core.configurations as configurations + + class UserCacheUnderTmp: + def __init__(self, appname, appauthor): + self.user_cache_dir = str(tmp_path / "user-cache" / "numba") + + monkeypatch.setattr(caching, "AppDirs", UserCacheUnderTmp) + monkeypatch.setattr(configurations, "AppDirs", UserCacheUnderTmp) + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + directory = tmp_path / os.fsdecode(b"site\xff") + directory.mkdir() + py_file = str(directory / "module.py") + with open(py_file, "w", encoding="utf-8") as handle: + handle.write("x = 1\n") + pycache = directory / "__pycache__" + pycache.mkdir() + pycache.chmod(0o555) + try: + with pytest.raises(UnicodeEncodeError) as raised: + configurations.check_cache_location(py_file, configurations.LONGEST_CACHE_FILE_NAME) + failure = raised.value + assert failure.reason == "surrogates not allowed" + assert configurations.is_a_cache_error(failure) + remedy = configurations.cache_remedy(py_file, failure, "silence") + finally: + pycache.chmod(0o755) + assert remedy == ( + "numba hashes the source's directory as UTF-8 for the directory it caches this file in under NUMBA_CACHE_DIR " + f"or the user's cache directory, and {directory} is not: put the file under a UTF-8 path, or make the " + f"__pycache__ beside the source, {pycache}, writable, where InTreeCacheLocator caches it hashing no path; or " + "silence"), remedy + assert not configurations.is_a_cache_error(UnicodeEncodeError("ascii", "\u00e9", 0, 1, "ordinal not in range(128)")) + + +@pytest.mark.skipif(os.name == "nt", reason="Windows decodes any byte of a path") +def test_an_error_naming_a_file_with_a_byte_that_is_not_utf8_is_answered_where_names_are_counted_in_utf16( + tmp_path, monkeypatch): + # A name an error carries decoded with surrogateescape holds a surrogate + # for a byte that is not UTF-8, and Windows takes a lone surrogate in a + # name of its own; counting the name in UTF-16 units, as Windows and + # macOS count one, raised UnicodeEncodeError on it, and the remedy died on + # an error it admits instead of answering it. + import numba + import numbox.core.configurations as configurations + + monkeypatch.setattr(sys, "platform", "darwin") + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + site = tmp_path / "site" + site.mkdir() + py_file = str(site / "module.py") + with open(py_file, "w", encoding="utf-8") as handle: + handle.write("x = 1\n") + named = str(site / "__pycache__" / os.fsdecode(b"module.f-1.py312.nbi\xff")) + with pytest.raises(UnicodeEncodeError): + os.path.basename(named).encode("utf-16-le") + assert configurations._measured(os.path.basename(named)) == (21, "characters") + remedy = configurations.cache_remedy(py_file, OSError(errno.EACCES, "Permission denied", named), "silence") + assert remedy == ( + f"numba caches this file in {site / '__pycache__'}, where no file can be written (Permission denied): make " + "room there, or make it writable, or NUMBA_CACHE_DIR set to a writable directory; or silence"), remedy + + +@from_061 +@pytest.mark.skipif(os.name == "nt", reason="Windows decodes any byte of a path") +@pytest.mark.skipif(sys.platform == "darwin", reason="APFS takes no name that is not UTF-8") +def test_a_zip_under_a_directory_that_is_not_utf8_is_a_cache_error_told_the_archive_under_a_utf8_path( + tmp_path, monkeypatch): + # numba's .zip locator names its directory under the user's cache + # directory after the source's directory, the archive's path in it, hashed + # as UTF-8, and raises UnicodeEncodeError for bytes that are not, at + # decoration and in the check alike; the remedy took the error for a + # caller's with no place for the file, and offered the .zip the reader was + # importing from, with nothing of the path. + import numba + from numba.core import caching + import numbox.core.configurations as configurations + + class UserCacheUnderTmp: + def __init__(self, appname, appauthor): + self.user_cache_dir = str(tmp_path / "user-cache" / "numba") + + monkeypatch.setattr(caching, "AppDirs", UserCacheUnderTmp) + monkeypatch.setattr(configurations, "AppDirs", UserCacheUnderTmp) + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + directory = tmp_path / os.fsdecode(b"caf\xe9") + directory.mkdir() + archive = directory / "app.zip" + with zipfile.ZipFile(archive, "w") as zipped: + zipped.writestr("package/module.py", "x = 1\n") + py_file = str(archive / "package" / "module.py") + with pytest.raises(UnicodeEncodeError) as raised: + configurations.check_cache_location(py_file, configurations.LONGEST_CACHE_FILE_NAME) + failure = raised.value + assert configurations.is_a_cache_error(failure) + assert configurations.cache_remedy(py_file, failure, "silence") == ( + "numba caches a .zip in the user's cache directory, and NUMBA_CACHE_DIR has no effect here, because the " + "source is not a file on disk: numba hashes the source's directory as UTF-8 for the directory it caches this " + f"file in there, and {archive / 'package'} is not: put the archive under a UTF-8 path, or silence") + + +@pytest.mark.skipif(os.name == "nt", reason="Windows decodes any byte of a path") +@pytest.mark.skipif(sys.platform == "darwin", reason="APFS takes no name that is not UTF-8") +def test_an_anchor_under_a_cache_directory_that_is_not_utf8_compiles_uncached_and_the_warning_asks_for_a_utf8_path( + tmp_path, monkeypatch): + # numba hashes the anchor's directory path as UTF-8 to name its cache + # directory under NUMBA_CACHE_DIR or the user's cache directory, and + # raises UnicodeEncodeError for bytes that are not, which is a cache + # error; the anchor's warning offered a writable directory, which the + # anchor's is, and said nothing of the path. + import warnings + import numba + from numbox.utils import preprocessing + + cache_dir = tmp_path / os.fsdecode(b"caf\xe9") + cache_dir.mkdir() + monkeypatch.setattr(numba.config, "CACHE_DIR", str(cache_dir)) + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + code_txt = "def f():\n return 1\n" + path = preprocessing._structref_anchor_path("S", code_txt) + assert path.parent == cache_dir / "numbox-structref" + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + options = preprocessing._anchored_or_uncached(path, code_txt, {"cache": True}) + assert options == {"cache": False} + assert path.is_file() + assert len(caught) == 1, [str(w.message) for w in caught] + message = str(caught[0].message) + assert "surrogates not allowed" in message + assert message.endswith( + f"numba hashes the anchor's directory path as UTF-8 to name its cache directory, and {path.parent} is not: " + f"NUMBA_CACHE_DIR at a UTF-8 path, or {preprocessing._SILENCE}"), message + + +@pytest.mark.skipif(os.name == "nt", reason="Windows decodes any byte of a path") +@pytest.mark.skipif(sys.platform == "darwin", reason="APFS takes no name that is not UTF-8") +def test_a_source_under_a_directory_that_is_not_utf8_with_numba_cache_dir_set_is_told_to_unset_it( + tmp_path, monkeypatch): + # With NUMBA_CACHE_DIR set, numba's user-provided locator hashes the + # source's directory path as UTF-8 for its directory under the variable + # and raises for bytes that are not, before the in-tree locator is + # reached, which caches beside the source and hashes no path: the same + # install imports cached with the variable unset. The remedy said the + # variable cures nothing and asked for a UTF-8 path alone. + import numba + import numbox.core.configurations as configurations + + cache_dir = tmp_path / "cache" + cache_dir.mkdir() + monkeypatch.setattr(numba.config, "CACHE_DIR", str(cache_dir)) + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + directory = tmp_path / os.fsdecode(b"site\xff") + directory.mkdir() + py_file = str(directory / "module.py") + with open(py_file, "w", encoding="utf-8") as handle: + handle.write("x = 1\n") + with pytest.raises(UnicodeEncodeError) as raised: + configurations.check_cache_location(py_file, configurations.LONGEST_CACHE_FILE_NAME) + failure = raised.value + assert configurations.is_a_cache_error(failure) + assert configurations.cache_remedy(py_file, failure, "silence") == ( + "numba hashes the source's directory as UTF-8 for the directory it caches this file in under NUMBA_CACHE_DIR " + f"or the user's cache directory, and {directory} is not, and NUMBA_CACHE_DIR, which is set to {cache_dir}, is " + f"hashed for first: unset it, and numba caches beside the source, in its __pycache__, {directory / '__pycache__'}, " + "hashing no path; or put the file under a UTF-8 path; or silence") + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + configurations.check_cache_location(py_file, configurations.LONGEST_CACHE_FILE_NAME) + assert (directory / "__pycache__").is_dir() + + +@from_062 +@pytest.mark.skipif(os.name == "nt", reason="Windows decodes any byte of a path") +@pytest.mark.skipif(sys.platform == "darwin", reason="APFS takes no name that is not UTF-8") +def test_a_source_under_a_directory_that_is_not_utf8_under_the_user_wide_locator_alone_is_told_a_utf8_path( + tmp_path, monkeypatch): + # Under a list of the user-wide locator alone, numba hashes the source's + # directory path as UTF-8 for its directory under the user's cache + # directory and raises for bytes that are not; the remedy told the list + # and its locations to make writable, which hashing refuses whatever + # they are, with nothing of the path or of the in-tree locator, which + # caches beside the source hashing no path. + import numba + from numba.core import caching + import numbox.core.configurations as configurations + + class UserCacheUnderTmp: + def __init__(self, appname, appauthor): + self.user_cache_dir = str(tmp_path / "user-cache" / "numba") + + monkeypatch.setattr(caching, "AppDirs", UserCacheUnderTmp) + monkeypatch.setattr(configurations, "AppDirs", UserCacheUnderTmp) + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "UserWideCacheLocator", raising=False) + directory = tmp_path / os.fsdecode(b"site\xff") + directory.mkdir() + py_file = str(directory / "module.py") + with open(py_file, "w", encoding="utf-8") as handle: + handle.write("x = 1\n") + with pytest.raises(UnicodeEncodeError) as raised: + configurations.check_cache_location(py_file, configurations.LONGEST_CACHE_FILE_NAME) + failure = raised.value + assert configurations.is_a_cache_error(failure) + assert configurations.cache_remedy(py_file, failure, "silence") == ( + "numba hashes the source's directory as UTF-8 for the directory it caches this file in under NUMBA_CACHE_DIR " + f"or the user's cache directory, and {directory} is not: put the file under a UTF-8 path, or list " + "InTreeCacheLocator before UserWideCacheLocator, and numba caches beside the source, in its __pycache__, " + f"{directory / '__pycache__'}, hashing no path; or silence") + + +@from_062 +@needs_a_directory_it_cannot_write +@pytest.mark.skipif(os.name == "nt", reason="Windows decodes any byte of a path") +@pytest.mark.skipif(sys.platform == "darwin", reason="APFS takes no name that is not UTF-8") +def test_a_source_under_a_zip_named_directory_on_a_path_that_is_not_utf8_is_told_the_path_not_a_missing_archive( + tmp_path, monkeypatch): + # numba's .zip locator, listed after the in-tree one, takes a source on + # disk under a directory named .zip once an unwritable __pycache__ left + # it, and hashes the source's directory path as UTF-8 for its directory, + # raising UnicodeEncodeError for bytes that are not; the remedy read + # every ValueError the .zip locator reaches as its own for a path with + # no archive, and told a missing archive and locations to make writable. + import numba + import numbox.core.configurations as configurations + + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "InTreeCacheLocator,ZipCacheLocator", raising=False) + directory = tmp_path / os.fsdecode(b"site\xff") / "proj.zip" + directory.mkdir(parents=True) + py_file = str(directory / "module.py") + with open(py_file, "w", encoding="utf-8") as handle: + handle.write("x = 1\n") + pycache = directory / "__pycache__" + pycache.mkdir() + pycache.chmod(0o555) + try: + with pytest.raises(UnicodeEncodeError) as raised: + configurations.check_cache_location(py_file, configurations.LONGEST_CACHE_FILE_NAME) + failure = raised.value + assert configurations.is_a_cache_error(failure) + remedy = configurations.cache_remedy(py_file, failure, "silence") + finally: + pycache.chmod(0o755) + assert remedy == ( + "numba hashes the source's directory as UTF-8 for the directory it caches this file in under NUMBA_CACHE_DIR " + f"or the user's cache directory, and {directory} is not: put the file under a UTF-8 path, or make the " + f"__pycache__ beside the source, {pycache}, writable, where InTreeCacheLocator caches it hashing no path; or " + "silence"), remedy + + +@pytest.mark.skipif(os.name == "nt", reason="Windows decodes any byte of a path") +@pytest.mark.skipif(sys.platform == "darwin", reason="APFS takes no name that is not UTF-8") +def test_a_frozen_application_under_a_path_that_is_not_utf8_is_told_the_application_under_a_utf8_path( + tmp_path, monkeypatch): + # numba's user-wide locator, the one of numba's that takes a frozen + # application's file, hashes the file's directory path as UTF-8 for its + # directory under the user's cache directory and raises for bytes that + # are not; the remedy took the error for a caller's with no place for + # the file and offered the source files on disk, or a .zip holding them, + # with nothing of the path. + import numba + from numba.core import caching + import numbox.core.configurations as configurations + + class UserCacheUnderTmp: + def __init__(self, appname, appauthor): + self.user_cache_dir = str(tmp_path / "user-cache" / "numba") + + monkeypatch.setattr(caching, "AppDirs", UserCacheUnderTmp) + monkeypatch.setattr(configurations, "AppDirs", UserCacheUnderTmp) + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + monkeypatch.setattr(sys, "frozen", True, raising=False) + directory = tmp_path / os.fsdecode(b"app\xff") / "numbox" / "core" + py_file = str(directory / "configurations.py") + with pytest.raises(UnicodeEncodeError) as raised: + configurations.check_cache_location(py_file, configurations.LONGEST_CACHE_FILE_NAME) + failure = raised.value + assert configurations.is_a_cache_error(failure) + assert configurations.cache_remedy(py_file, failure, "silence") == ( + "numba caches a frozen application in the user's cache directory, and NUMBA_CACHE_DIR has no effect here, " + "because the source is not a file on disk: numba hashes the source's directory as UTF-8 for the directory it " + f"caches this file in there, and {directory} is not: put the application under a UTF-8 path, or silence") + + +@from_062 +@pytest.mark.skipif(os.name == "nt", reason="Windows decodes any byte of a path") +@pytest.mark.skipif(sys.platform == "darwin", reason="APFS takes no name that is not UTF-8") +@pytest.mark.parametrize("listed", [ + pytest.param("UserProvidedCacheLocator,UserWideCacheLocator", id="the in-tree locator left out, the variable set"), + pytest.param("UserWideCacheLocator,InTreeCacheLocator", id="the in-tree locator after the user-wide one"), +]) +def test_a_source_under_a_directory_that_is_not_utf8_is_told_the_in_tree_locator_before_the_first_that_hashes( + tmp_path, monkeypatch, listed): + # The in-tree locator caches beside the source hashing no path, and + # listed after one that hashes, the variable's set or the user-wide one, + # it is never reached: the remedy asked for it to be listed, with no + # place, where appended it leaves numba raising the same error, and, for + # one listed after the user-wide locator, for its __pycache__ to be made + # writable, which it was. + import numba + from numba.core import caching + import numbox.core.configurations as configurations + + class UserCacheUnderTmp: + def __init__(self, appname, appauthor): + self.user_cache_dir = str(tmp_path / "user-cache" / "numba") + + monkeypatch.setattr(caching, "AppDirs", UserCacheUnderTmp) + monkeypatch.setattr(configurations, "AppDirs", UserCacheUnderTmp) + cache_dir = tmp_path / "cache" + cache_dir.mkdir() + monkeypatch.setattr(numba.config, "CACHE_DIR", str(cache_dir) if "UserProvided" in listed else "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", listed, raising=False) + directory = tmp_path / os.fsdecode(b"site\xff") + directory.mkdir() + (directory / "__pycache__").mkdir() + py_file = str(directory / "module.py") + with open(py_file, "w", encoding="utf-8") as handle: + handle.write("x = 1\n") + with pytest.raises(UnicodeEncodeError) as raised: + configurations.check_cache_location(py_file, configurations.LONGEST_CACHE_FILE_NAME) + failure = raised.value + assert configurations.is_a_cache_error(failure) + remedy = configurations.cache_remedy(py_file, failure, "silence") + hashed = ( + "numba hashes the source's directory as UTF-8 for the directory it caches this file in under NUMBA_CACHE_DIR " + f"or the user's cache directory, and {directory} is not") + beside = f"numba caches beside the source, in its __pycache__, {directory / '__pycache__'}, hashing no path" + if "UserProvided" in listed: + assert remedy == ( + f"{hashed}: put the file under a UTF-8 path, or list InTreeCacheLocator before UserProvidedCacheLocator, " + f"and {beside}; or silence"), remedy + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "InTreeCacheLocator," + listed, raising=False) + else: + assert remedy == ( + f"{hashed}, and NUMBA_CACHE_LOCATOR_CLASSES puts UserWideCacheLocator, which hashes it, before " + f"InTreeCacheLocator: list InTreeCacheLocator before it, and {beside}; or put the file under a UTF-8 path; " + "or silence"), remedy + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "InTreeCacheLocator,UserWideCacheLocator", + raising=False) + # Listed as the remedy says, numba caches the file. + configurations.check_cache_location(py_file, configurations.LONGEST_CACHE_FILE_NAME) + + +@pytest.mark.skipif(os.name == "nt", reason="Windows decodes any byte of a path") +@pytest.mark.skipif(sys.platform == "darwin", reason="APFS takes no name that is not UTF-8") +def test_a_frozen_application_under_a_zip_named_directory_on_a_path_that_is_not_utf8_is_told_the_application( + tmp_path, monkeypatch): + # numba's user-wide locator, before the .zip one in numba's order, takes + # a frozen application's file and hashes its directory path as UTF-8, + # raising for bytes that are not; the remedy read the .zip in the path + # and told an archive to put under a UTF-8 path, where there is none. + import numba + from numba.core import caching + import numbox.core.configurations as configurations + + class UserCacheUnderTmp: + def __init__(self, appname, appauthor): + self.user_cache_dir = str(tmp_path / "user-cache" / "numba") + + monkeypatch.setattr(caching, "AppDirs", UserCacheUnderTmp) + monkeypatch.setattr(configurations, "AppDirs", UserCacheUnderTmp) + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + monkeypatch.setattr(sys, "frozen", True, raising=False) + directory = tmp_path / os.fsdecode(b"app\xff.zip") / "frozen" / "numbox" / "core" + directory.mkdir(parents=True) + py_file = str(directory / "module.py") + with pytest.raises(UnicodeEncodeError) as raised: + configurations.check_cache_location(py_file, configurations.LONGEST_CACHE_FILE_NAME) + failure = raised.value + assert configurations.is_a_cache_error(failure) + assert configurations.cache_remedy(py_file, failure, "silence") == ( + "numba caches a frozen application in the user's cache directory, and NUMBA_CACHE_DIR has no effect here, " + "because the source is not a file on disk: numba hashes the source's directory as UTF-8 for the directory it " + f"caches this file in there, and {directory} is not: put the application under a UTF-8 path, or silence") + + +@from_062 +def test_a_source_that_is_nowhere_under_a_directory_named_zip_is_not_told_to_list_the_zip_locator(tmp_path, monkeypatch): + # A list without numba's .zip locator was told to list it for any path + # with .zip in it, where a directory so named above a source that is + # nowhere is no archive: listed, that locator takes the directory for one + # and caches nothing, numba's check raising for it. Such a source is told + # the source files, on disk or in a .zip. + import numba + import numbox.core.configurations as configurations + + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "InTreeCacheLocator", raising=False) + directory = tmp_path / "proj.zip" / "numbox" / "core" + directory.mkdir(parents=True) + py_file = str(directory / "module.py") + with pytest.raises(RuntimeError, match="no locator available") as raised: + configurations.check_cache_location(py_file, configurations.LONGEST_CACHE_FILE_NAME) + failure = raised.value + assert configurations.is_a_cache_error(failure) + remedy = configurations.cache_remedy(py_file, failure, "silence") + assert "list it" not in remedy and "ZipCacheLocator alone" not in remedy, remedy + assert remedy.startswith("NUMBA_CACHE_DIR has no effect here, because the source is not a file on disk: to cache, ") + assert "install numbox with its source files on disk" in remedy and remedy.endswith("; or silence"), remedy + + +def test_a_cell_under_a_directory_named_zip_whose_source_linecache_does_not_hold_is_told_the_cell_remedy( + tmp_path, monkeypatch): + # inspect raises for a cell whose source linecache does not hold, before + # numba's IPython locator tries its location, and the remedy tells it as + # such, but for a member of an archive, told the source files to ship: a + # directory named .zip above the cell was read as an archive, and the + # cell was told to ship source files. + import types + import numba + import numbox.core.configurations as configurations + + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + paths = types.ModuleType("IPython.paths") + paths.get_ipython_cache_dir = lambda: str(tmp_path / "ipython") + monkeypatch.setitem(sys.modules, "IPython", types.ModuleType("IPython")) + monkeypatch.setitem(sys.modules, "IPython.paths", paths) + directory = tmp_path / "proj.zip" / "ipykernel_123" + directory.mkdir(parents=True) + py_file = str(directory / "cell.py") + with pytest.raises(OSError, match="could not get source code") as raised: + configurations.check_cache_location(py_file, 0) + failure = raised.value + assert failure.errno is None and failure.filename is None + assert configurations.cache_remedy(py_file, failure, "silence") == ( + f"numba's IPython locator reads a cell's source from linecache, which holds none for {py_file} ({failure}), " + "and numba caches a cell only while IPython holds its source; or silence") + + +@from_062 +@on_windows_under_numba_0_68_0 +def test_an_egg_under_a_directory_named_zip_is_told_to_put_the_renamed_archive_out_from_under_it(tmp_path, monkeypatch): + # numba's .zip locator takes the first part of the path ending in .zip for + # the archive, a directory so named above an .egg included, and opens it: + # the remedy told the .egg renamed to end in .zip, under that directory + # still, where numba then raises IsADirectoryError for the directory. + import numba + import numbox.core.configurations as configurations + + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "InTreeCacheLocator", raising=False) + in_the_way = tmp_path / "dir.zip" + in_the_way.mkdir() + egg = in_the_way / "numbox.egg" + with zipfile.ZipFile(egg, "w") as zipped: + zipped.writestr("pkg/m.py", "x = 1\n") + py_file = str(egg / "pkg" / "m.py") + with pytest.raises(RuntimeError, match="no locator available") as raised: + configurations.check_cache_location(py_file, configurations.LONGEST_CACHE_FILE_NAME) + assert configurations.cache_remedy(py_file, raised.value, "silence") == ( + "NUMBA_CACHE_DIR has no effect here, because the source is not a file on disk: to cache, install numbox with " + f"its source files on disk, unpacked from any archive, or rename the archive, {egg}, to end in .zip, " + "lower-case, which numba's .zip locator looks for in the path, with its source files in it, and put it under a " + f"path with no part ending in .zip above it, {in_the_way} being one, which that locator takes for the archive " + "first, so that numba 0.61 and later cache it in the user's cache directory through ZipCacheLocator, once " + "NUMBA_CACHE_LOCATOR_CLASSES, InTreeCacheLocator, lists it; or silence") + # Done as told, numba caches the member. + moved = tmp_path / "numbox.zip" + egg.rename(moved) + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "InTreeCacheLocator,ZipCacheLocator", raising=False) + configurations.check_cache_location(str(moved / "pkg" / "m.py"), configurations.LONGEST_CACHE_FILE_NAME) + + +@pytest.mark.parametrize("units", [230, 231, 251, 252]) +def test_the_offer_of_a_file_system_that_takes_a_cache_file_name_holds_at_the_usual_limit(tmp_path, monkeypatch, units): + # NUMBA_CACHE_DIR on a file system that takes the name is offered where + # the longest name numba writes for the function fits 255, the file + # systems in use's limit: the index's base, ".999.nbc" for a data file + # with the three digits reserved for its number, and the 21 of the + # temporary name numba writes each file through, ".tmp." and sixteen hex + # digits; and not for one over. No test held the boundary, and a count of + # less than 255 passed them all; then the offer went to a name of 255, + # whose temporary name numba's save of it failed on; then to an index of + # 234, whose data file's did; then two digits left a hundredth + # signature's data file a byte over. + import numba + import numbox.core.configurations as configurations + + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + monkeypatch.setattr(configurations, "_name_limit", lambda location: 143) + site = tmp_path / "site" + site.mkdir() + py_file = str(site / "module.py") + with open(py_file, "w", encoding="utf-8") as handle: + handle.write("x = 1\n") + tail = f"-1.py{sys.version_info[0]}{sys.version_info[1]}.nbi" + if units > 240: + # numba's save names the temporary file, the name and 21 more. + tail += ".tmp.0123456789abcdef" + name = f"module.{'f' * (units - len('module.') - len(tail))}{tail}" + counted_in = "characters" if os.name == "nt" or sys.platform == "darwin" else "bytes" + assert configurations._measured(name) == (units, counted_in) + failure = OSError(errno.ENAMETOOLONG, "File name too long", str(site / "__pycache__" / name)) + remedy = configurations.cache_remedy(py_file, failure, "silence") + base = units - len(".nbi") - (len(".tmp.0123456789abcdef") if units > 240 else 0) + fits = base + len(".999.nbc") + 21 <= 255 + assert fits == (units in (230, 251)) + offer = ", or NUMBA_CACHE_DIR set to a directory on a file system that takes a name that long" if fits else "" + assert remedy == ( + "numba names the cache files for a function after the source's name and the function's qualified name, " + f"{name}, {units} {counted_in}, which is longer than a name can be on that file system (143), and no shorter " + f"path cures it: a shorter name for the function or its file, which is numbox's to give{offer}; or silence"), remedy + + +def test_a_callers_error_naming_another_numba_cache_for_a_member_is_told_that_location_under_a_temporary_ipython_too( + tmp_path, monkeypatch): + # The error numba's first save raises under an IPython giving a temporary + # directory per call is FileNotFoundError, naming the numba_cache it could + # not save in; the remedy for it took any OSError naming a numba_cache, + # a caller's refused directory of its own among them, for a member in an + # ipykernel directory, and told IPYTHONDIR where, with IPython giving its + # directory, the same error is told as the location it names. + import tempfile + import types + import numba + import numbox.core.configurations as configurations + + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + stable = tmp_path / "stable-cache" + stable.mkdir() + paths = types.ModuleType("IPython.paths") + paths.get_ipython_cache_dir = lambda: str(stable) + monkeypatch.setitem(sys.modules, "IPython", types.ModuleType("IPython")) + monkeypatch.setitem(sys.modules, "IPython.paths", paths) + archive = tmp_path / "bundle.zip" + with zipfile.ZipFile(archive, "w") as zipped: + zipped.writestr("ipykernel_123/cell.py", "def cell():\n return 1\n") + py_file = str(archive / "ipykernel_123" / "cell.py") + unrelated = str(tmp_path / "elsewhere" / "numba_cache") + refused = OSError(errno.EACCES, "Permission denied", unrelated) + told = ( + "NUMBA_CACHE_DIR has no effect here, because the source is not a file on disk: no file can be written at that " + f"location, {unrelated} (Permission denied), so make room there, or make it writable, or silence") + assert configurations.cache_remedy(py_file, refused, "silence") == told + paths.get_ipython_cache_dir = lambda: tempfile.mkdtemp(dir=tmp_path) + assert configurations.cache_remedy(py_file, refused, "silence") == told + # The shape numba's own first save has there is told IPYTHONDIR still. + missing = FileNotFoundError(errno.ENOENT, "No such file or directory", unrelated) + assert configurations.cache_remedy(py_file, missing, "silence") == ( + "numba caches this file in numba_cache under IPython's cache directory, and IPython, able to write neither " + "that directory nor its own, gives a new temporary directory on every call, so numba makes numba_cache in one " + "and saves in another: set IPYTHONDIR, or XDG_CACHE_HOME on a POSIX system, to a writable directory; or silence") + + +@needs_a_directory_it_cannot_write +@pytest.mark.skipif(os.name == "nt", reason="Windows decodes any byte of a path") +@pytest.mark.skipif(sys.platform == "darwin", reason="APFS takes no name that is not UTF-8") +def test_a_source_under_a_directory_that_is_not_utf8_with_the_variable_set_and_its_pycache_unwritable_is_told_both( + tmp_path, monkeypatch): + # With NUMBA_CACHE_DIR set, its locator hashes the source's directory path + # and raises before the in-tree locator is tried, so the remedy cannot + # read from numba's error whether the __pycache__ beside the source can + # be written; it asks numba, and tells the variable to unset and the + # __pycache__ to make writable where it cannot be: the remedy hedged + # "where that directory can be written" and never asked for it, and the + # variable unset as told left numba raising the same error. + import numba + import numbox.core.configurations as configurations + + cache_dir = tmp_path / "cache" + cache_dir.mkdir() + monkeypatch.setattr(numba.config, "CACHE_DIR", str(cache_dir)) + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + directory = tmp_path / os.fsdecode(b"site\xff") + directory.mkdir() + py_file = str(directory / "module.py") + with open(py_file, "w", encoding="utf-8") as handle: + handle.write("x = 1\n") + pycache = directory / "__pycache__" + pycache.mkdir() + pycache.chmod(0o555) + try: + with pytest.raises(UnicodeEncodeError) as raised: + configurations.check_cache_location(py_file, configurations.LONGEST_CACHE_FILE_NAME) + remedy = configurations.cache_remedy(py_file, raised.value, "silence") + assert remedy == ( + "numba hashes the source's directory as UTF-8 for the directory it caches this file in under " + f"NUMBA_CACHE_DIR or the user's cache directory, and {directory} is not, and NUMBA_CACHE_DIR, which is set " + f"to {cache_dir}, is hashed for first: unset it, and make the __pycache__ beside the source, {pycache}, " + "writable, and numba caches there, hashing no path; or put the file under a UTF-8 path; or silence"), remedy + # Unset alone, numba raises the same error; both as told, it caches. + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + with pytest.raises(UnicodeEncodeError): + configurations.check_cache_location(py_file, configurations.LONGEST_CACHE_FILE_NAME) + finally: + pycache.chmod(0o755) + configurations.check_cache_location(py_file, configurations.LONGEST_CACHE_FILE_NAME) + + +MAKE_A_STRUCTREF_OF_FORTY_CHARACTER_NAMES = MAKE_A_STRUCTREF.replace( + "make_structref('Struct', {'value': float32}, TypeClass)", + "make_structref('S' * 40, {'v' * 40: float32}, TypeClass)").replace( + "assert Struct(2.5).value == 2.5", "assert getattr(Struct(2.5), 'v' * 40) == 2.5") + + +@pytest.mark.skipif(sys.platform != "linux", reason="a path of 4096 bytes is Linux's limit") +@pytest.mark.parametrize("short_of_the_limit", [230, 190]) +def test_an_anchor_within_numbas_longest_name_of_the_path_limit_compiles_uncached_and_the_warning_says_so( + tmp_path, short_of_the_limit): + # The anchor's check asked numba's location for a name of eight, which + # numba's own check makes too, so a NUMBA_CACHE_DIR within the longest + # name numba writes for a struct of the path limit passed both, and the + # struct's making died at numba's first save with no warning; the check + # asks for a name as long as that, and the code compiles without a cache + # after the warning naming the remedy. + deep = _directory_of_length(tmp_path, 4096 - short_of_the_limit) + env = dict(os.environ, PYTHONPATH=str(REPO), NUMBA_CACHE_DIR=str(deep)) + for name in ("NUMBOX_JIT_OPTIONS", "XDG_CACHE_HOME"): + env.pop(name, None) + run = subprocess.run([sys.executable, "-W", "always", "-c", MAKE_A_STRUCTREF_OF_FORTY_CHARACTER_NAMES], + capture_output=True, text=True, env=env, cwd=str(tmp_path)) + assert run.returncode == 0 and "made" in run.stdout, run.stderr + assert run.stderr.count("compiles without a cache") == 1, run.stderr + assert "The path is too long for the file system: NUMBA_CACHE_DIR at a shorter path" in run.stderr, run.stderr + + +@from_061 +def test_an_error_naming_a_zips_location_by_a_relative_path_under_nothing_yet_names_the_directory_it_is_made_under( + tmp_path, monkeypatch): + # The first path at or above the location that is there is the directory + # it cannot be made under; walked up a relative path with no ancestor on + # the way, it came to nothing, and the remedy said a file with no name + # was in the way. + import numba + import numbox.core.configurations as configurations + from numba.core.caching import _CacheLocator + + class UserCacheUnderNothingYet: + def __init__(self, appname, appauthor): + self.user_cache_dir = str(tmp_path / "home" / ".cache" / "numba") + + monkeypatch.setattr(configurations, "AppDirs", UserCacheUnderNothingYet) + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + monkeypatch.chdir(tmp_path) + archive = tmp_path / "bundle.zip" + with zipfile.ZipFile(archive, "w") as zipped: + zipped.writestr("package/module.py", "def f():\n pass\n") + py_file = str(archive / "package" / "module.py") + location = os.path.join("home", ".cache", "numba", _CacheLocator.get_suitable_cache_subpath(py_file)) + remedy = configurations.cache_remedy(py_file, OSError(errno.EACCES, "Permission denied", location), "silence") + assert remedy == ( + "numba caches a .zip, or a frozen application, in the user's cache directory, and NUMBA_CACHE_DIR has no " + f"effect here, because the source is not a file on disk: no file can be written in that directory, {location}, " + f"which cannot be made under {tmp_path} (Permission denied), so make room there, or make it writable, or " + "silence"), remedy + + +def test_a_name_limit_below_the_longest_name_the_check_asks_for_is_told_the_length_not_a_shorter_path( + tmp_path, monkeypatch): + # The check makes a file of the longest name the package's functions get + # in the location, and where the file system takes no name as long, its + # error, re-raised as the location's, was read as a path too long and a + # shorter NUMBA_CACHE_DIR offered, which cures nothing of a name; the + # error carries the length now, and the remedy asks for a file system + # that takes it. The anchor's warning does the same for a struct's. + import tempfile + import warnings + import numba + import numbox.core.configurations as configurations + from numbox.utils import preprocessing + + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + monkeypatch.setattr(configurations, "_name_limit", lambda location: 100) + real = tempfile.NamedTemporaryFile + + def refusing_a_long_name(*args, **kwargs): + if len(kwargs.get("prefix", "")) + 8 > configurations._name_limit(kwargs["dir"]): + raise OSError(errno.ENAMETOOLONG, "File name too long", os.path.join(kwargs["dir"], kwargs["prefix"])) + return real(*args, **kwargs) + + monkeypatch.setattr(configurations.tempfile, "NamedTemporaryFile", refusing_a_long_name) + py_file = tmp_path / "site" / "module.py" + py_file.parent.mkdir() + py_file.write_text("x = 1\n") + with pytest.raises(OSError) as raised: + configurations.check_cache_location(str(py_file), configurations.LONGEST_CACHE_FILE_NAME) + failure = raised.value + assert failure.errno == errno.ENAMETOOLONG and failure.filename == str(py_file.parent / "__pycache__") + assert failure.longest_file_name == configurations.LONGEST_CACHE_FILE_NAME + assert configurations.is_a_cache_error(failure) + counted_in = "characters" if os.name == "nt" or sys.platform == "darwin" else "bytes" + assert configurations.cache_remedy(str(py_file), failure, "silence") == ( + f"numba's cache files for numbox's functions have names of up to {configurations.LONGEST_CACHE_FILE_NAME} " + f"{counted_in}, longer than a name can be on that file system (100), so no shorter path cures it, or " + "NUMBA_CACHE_DIR set to a directory on a file system that takes a name that long; or silence") + # The anchor, whose check asks for numba's longest name for the code's + # functions, read off the code. + monkeypatch.setattr(numba.config, "CACHE_DIR", str(tmp_path / "cache")) + code_txt = "@njit(**jit_options)\ndef f():\n return 1\n" + path = preprocessing._structref_anchor_path("S", code_txt) + longest = preprocessing._longest_cache_file_name(path.stem, code_txt) + assert 40 < longest < 100 + monkeypatch.setattr(configurations, "_name_limit", lambda location: 40) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + options = preprocessing._anchored_or_uncached(path, code_txt, {"cache": True}) + assert options == {"cache": False} + assert len(caught) == 1, [str(w.message) for w in caught] + assert str(caught[0].message).endswith( + f"numba's cache files for the functions anchored here have names of up to {longest} bytes, longer than a " + f"name can be on that file system (40): NUMBA_CACHE_DIR at a directory on a file system that takes a name " + f"that long, or {preprocessing._SILENCE}"), str(caught[0].message) + + +def test_the_anchors_check_asks_for_the_longest_name_the_code_gets_not_a_bound_for_any_struct(tmp_path, monkeypatch): + # The anchor's check asked numba's location for a name of 251, a bound + # for any struct, so a file system whose name limit is under it, eCryptfs's + # 143, turned caching off for every struct, one whose names were 58 + # included; the check asks for the longest name numba writes for the + # code's own functions, read off the code: the stem, the qualified name + # with the angle brackets numba drops, the line, Python's version, two + # digits for the overload and the temporary name, of the functions the + # code jits alone, a struct's method under its own name, which calls the + # jitted thunk, and an overload's getting no cache file. + import tempfile + import warnings + import numba + import numbox.core.configurations as configurations + from numbox.utils import preprocessing + + monkeypatch.setattr(numba.config, "CACHE_DIR", str(tmp_path / "cache")) + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + monkeypatch.setattr(configurations, "_name_limit", lambda location: 143) + real = tempfile.NamedTemporaryFile + + def refusing_a_long_name(*args, **kwargs): + if len(kwargs.get("prefix", "")) + 8 > 143: + raise OSError(errno.ENAMETOOLONG, "File name too long", os.path.join(kwargs["dir"], kwargs["prefix"])) + return real(*args, **kwargs) + + monkeypatch.setattr(configurations.tempfile, "NamedTemporaryFile", refusing_a_long_name) + short = "@njit(**jit_options)\ndef f():\n return 1\n" + path = preprocessing._structref_anchor_path("Point", short) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + assert preprocessing._anchored_or_uncached(path, short, {"cache": True}) == {"cache": True} + assert not caught, [str(w.message) for w in caught] + assert preprocessing._longest_cache_file_name(path.stem, short) < 143 + long = f"@njit(**jit_options)\ndef {'f' * 120}():\n return 1\n" + path = preprocessing._structref_anchor_path("Point", long) + longest = preprocessing._longest_cache_file_name(path.stem, long) + assert longest > 143 + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + assert preprocessing._anchored_or_uncached(path, long, {"cache": True}) == {"cache": False} + assert len(caught) == 1 and f"names of up to {longest} bytes, longer than a name can be on that file system (143)" in str( + caught[0].message), [str(w.message) for w in caught] + # The names counted: a function decorated itself, with njit or a decorator + # of the package's own that jits, or defined within a decorated one, an + # overload's constructor among them, with numba's "locals" for the + # nesting; and not a plain method's under its own name. Counted by five + # decorators' names alone, the builder's derive function and the + # constructor overload's inner function were left out. + version = f"py{sys.version_info[0]}{sys.version_info[1]}{getattr(sys, 'abiflags', '')}" + code_txt = ("class C:\n @njit(**jit_options)\n def m(self):\n def inner():\n" + " return 1\n return inner\n\n def plain_but_long_" + "p" * 200 + "(self):\n" + " return 2\n\n\n@overload(S, strict=False, jit_options=jit_options)\ndef ol_s(x):\n" + " def a_constructor_named_" + "c" * 60 + "(x):\n return 3\n return ctor\n\n\n" + "@_cres\ndef derive_" + "d" * 50 + "(x):\n return 4\n") + counted = [f"stem.C.m-3.{version}.999.nbc.tmp.", f"stem.C.m.locals.inner-4.{version}.999.nbc.tmp.", + f"stem.ol_s-13.{version}.999.nbc.tmp.", + f"stem.ol_s.locals.a_constructor_named_{'c' * 60}-14.{version}.999.nbc.tmp.", + f"stem.derive_{'d' * 50}-20.{version}.999.nbc.tmp."] + assert preprocessing._longest_cache_file_name("stem", code_txt) == max(len(each) for each in counted) + 16 + assert preprocessing._longest_cache_file_name("stem", "x = 1\n") == 0 + assert preprocessing._longest_cache_file_name("stem", "def plain():\n return 1\n") == 0 + + +@pytest.mark.parametrize("number, base, offered", [ + pytest.param(1, 226, True, id="a one-digit number, three reserved, the base at the limit"), + pytest.param(100, 227, False, id="a hundredth signature at the base two digits left room for"), + pytest.param(1000, 226, False, id="a thousandth signature, wider than the reserve, read by its width"), +]) +def test_a_cache_file_name_with_a_number_wider_than_the_reserve_is_read_by_its_own_width( + tmp_path, monkeypatch, number, base, offered): + # numba numbers a function's data files from one without bound, so a + # hundredth signature's name is a byte longer than two reserved digits + # allowed for, and the offer of a file system that takes the name went to + # a name whose data file that file system then refused; three digits are + # reserved now, and a number the error names that is wider is read by its + # own width. + import numba + import numbox.core.configurations as configurations + + monkeypatch.setattr(numba.config, "CACHE_DIR", "") + monkeypatch.setattr(numba.config, "CACHE_LOCATOR_CLASSES", "", raising=False) + monkeypatch.setattr(configurations, "_name_limit", lambda location: 143) + site = tmp_path / "site" + site.mkdir() + py_file = str(site / "module.py") + with open(py_file, "w", encoding="utf-8") as handle: + handle.write("x = 1\n") + name = f"module.{'f' * (base - len('module.'))}.{number}.nbc.tmp.0123456789abcdef" + counted_in = "characters" if os.name == "nt" or sys.platform == "darwin" else "bytes" + units = len(name) + assert (base + len(".") + max(3, len(str(number))) + len(".nbc") + 21 <= 255) == offered + remedy = configurations.cache_remedy( + py_file, OSError(errno.ENAMETOOLONG, "File name too long", str(site / "__pycache__" / name)), "silence") + offer = ", or NUMBA_CACHE_DIR set to a directory on a file system that takes a name that long" if offered else "" + assert remedy == ( + "numba names the cache files for a function after the source's name and the function's qualified name, " + f"{name}, {units} {counted_in}, which is longer than a name can be on that file system (143), and no shorter " + f"path cures it: a shorter name for the function or its file, which is numbox's to give{offer}; or silence")