Fix inconsistencies

pyproject.toml

@@ -88,6 +88,9 @@ python_version = "3.10"
 warn_return_any = true
 warn_unused_configs = true
 disallow_untyped_defs = true
+# asanypath_native.pyi lives in rust/ (its canonical home, shipped in the native
+# wheel by maturin); keep it resolvable for dev type-checking.
+mypy_path = ["src", "rust"]
 
 [[tool.mypy.overrides]]
 module = "asanypath.cli"
@@ -111,7 +114,7 @@ disable_error_code = ["arg-type", "assignment", "misc", "attr-defined"]
 module = "asanypath.http"
 disallow_untyped_defs = false
 warn_return_any = false
-disable_error_code = ["override", "type-arg", "return-value", "assignment"]
+disable_error_code = ["override", "type-arg", "return-value", "assignment", "attr-defined"]
 
 [[tool.mypy.overrides]]
 module = ["asanypath.s3", "asanypath.gcs", "asanypath.azure", "asanypath.artifactory"]

src/asanypath_native.pyi → rust/asanypath_native.pyi

@@ -100,6 +100,8 @@ async def http_head(*args: Any, **kwargs: Any) -> Any: ...
 async def http_patch(*args: Any, **kwargs: Any) -> Any: ...
 async def http_options(*args: Any, **kwargs: Any) -> Any: ...
 async def http_exists(*args: Any, **kwargs: Any) -> Any: ...
+async def http_scrape_links(*args: Any, **kwargs: Any) -> Any: ...
+async def http_request(*args: Any, **kwargs: Any) -> Any: ...
 
 # -- Unified --
 async def range_read(path: str, start: int, end: int, **kwargs: Any) -> bytes: ...

src/asanypath/_transfer.py

@@ -15,6 +15,8 @@
 from pathlib import Path
 from typing import Any
 
+from asanypath.exceptions import UnsupportedProtocolError
+
 _CHECKSUM_PREFERENCE = ("sha256", "sha1", "md5")
 
 # Default streaming chunk size for chunk_size=None ("auto").
@@ -79,19 +81,25 @@ def _normalized_checksums(checksums: dict[str, str]) -> dict[str, str]:
 def is_remote_destination(dst: Any) -> bool:
     protocol = getattr(dst, "protocol", None)
     if protocol is not None:
-        return protocol != "file"
+        return bool(protocol != "file")
     value = str(dst)
     return "://" in value and not value.startswith("file:")
 
 
 def has_async_transfer_api(dst: Any) -> bool:
-    return inspect.iscoroutinefunction(
-        getattr(dst, "exists", None)
-    ) and inspect.iscoroutinefunction(getattr(dst, "write_bytes", None))
+    try:
+        exists = getattr(dst, "exists", None)
+        write_bytes = getattr(dst, "write_bytes", None)
+    except UnsupportedProtocolError:
+        return False
+    return inspect.iscoroutinefunction(exists) and inspect.iscoroutinefunction(write_bytes)
 
 
 def has_write_api(dst: Any) -> bool:
-    return callable(getattr(dst, "write_bytes", None))
+    try:
+        return callable(getattr(dst, "write_bytes", None))
+    except UnsupportedProtocolError:
+        return False
 
 
 def run_sync_maybe(value: Any) -> Any:

src/asanypath/artifactory.py

@@ -21,8 +21,6 @@
 if TYPE_CHECKING:
     from typing import Self
 
-from asanypath.cloud import CloudPathMixin
-from asanypath.options import AccessGrant, AccessPolicy, AccessPolicyPatch, BackendOptions
 from asanypath_native import (
     art_copy_batch,
     art_delete_batch,
@@ -36,6 +34,9 @@
     art_storage_info,
 )
 
+from asanypath.cloud import CloudPathMixin
+from asanypath.options import AccessGrant, AccessPolicy, AccessPolicyPatch, BackendOptions
+
 
 def _jfrog_conf_token(base_url: str) -> str | None:
     """Return an access token from the JFrog CLI config matching ``base_url``.

src/asanypath/asanypath.py

@@ -447,7 +447,7 @@ async def rglob(
         pass
 
     @abstractmethod
-    async def rmdir(self) -> None:
+    async def rmdir(self, *, recursive: bool = False) -> None:
         pass
 
     @abstractmethod

src/asanypath/asanypath.pyi

@@ -185,7 +185,7 @@ class AsAnyPath(AsAnyPurePath):
     async def rglob(
         self, pattern: str, *, case_sensitive: bool | None = None
     ) -> AsyncIterator[Self]: ...
-    async def rmdir(self) -> None: ...
+    async def rmdir(self, *, recursive: bool = False) -> None: ...
     def samefile(self, other_path: str | PathLike[str]) -> bool: ...
     async def stat(self, *, follow_symlinks: bool = True) -> stat_result: ...
     def symlink_to(

src/asanypath/azure.py

@@ -18,9 +18,6 @@
 from types import SimpleNamespace
 
 import msgspec
-
-from asanypath.cloud import CloudPathMixin
-from asanypath.options import AccessGrant, AccessPolicy, AccessPolicyPatch, BackendOptions
 from asanypath_native import (
     az_copy_batch,
     az_delete_batch,
@@ -37,6 +34,9 @@
     az_put_container_acl,
 )
 
+from asanypath.cloud import CloudPathMixin
+from asanypath.options import AccessGrant, AccessPolicy, AccessPolicyPatch, BackendOptions
+
 AZURE_API_VERSION = "2023-11-03"
 
 

src/asanypath/cloud.py

@@ -6,6 +6,7 @@
 
 from __future__ import annotations
 
+import asyncio
 import fnmatch
 import inspect
 import io
@@ -329,6 +330,7 @@ async def write_bytes(
             ),
             **native_kwargs,
         )
+        type(self)._listing_cache.pop(self._listing_cache_key(), None)
         return len(data)
 
     async def write_text(
@@ -358,6 +360,7 @@ async def unlink(self, missing_ok: bool = False) -> None:
                 item=self._item_path,
                 **native_kwargs,
             )
+            type(self)._listing_cache.pop(self._listing_cache_key(), None)
         except FileNotFoundError:  # pragma: no cover
             if not missing_ok:  # pragma: no cover
                 raise
@@ -610,19 +613,31 @@ async def rmdir(self, *, recursive: bool = False) -> None:
         Without it, only succeeds if the prefix is empty (no children).
         """
         if not recursive:
-            children = [c async for c in self.iterdir()]
-            if children:
+            async for _ in self.iterdir(fresh=True):
                 raise OSError(39, f"Directory not empty: '{self}'")
             return
-        # Recursive: collect all files, then unlink
-        files = []
-        async for child in self.iterdir():
+        await self._rmtree()
+
+    async def _rmtree(self) -> None:
+        """Recursively delete this prefix's contents with concurrency.
+
+        Children are deleted concurrently so the :class:`MicroBatcher` can
+        coalesce the in-flight object deletes into batch API calls; no
+        Python-side semaphore is used because a small bound would only cap
+        the batch size and defeat that coalescing.
+        """
+
+        async def _remove(child: Self) -> None:
             if await child.is_dir():
-                await child.rmdir(recursive=True)
+                await child._rmtree()
             else:
-                files.append(child)
-        for f in files:
-            await f.unlink()
+                await child.unlink()
+
+        children = [_remove(child) async for child in self.iterdir(fresh=True)]
+        for result in await asyncio.gather(*children, return_exceptions=True):
+            if isinstance(result, BaseException):
+                raise result
+        type(self)._listing_cache.pop(self._listing_cache_key(), None)
 
     async def copy(
         self,
@@ -854,13 +869,13 @@ async def walk(
                 **(await self._get_native_kwargs()),
             )
         elif entries:
-            import asyncio
-
             is_dir_results = await asyncio.gather(*(e.is_dir() for e in entries))
         else:
             is_dir_results = []
-        dirs = [e for e, d in zip(entries, is_dir_results) if d]
-        files = [e for e, d in zip(entries, is_dir_results) if not d]
+        dirs: list[Self] = []
+        files: list[Self] = []
+        for entry, is_directory in zip(entries, is_dir_results):
+            (dirs if is_directory else files).append(entry)
         if top_down:
             yield self, [d.name for d in dirs], [f.name for f in files]
         for d in dirs:

src/asanypath/gcs.py

@@ -24,9 +24,6 @@
 from types import SimpleNamespace
 
 import msgspec
-
-from asanypath.cloud import CloudPathMixin
-from asanypath.options import AccessGrant, AccessPolicy, AccessPolicyPatch, BackendOptions
 from asanypath_native import (
     gcs_copy_batch,
     gcs_delete_batch,
@@ -44,6 +41,9 @@
     gcs_put_batch,
 )
 
+from asanypath.cloud import CloudPathMixin
+from asanypath.options import AccessGrant, AccessPolicy, AccessPolicyPatch, BackendOptions
+
 GCS_API_BASE = "https://storage.googleapis.com"
 
 

src/asanypath/http.py

@@ -9,12 +9,10 @@
 from base64 import b64encode
 from collections.abc import AsyncGenerator, AsyncIterator, Callable
 from os import getenv, linesep
-from typing import Literal, TypeVar
+from typing import TYPE_CHECKING, Literal
 from xml.etree.ElementTree import XMLParser
 
 import msgspec
-
-from asanypath.cloud import CloudPathMixin
 from asanypath_native import (
     http_delete,
     http_exists,
@@ -27,12 +25,15 @@
     http_scrape_links,
 )
 
+from asanypath.cloud import CloudPathMixin
+
 try:
     from asanypath_native import http_request
 except ImportError:  # pragma: no cover
     http_request = None  # type: ignore[assignment]
 
-T = TypeVar("T", bound="HTTPPath")
+if TYPE_CHECKING:
+    from typing import Self
 
 
 class _TagCollector:
@@ -306,7 +307,7 @@ async def is_file(self) -> bool:
             return False  # pragma: no cover
         return await self.exists()  # pragma: no cover
 
-    async def iterdir(self) -> AsyncIterator[T]:
+    async def iterdir(self) -> AsyncIterator[Self]:
         """Yield child paths by scraping links from this URL.
 
         Uses the path's ``listing`` configuration (default ``"auto"``).
@@ -334,23 +335,26 @@ async def iterdir(self) -> AsyncIterator[T]:
             headers=self._merge_headers(None),
         )
         for link in urls:
-            # children keep the parent's listing config; _spawn copies it.
-            yield self._child(link.rstrip("/") or link, _trailing_slash=link.endswith("/"))
+            yield type(self)(
+                link,
+                listing=self._listing,
+                listing_attr=self._listing_attr,
+            )
 
     async def walk(
         self,
         top_down: bool = True,
         on_error: Callable[[OSError], object] | None = None,
         follow_symlinks: bool = False,
-    ) -> AsyncIterator[tuple[T, list[str], list[str]]]:
+    ) -> AsyncIterator[tuple[Self, list[str], list[str]]]:
         """Recursively walk the listing tree, yielding (root, dirs, files).
 
         ``follow_symlinks`` is accepted for API parity; HTTP listings have no symlinks.
         """
         if self._resolve_listing() is None:
             raise ValueError("HTTPPath.walk() called with listing=None")
-        dirs: list[T] = []
-        files: list[T] = []
+        dirs: list[Self] = []
+        files: list[Self] = []
         try:
             async for child in self.iterdir():
                 if child._trailing_slash:
@@ -372,7 +376,9 @@ async def walk(
         if not top_down:
             yield self, [d.name for d in dirs], [f.name for f in files]
 
-    async def glob(self, pattern: str, *, case_sensitive: bool | None = None) -> AsyncIterator[T]:
+    async def glob(
+        self, pattern: str, *, case_sensitive: bool | None = None
+    ) -> AsyncIterator[Self]:
         """Match descendants against a glob pattern (uses ``listing``)."""
         if self._resolve_listing() is None:
             raise ValueError("HTTPPath.glob() called with listing=None")
@@ -409,10 +415,10 @@ async def read_text(
             case _:
                 return text.replace(newline, "\n")
 
-    async def rename(self, target: str, *, force: bool = False) -> T:
+    async def rename(self, target: str, *, force: bool = False) -> Self:
         return await super().rename(target, force=force)
 
-    async def replace(self, target: str) -> T:
+    async def replace(self, target: str) -> Self:
         return await self.rename(target, force=True)
 
     async def touch(self, mode: int = 0o666, exist_ok: bool = True) -> None:
@@ -433,13 +439,15 @@ async def unlink(self, missing_ok: bool = False) -> None:
         except Exception as e:
             raise OSError(f"Failed to delete {self}: {e}") from None
 
-    async def rglob(self, pattern: str, *, case_sensitive: bool | None = None) -> AsyncIterator[T]:
+    async def rglob(
+        self, pattern: str, *, case_sensitive: bool | None = None
+    ) -> AsyncIterator[Self]:
         if self._resolve_listing() is None:
             raise ValueError("HTTPPath.rglob() called with listing=None")
         async for entry in self.glob(pattern, case_sensitive=case_sensitive):
             yield entry
 
-    async def rmdir(self) -> None:
+    async def rmdir(self, *, recursive: bool = False) -> None:
         raise NotImplementedError(f"{type(self).__name__} does not support rmdir")
 
     async def stat(self, *, follow_symlinks: bool = True):

src/asanypath/s3.py

@@ -12,16 +12,6 @@
 from typing import cast
 
 import msgspec
-from yarl import URL
-
-from asanypath.cloud import CloudPathMixin
-from asanypath.options import (
-    AccessAction,
-    AccessGrant,
-    AccessPolicy,
-    AccessPolicyPatch,
-    BackendOptions,
-)
 from asanypath_native import (
     s3_copy_batch,
     s3_delete_batch,
@@ -38,6 +28,16 @@
     s3_put_acl,
     s3_put_batch,
 )
+from yarl import URL
+
+from asanypath.cloud import CloudPathMixin
+from asanypath.options import (
+    AccessAction,
+    AccessGrant,
+    AccessPolicy,
+    AccessPolicyPatch,
+    BackendOptions,
+)
 
 
 def _default_s3_endpoint(region: str) -> str: