Skip to content

Fix/ls cli #42

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
hoshimura merged 5 commits into from
Oct 21, 2026
Merged

Fix/ls cli #42

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
ded5a05
Return listing metadata from one call; rebuild ls on scandir
hoshimura Oct 16, 2026
2133893
ls: hide dotfiles unless -a; render with plain ANSI instead of Rich Text
hoshimura Oct 19, 2026
23ddaae
ls: render with Rich Text again; keep -a dotfile handling
hoshimura Oct 19, 2026
07bb0b6
Import backends lazily so local commands skip cloud modules
hoshimura Oct 19, 2026
59f26d9
Walk FTP directories through the confined listing
hoshimura Oct 21, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 14 additions & 1 deletion README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,11 @@ info = await path.stat()
async for child in path.parent.iterdir():
print(child.name)

# List with metadata: is_dir always; size/mtime with stat=True.
# Fields the listing already reports cost no extra requests.
async for entry in path.parent.scandir(stat=True):
print(entry.path.name, entry.is_dir, entry.size, entry.mtime)

# Glob
async for p in AsAnyPath("s3://bucket/logs/").glob("**/*.json"):
print(p)
Expand Down Expand Up @@ -404,6 +409,14 @@ asanypath exists s3://bucket/file.txt
# Print file contents
asanypath cat s3://bucket/file.txt

# List directory in columns; directories end with "/", dotfiles need -a
asanypath ls s3://bucket/prefix/
asanypath ls -a s3://bucket/prefix/

# Long listing (size, modified) and tree view
asanypath ls -l s3://bucket/prefix/
asanypath ls -r -d 2 s3://bucket/prefix/

# List directory (Artifactory with token)
asanypath ls art://repo/path/ --token mytoken

Expand Down Expand Up @@ -650,7 +663,7 @@ All path implementations support:
- `stat()` / `checksums()` (cloud backends)
- `mkdir(parents, exist_ok)` / `touch(exist_ok)`
- `unlink(missing_ok)` / `rmdir(recursive)`
- `iterdir()` / `rename(target)`
- `iterdir()` / `scandir(stat)` / `rename(target)`
- `get_access_policy()` / `update_access_policy(policy_patch)`

### Backend Operation Options
Expand Down
3 changes: 0 additions & 3 deletions rust/asanypath_native.pyi
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@ async def s3_exists_batch(*args: Any, **kwargs: Any) -> Any: ...
async def s3_list_batch(*args: Any, **kwargs: Any) -> Any: ...
async def s3_head_batch(*args: Any, **kwargs: Any) -> Any: ...
async def s3_is_dir(*args: Any, **kwargs: Any) -> Any: ...
async def s3_is_dir_batch(*args: Any, **kwargs: Any) -> Any: ...
async def s3_list_buckets(*args: Any, **kwargs: Any) -> Any: ...
async def s3_get_acl(*args: Any, **kwargs: Any) -> Any: ...
async def s3_put_acl(*args: Any, **kwargs: Any) -> Any: ...
Expand Down Expand Up @@ -66,7 +65,6 @@ async def gcs_list_batch(*args: Any, **kwargs: Any) -> Any: ...
async def gcs_head(*args: Any, **kwargs: Any) -> Any: ...
async def gcs_head_batch(*args: Any, **kwargs: Any) -> Any: ...
async def gcs_is_dir(*args: Any, **kwargs: Any) -> Any: ...
async def gcs_is_dir_batch(*args: Any, **kwargs: Any) -> Any: ...
async def gcs_list_buckets(*args: Any, **kwargs: Any) -> Any: ...
def gcs_presign(
bucket: str,
Expand All @@ -87,7 +85,6 @@ async def az_exists_batch(*args: Any, **kwargs: Any) -> Any: ...
async def az_list_batch(*args: Any, **kwargs: Any) -> Any: ...
async def az_head_batch(*args: Any, **kwargs: Any) -> Any: ...
async def az_is_dir(*args: Any, **kwargs: Any) -> Any: ...
async def az_is_dir_batch(*args: Any, **kwargs: Any) -> Any: ...
async def az_list_containers(*args: Any, **kwargs: Any) -> Any: ...
def az_presign(
endpoint: str,
Expand Down
211 changes: 88 additions & 123 deletions rust/src/azure.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,9 @@ use quick_xml::Reader;
use quick_xml::events::Event;
use serde::{Deserialize, Serialize};
use sha2::Sha256;
use std::collections::HashSet;

use crate::http::{do_request, do_request_light, status_to_pyerr};
use crate::listing::{ListEntry, Listing};
use crate::xml::local_name;

type HmacSha256 = Hmac<Sha256>;
Expand Down Expand Up @@ -428,16 +428,9 @@ async fn do_az_is_dir(creds: &AzCreds, prefix: &str, use_h2: bool) -> Result<boo
Ok(false)
}

async fn do_az_list(creds: &AzCreds, prefix: &str, use_h2: bool) -> Result<Vec<String>, PyErr> {
let current_key = prefix.trim_matches('/').to_string();
let search_prefix = if current_key.is_empty() {
String::new()
} else {
format!("{}/", current_key)
};

let mut found_all = HashSet::new();
let mut uris = Vec::new();
async fn do_az_list(creds: &AzCreds, prefix: &str, use_h2: bool) -> Result<Vec<ListEntry>, PyErr> {
let mut listing = Listing::new(format!("az://{}", creds.container), prefix);
let search_prefix = listing.search_prefix().to_string();
let mut marker: Option<String> = None;

loop {
Expand All @@ -464,100 +457,68 @@ async fn do_az_list(creds: &AzCreds, prefix: &str, use_h2: bool) -> Result<Vec<S
&body,
));
}
marker = parse_az_listing_page(&body, &mut listing);
if marker.is_none() {
break;
}
}

// Parse XML
let mut reader = Reader::from_reader(body.as_ref());
reader.config_mut().trim_text(true);
let mut buf = Vec::new();
let mut next_marker: Option<String> = None;
let mut in_blob_prefix = false;
Ok(listing.finish())
}

loop {
match reader.read_event_into(&mut buf) {
Ok(Event::Start(ref e)) => {
let name = e.name();
let tag = local_name(name.as_ref());
match tag {
"BlobPrefix" => {
in_blob_prefix = true;
}
"Name" => {
let text = reader
.read_text(e.name())
.ok()
.map(|text| text.xml10_content().into_owned())
.unwrap_or_default();
if in_blob_prefix {
let value = text.trim_end_matches('/').to_string();
if !value.is_empty()
&& value != current_key
&& found_all.insert(value.clone())
{
uris.push(format!("az://{}/{}", creds.container, value));
}
} else {
// Regular blob
let name = &text;
if name.is_empty() || name.trim_end_matches('/') == current_key {
// skip
} else {
let relative = if !search_prefix.is_empty()
&& name.starts_with(&search_prefix)
{
&name[search_prefix.len()..]
} else {
name.as_str()
};
let relative = relative.trim_start_matches('/');
let resolved = if !relative.is_empty() {
let first = relative.split('/').next().unwrap_or(relative);
format!("{}{}", search_prefix, first)
} else {
name.to_string()
};
let value = resolved.trim_end_matches('/').to_string();
if !value.is_empty()
&& value != current_key
&& found_all.insert(value.clone())
{
uris.push(format!("az://{}/{}", creds.container, value));
}
}
}
}
"NextMarker" => {
let text = reader
.read_text(e.name())
.ok()
.map(|text| text.xml10_content().into_owned())
.unwrap_or_default();
if !text.is_empty() {
next_marker = Some(text);
}
}
_ => {}
fn parse_az_listing_page(body: &[u8], listing: &mut Listing) -> Option<String> {
let mut reader = Reader::from_reader(body);
reader.config_mut().trim_text(true);
let mut buf = Vec::new();
let mut text = String::new();
let mut in_blob = false;
let mut in_prefix = false;
let mut name = String::new();
let mut size = None;
let mut modified = None;
let mut next_marker = None;

loop {
match reader.read_event_into(&mut buf) {
Ok(Event::Start(e)) => {
let tag = e.name();
match local_name(tag.as_ref()) {
"Blob" => {
in_blob = true;
name.clear();
size = None;
modified = None;
}
"BlobPrefix" => in_prefix = true,
_ => {}
}
Ok(Event::End(ref e)) => {
let name = e.name();
if local_name(name.as_ref()) == "BlobPrefix" {
in_blob_prefix = false;
text.clear();
}
Ok(Event::Text(e)) => text.push_str(&e.xml10_content()),
Ok(Event::End(e)) => {
let tag = e.name();
match local_name(tag.as_ref()) {
"Name" if in_prefix => listing.add_prefix(&text),
"Name" if in_blob => name = std::mem::take(&mut text),
"Content-Length" if in_blob => size = text.parse().ok(),
"Last-Modified" if in_blob => modified = Some(std::mem::take(&mut text)),
"Blob" => {
in_blob = false;
listing.add_object(&name, size, modified.take());
}
"BlobPrefix" => in_prefix = false,
"NextMarker" if !text.is_empty() => {
next_marker = Some(std::mem::take(&mut text));
}
_ => {}
}
Ok(Event::Eof) => break,
Err(_) => break,
_ => {}
}
buf.clear();
}

marker = next_marker;
if marker.is_none() {
break;
Ok(Event::Eof) | Err(_) => break,
_ => {}
}
buf.clear();
}

Ok(uris)
next_marker
}

/// List all containers in the account (GET /?comp=list). Returns bare names.
Expand Down Expand Up @@ -1096,33 +1057,6 @@ pub fn az_exists_batch<'py>(
})
}

#[pyfunction]
pub fn az_is_dir_batch<'py>(
py: Python<'py>,
endpoint: String,
container: String,
prefixes: Vec<String>,
account_name: String,
account_key: Option<String>,
sas_token: Option<String>,
use_h2: Option<bool>,
) -> PyResult<Bound<'py, PyAny>> {
let h2 = use_h2.unwrap_or(false);
pyo3_async_runtimes::tokio::future_into_py(py, async move {
let creds = az_creds!(endpoint, container, account_name, account_key, sas_token);
let futs: Vec<_> = prefixes
.iter()
.map(|p| do_az_is_dir(&creds, p, h2))
.collect();
let results = join_all(futs).await;
let mut out = Vec::with_capacity(results.len());
for r in results {
out.push(r?);
}
Ok(out)
})
}

// ---------------------------------------------------------------------------
// Presigned URL generation (Service SAS)
// ---------------------------------------------------------------------------
Expand Down Expand Up @@ -1226,3 +1160,34 @@ fn format_iso8601(secs: u64) -> String {
year, month, day, hours, minutes, seconds
)
}

#[cfg(test)]
mod tests {
use super::{Listing, parse_az_listing_page};

#[test]
fn parses_listing_page_metadata() {
let body = br#"<?xml version="1.0" encoding="utf-8"?>
<EnumerationResults ContainerName="c"><Prefix>dir/</Prefix><Blobs>
<Blob><Name>dir/a.txt</Name><Properties>
<Last-Modified>Tue, 02 Jan 2024 03:04:05 GMT</Last-Modified>
<Content-Length>42</Content-Length>
</Properties></Blob>
<BlobPrefix><Name>dir/sub/</Name></BlobPrefix>
</Blobs><NextMarker /></EnumerationResults>"#;
let mut listing = Listing::new("az://c".into(), "dir");
assert_eq!(parse_az_listing_page(body, &mut listing), None);
assert_eq!(
listing.finish(),
vec![
(
"az://c/dir/a.txt".into(),
false,
Some(42),
Some("Tue, 02 Jan 2024 03:04:05 GMT".into())
),
("az://c/dir/sub".into(), true, None, None),
]
);
}
}
Loading
Loading