diff --git a/src/install/PackageInstall.rs b/src/install/PackageInstall.rs index 6adf994d1ad9..f9b2b5b19ee8 100644 --- a/src/install/PackageInstall.rs +++ b/src/install/PackageInstall.rs @@ -1024,7 +1024,6 @@ impl<'a> PackageInstall<'a> { Ok(w) => w, Err(err) => return Ok(InstallResult::fail(err.into(), Step::OpeningCacheDir, None)), }; - walker_.resolve_unknown_entry_types = true; fn copy(destination_dir_: &Dir, walker: &mut Walker) -> crate::Result<()> { let mut stackpath = [0u8; path::MAX_PATH_BYTES]; @@ -1181,12 +1180,11 @@ impl<'a> PackageInstall<'a> { &[] }; - let mut walker = bun_core::handle_oom(walker_skippable::walk_owned( + let walker = bun_core::handle_oom(walker_skippable::walk_owned( cached_package_dir, &[] as &[&OSPathSlice], skip_dirs, )); - walker.resolve_unknown_entry_types = true; #[cfg(not(windows))] { diff --git a/src/install/PackageManager/PackageManagerResolution.rs b/src/install/PackageManager/PackageManagerResolution.rs index 1f1b0b65edf0..2b3feaf27e24 100644 --- a/src/install/PackageManager/PackageManagerResolution.rs +++ b/src/install/PackageManager/PackageManagerResolution.rs @@ -119,6 +119,7 @@ impl PackageManager { Err(e) => return Err(e), }; let mut iter = bun_sys::iterate_dir(dir.fd); + iter.resolve_unknown_entry_types = true; loop { let entry = match iter.next() { diff --git a/src/install/PackageManager/updatePackageJSONAndInstall.rs b/src/install/PackageManager/updatePackageJSONAndInstall.rs index cb75ce464347..f206bb661ecf 100644 --- a/src/install/PackageManager/updatePackageJSONAndInstall.rs +++ b/src/install/PackageManager/updatePackageJSONAndInstall.rs @@ -742,6 +742,7 @@ pub(super) fn remove_leftover_node_modules( match bun_sys::open_dir_for_iteration(cwd.fd(), manager.options.bin_path.as_bytes()) { Ok(node_modules_bin) => { let mut iter = bun_sys::iterate_dir(node_modules_bin); + iter.resolve_unknown_entry_types = true; 'iterator: loop { let Ok(Some(entry)) = iter.next() else { break }; match entry.kind { diff --git a/src/install/bin.rs b/src/install/bin.rs index 7bb19bf33bbf..09c03905452b 100644 --- a/src/install/bin.rs +++ b/src/install/bin.rs @@ -755,7 +755,7 @@ fn normalized_bin_name(name: &[u8]) -> &[u8] { /// verbatim from package.json, so without this check a malicious package could /// point a bin link at (and chmod) an arbitrary file on disk (the bug class /// npm fixed as CVE-2019-16775). -pub(crate) fn bin_target_escapes_package_dir(target: &[u8]) -> bool { +pub fn bin_target_escapes_package_dir(target: &[u8]) -> bool { if path::is_absolute(target) { return true; } @@ -1796,6 +1796,7 @@ impl<'a> Linker<'a> { let abs_dest_dir_end = dest_off; let mut iter = sys::iterate_dir(target_dir); + iter.resolve_unknown_entry_types = true; while let Some(entry) = iter.next().unwrap_or(None) { match entry.kind { sys::EntryKind::SymLink | sys::EntryKind::File => { @@ -1953,6 +1954,7 @@ impl<'a> Linker<'a> { let abs_dest_dir_end = dest_off; let mut iter = sys::iterate_dir(target_dir); + iter.resolve_unknown_entry_types = true; while let Some(entry) = iter.next().unwrap_or(None) { match entry.kind { sys::EntryKind::SymLink | sys::EntryKind::File => { diff --git a/src/install/isolated_install/FileCopier.rs b/src/install/isolated_install/FileCopier.rs index 2cbed827c689..aae813116410 100644 --- a/src/install/isolated_install/FileCopier.rs +++ b/src/install/isolated_install/FileCopier.rs @@ -35,16 +35,7 @@ impl FileCopier { Ok(FileCopier { src_path, dest_subpath, - walker: { - let mut w = walker_skippable::walk( - src_dir, - // bun.default_allocator → deleted (global mimalloc) - &[], - skip_dirnames, - )?; - w.resolve_unknown_entry_types = true; - w - }, + walker: walker_skippable::walk(src_dir, &[], skip_dirnames)?, }) } diff --git a/src/install/isolated_install/Hardlinker.rs b/src/install/isolated_install/Hardlinker.rs index 4856f2de0c90..186f2e33748c 100644 --- a/src/install/isolated_install/Hardlinker.rs +++ b/src/install/isolated_install/Hardlinker.rs @@ -34,16 +34,7 @@ impl Hardlinker { Ok(Hardlinker { src, dest, - walker: { - let mut w = bun_sys::walker_skippable::walk( - folder_dir, - // bun.default_allocator dropped — global mimalloc - &[], - skip_dirnames, - )?; - w.resolve_unknown_entry_types = true; - w - }, + walker: bun_sys::walker_skippable::walk(folder_dir, &[], skip_dirnames)?, }) } diff --git a/src/install/prune.rs b/src/install/prune.rs index 01aa08b4607c..488295d72f56 100644 --- a/src/install/prune.rs +++ b/src/install/prune.rs @@ -1343,24 +1343,17 @@ fn lstat_kind(dir: &Dir, name: &[u8]) -> EntryKind { } } -fn entry_kind(dir: &Dir, name: &[u8], kind: EntryKind) -> EntryKind { - if kind != EntryKind::Unknown { - return kind; - } - lstat_kind(dir, name) -} - fn read_entries(dir: &Dir) -> Vec<(Box<[u8]>, EntryKind)> { let mut out = Vec::new(); let mut iter = sys::iterate_dir(dir.fd()); + iter.resolve_unknown_entry_types = true; while let Ok(Some(entry)) = iter.next() { let name = entry.name.slice_u8(); if name.first() == Some(&b'.') { continue; } - let kind = entry_kind(dir, name, entry.kind); - if kind == EntryKind::Directory || kind == EntryKind::SymLink { - out.push((name.into(), kind)); + if entry.kind == EntryKind::Directory || entry.kind == EntryKind::SymLink { + out.push((name.into(), entry.kind)); } } out @@ -1780,9 +1773,10 @@ fn prune_bins(dir: &Dir) { }; let mut dangling: Vec> = Vec::new(); let mut iter = sys::iterate_dir(bin.fd()); + iter.resolve_unknown_entry_types = true; while let Ok(Some(entry)) = iter.next() { let name = entry.name.slice_u8(); - if entry_kind(&bin, name, entry.kind) == EntryKind::SymLink && is_dangling(&bin, name) { + if entry.kind == EntryKind::SymLink && is_dangling(&bin, name) { dangling.push(name.into()); } } diff --git a/src/runtime/cli/build_command.rs b/src/runtime/cli/build_command.rs index 92003b6f7c64..5aa8f63165fc 100644 --- a/src/runtime/cli/build_command.rs +++ b/src/runtime/cli/build_command.rs @@ -1355,7 +1355,6 @@ pub(crate) fn collect_compile_assets( Ok(w) => w, Err(_) => bun_core::out_of_memory(), }; - walker.resolve_unknown_entry_types = true; loop { let entry = match walker.next() { Ok(Some(e)) => e, diff --git a/src/runtime/cli/bunx_command.rs b/src/runtime/cli/bunx_command.rs index 15e2b2287af6..79a83dbe81dd 100644 --- a/src/runtime/cli/bunx_command.rs +++ b/src/runtime/cli/bunx_command.rs @@ -335,11 +335,22 @@ impl BunxCommand { if let Some(dirs) = expr.as_property(b"directories") { if let Some(bin_prop) = dirs.expr.as_property(b"bin") { - if let Some(dir_name) = bin_prop.expr.as_utf8_string_literal() { - let bin_dir = bun_sys::openat_a(dir_fd, dir_name, O::RDONLY | O::DIRECTORY, 0)?; + // Same values the bin linker refuses to link from (`bin.rs`, `Tag::Dir`). + if let Some(dir_name) = bin_prop.expr.as_utf8_string_literal().filter(|dir| { + !dir.is_empty() && !bun_install::bin::bin_target_escapes_package_dir(dir) + }) { + // `directories.bin` is relative to the package, not to `dir_fd`. + use bun_paths::platform::Auto; + let package_dir = + bun_paths::resolve_path::dirname::(subpath_z.as_bytes()); + let bin_dir_path = + bun_paths::resolve_path::join_z::(&[package_dir, dir_name]); + let bin_dir = + bun_sys::openat(dir_fd, bin_dir_path, O::RDONLY | O::DIRECTORY, 0)?; // Fd is non-owning Copy; guard it. let _close_bin_dir = bun_sys::CloseOnDrop::new(bin_dir); let mut iterator = bun_sys::dir_iterator::iterate(bin_dir); + iterator.resolve_unknown_entry_types = true; let mut entry = iterator.next(); loop { let current = match entry { diff --git a/src/runtime/cli/create_command.rs b/src/runtime/cli/create_command.rs index 62f438717a75..dd87b5b68e1c 100644 --- a/src/runtime/cli/create_command.rs +++ b/src/runtime/cli/create_command.rs @@ -1820,6 +1820,7 @@ impl Example { for folder in &folders { if folder.fd() != bun_sys::Fd::invalid() { let mut iter = bun_sys::dir_iterator::iterate(folder.fd()); + iter.resolve_unknown_entry_types = true; 'loop_: while let Some(entry) = iter.next().ok().flatten() { let entry_name = entry.name.slice_u8(); diff --git a/src/runtime/cli/init_command.rs b/src/runtime/cli/init_command.rs index 142e3db5f694..ffa25c30e523 100644 --- a/src/runtime/cli/init_command.rs +++ b/src/runtime/cli/init_command.rs @@ -541,6 +541,7 @@ impl InitCommand { let _ = bun_sys::close(d); }); let mut it = bun_sys::iterate_dir(dir); + it.resolve_unknown_entry_types = true; while let Some(file) = it.next().map_err(crate::Error::from)? { if file.kind != bun_sys::FileKind::File { continue; diff --git a/src/runtime/cli/pack_command.rs b/src/runtime/cli/pack_command.rs index 4b767be83280..b0e02568b97e 100644 --- a/src/runtime/cli/pack_command.rs +++ b/src/runtime/cli/pack_command.rs @@ -476,6 +476,7 @@ fn iterate_included_project_tree( }); let mut dir_iter = DirIterator::iterate(Fd::from_std_dir(&dir)); + dir_iter.resolve_unknown_entry_types = true; 'next_entry: while let Some(entry) = dir_iter.next().ok().flatten() { // On iterator error, treat as end of iteration. if entry.kind != bun_sys::FileKind::File && entry.kind != bun_sys::FileKind::Directory { @@ -713,6 +714,7 @@ fn add_entire_tree( } let mut iter = DirIterator::iterate(Fd::from_std_dir(&dir)); + iter.resolve_unknown_entry_types = true; 'next_entry: while let Some(entry) = iter.next().ok().flatten() { if entry.kind != bun_sys::FileKind::File && entry.kind != bun_sys::FileKind::Directory { continue; @@ -884,6 +886,7 @@ fn iterate_bundled_deps( let mut additional_bundled_deps: Vec = Vec::new(); let mut iter = DirIterator::iterate(Fd::from_std_dir(&dir)); + iter.resolve_unknown_entry_types = true; while let Some(entry) = iter.next().ok().flatten() { if entry.kind != bun_sys::FileKind::Directory { continue; @@ -1022,6 +1025,7 @@ fn add_bundled_dep( let DirInfo(dir, dir_subpath, dir_depth) = dir_info; let mut iter = DirIterator::iterate(Fd::from_std_dir(&dir)); + iter.resolve_unknown_entry_types = true; while let Some(entry) = iter.next().ok().flatten() { if entry.kind != bun_sys::FileKind::File && entry.kind != bun_sys::FileKind::Directory { continue; @@ -1284,6 +1288,7 @@ fn iterate_project_tree( } let mut dir_iter = DirIterator::iterate(Fd::from_std_dir(&dir)); + dir_iter.resolve_unknown_entry_types = true; 'next_entry: while let Some(entry) = dir_iter.next().ok().flatten() { if entry.kind != bun_sys::FileKind::File && entry.kind != bun_sys::FileKind::Directory { continue; diff --git a/src/runtime/cli/pm_licenses_command.rs b/src/runtime/cli/pm_licenses_command.rs index 0079a85383ee..f7f23f5a319d 100644 --- a/src/runtime/cli/pm_licenses_command.rs +++ b/src/runtime/cli/pm_licenses_command.rs @@ -590,6 +590,7 @@ fn list_dir(path: &[u8]) -> Vec<(Box<[u8]>, FileKind)> { return out; }; let mut iter = bun_sys::iterate_dir(dir.fd()); + iter.resolve_unknown_entry_types = true; while let Ok(Some(entry)) = iter.next() { out.push((entry.name.slice_u8().into(), entry.kind)); } diff --git a/src/runtime/cli/publish_command.rs b/src/runtime/cli/publish_command.rs index b408185fc92f..cf9dca3972eb 100644 --- a/src/runtime/cli/publish_command.rs +++ b/src/runtime/cli/publish_command.rs @@ -1589,6 +1589,7 @@ impl PublishCommand { }); let mut iter = DirIterator::iterate(workspace_dir); + iter.resolve_unknown_entry_types = true; while let Some(entry) = iter.next().ok().flatten() { if entry.kind == bun_sys::EntryKind::Directory { continue; @@ -1811,6 +1812,7 @@ impl PublishCommand { }); let mut iter = DirIterator::iterate(dir); + iter.resolve_unknown_entry_types = true; while let Some(entry) = iter.next().ok().flatten() { let (name, subpath): (&'static ZStr, &'static ZStr) = { // Entry names are UTF-8 on every platform. diff --git a/src/runtime/node/dir_iterator.rs b/src/runtime/node/dir_iterator.rs index 1aab5848dd5c..a3fbda99b107 100644 --- a/src/runtime/node/dir_iterator.rs +++ b/src/runtime/node/dir_iterator.rs @@ -19,21 +19,30 @@ pub struct IteratorResult { /// `RawSlice` invariant: borrows the iterator's `getdents` buffer /// (streaming-iterator contract — invalidated on next `next()` call). /// The kernel writes `d_name` NUL-terminated, so the backing has a NUL at - /// `[name.len()]` (see `name_assume_z`). + /// `[name.len()]` (see `resolve_unknown_kind`). pub name: RawSlice, pub(crate) kind: EntryKind, } impl IteratorResult { - /// The entry name as a NUL-terminated `&ZStr` — the POSIX `d_name` is always - /// NUL-terminated in the `getdents` buffer. - #[inline] - pub(crate) fn name_assume_z(&self) -> &bun_core::ZStr { + /// See `NewWrappedIterator::resolve_unknown_entry_types`. + #[cfg(not(windows))] + fn resolve_unknown_kind(&mut self, dir: Fd) { + if self.kind != EntryKind::Unknown { + return; + } let s = self.name.slice(); // SAFETY: `d_name` is NUL-terminated by the kernel; `name` points at it // with len excluding the NUL, so `[len] == 0`. - unsafe { bun_core::ZStr::from_raw(s.as_ptr(), s.len()) } + let name = unsafe { bun_core::ZStr::from_raw(s.as_ptr(), s.len()) }; + if let Ok(st) = sys::lstatat(dir, name) { + self.kind = sys::kind_from_mode(st.st_mode as sys::Mode); + } } + + /// The Windows iterator always knows the kind. + #[cfg(windows)] + fn resolve_unknown_kind(&mut self, _dir: Fd) {} } pub type Result = sys::Result>; @@ -412,9 +421,7 @@ mod platform { libc::DT_LNK => EntryKind::SymLink, libc::DT_REG => EntryKind::File, libc::DT_SOCK => EntryKind::UnixDomainSocket, - // DT_UNKNOWN: Some filesystems (e.g., bind mounts, FUSE, NFS) - // don't provide d_type. Callers should use lstatat() to determine - // the type when needed (lazy stat pattern for performance). + // DT_UNKNOWN: see `NewWrappedIterator::resolve_unknown_entry_types`. _ => EntryKind::Unknown, }; return Ok(Some(IteratorResult { @@ -858,12 +865,20 @@ where (): WrappedSelect, { pub(crate) iter: NewIterator, + /// As `bun_sys::dir_iterator::WrappedIterator::resolve_unknown_entry_types`; ignored by the `IS_U16` (Windows) iterator. + pub(crate) resolve_unknown_entry_types: bool, } impl NewWrappedIterator { #[inline] pub(crate) fn next(&mut self) -> Result { - self.iter.next() + let mut entry = self.iter.next()?; + if self.resolve_unknown_entry_types { + if let Some(entry) = entry.as_mut() { + entry.resolve_unknown_kind(self.iter.dir); + } + } + Ok(entry) } } @@ -892,6 +907,7 @@ where buf: platform::DirentBuf([0u8; 8192]), received_eof: false, }, + resolve_unknown_entry_types: false, }; } #[cfg(any(target_os = "linux", target_os = "android"))] @@ -904,6 +920,7 @@ where // zero-init avoids the invalid_value lint on [u8; N] buf: platform::DirentBuf([0u8; 8192]), }, + resolve_unknown_entry_types: false, }; } #[cfg(target_os = "freebsd")] @@ -916,6 +933,7 @@ where // zero-init avoids the invalid_value lint on [u8; N] buf: platform::DirentBuf([0u8; 8192]), }, + resolve_unknown_entry_types: false, }; } #[cfg(windows)] @@ -932,6 +950,7 @@ where name_data: unsafe { bun_core::ffi::zeroed_unchecked() }, name_filter: None, }, + resolve_unknown_entry_types: false, }; } #[cfg(target_os = "wasi")] @@ -945,6 +964,7 @@ where // zero-init avoids the invalid_value lint on [u8; N] buf: [0u8; 8192], }, + resolve_unknown_entry_types: false, }; } } diff --git a/src/runtime/node/node_fs.rs b/src/runtime/node/node_fs.rs index 4919642333e9..7084d5b03ce5 100644 --- a/src/runtime/node/node_fs.rs +++ b/src/runtime/node/node_fs.rs @@ -2024,6 +2024,7 @@ mod _async_tasks { let mut iterator = DirIterator::iterate::(fd); #[cfg(not(windows))] let mut iterator = DirIterator::iterate::(fd); + iterator.resolve_unknown_entry_types = true; let mut entry = iterator.next(); loop { let current = match entry { @@ -6342,6 +6343,8 @@ impl NodeFS { // so it is called inline on every exit path below instead. let mut iterator = DirIterator::WrappedIterator::init(fd); + // Only Dirent results expose the kind. + iterator.resolve_unknown_entry_types = T::IS_DIRENT; loop { let current = match iterator.next() { Err(err) => { @@ -6365,18 +6368,13 @@ impl NodeFS { ); } - let utf8_name = current.name.slice(); - // On filesystems that return DT_UNKNOWN (e.g. FUSE, bind mounts), - // fall back to lstat to determine the real file kind. - let kind = if T::IS_DIRENT && current.kind == sys::FileKind::Unknown { - match sys::lstatat(fd, current.name_assume_z()) { - Ok(st) => sys::kind_from_mode(st.st_mode as Mode), - Err(_) => current.kind, - } - } else { - current.kind - }; - T::append_entry(entries, utf8_name, &dirent_path, kind, args.encoding); + T::append_entry( + entries, + current.name.slice(), + &dirent_path, + current.kind, + args.encoding, + ); } dirent_path.deref(); @@ -6517,6 +6515,7 @@ impl NodeFS { }); let mut iterator = DirIterator::WrappedIterator::init(fd); + iterator.resolve_unknown_entry_types = true; let mut dirent_path_prev = BunString::EMPTY; let mut spill: Vec = Vec::new(); let mut dirent_spill: Vec = Vec::new(); @@ -6560,9 +6559,6 @@ impl NodeFS { let name_to_copy_z = unsafe { ZStr::from_raw(name_to_copy.as_ptr(), name_to_copy.len()) }; - // Track effective kind - may be resolved from .unknown via stat - let mut effective_kind = current.kind; - 'enqueue: { match current.kind { // a symlink might be a directory or might not be @@ -6579,22 +6575,6 @@ impl NodeFS { if utf8_name.len() + 1 + name_to_copy.len() > paths::MAX_PATH_BYTES { break 'enqueue; } async_task.enqueue(name_to_copy_z); } - // Some filesystems (e.g., Docker bind mounts, FUSE, NFS) return - // DT_UNKNOWN for d_type. Use lstatat to determine the actual type. - sys::FileKind::Unknown => { - if utf8_name.len() + 1 + name_to_copy.len() > paths::MAX_PATH_BYTES { break 'enqueue; } - // Lazy stat to determine the actual kind (lstatat to not follow symlinks) - match sys::lstatat(fd, current.name_assume_z()) { - Ok(st) => { - let real_kind = sys::kind_from_mode(st.st_mode as Mode); - effective_kind = real_kind; - if matches!(real_kind, sys::FileKind::Directory | sys::FileKind::SymLink) { - async_task.enqueue(name_to_copy_z); - } - } - Err(_) => {} // Skip entries we can't stat - } - } _ => {} } } @@ -6616,7 +6596,7 @@ impl NodeFS { utf8_name, name_to_copy, &dirent_path_prev, - effective_kind, + current.kind, async_task.encoding, false, ); @@ -6714,6 +6694,7 @@ impl NodeFS { }); let mut iterator = DirIterator::WrappedIterator::init(fd); + iterator.resolve_unknown_entry_types = true; let mut dirent_path_prev = BunString::DEAD; loop { @@ -6738,9 +6719,6 @@ impl NodeFS { .as_bytes() }; - // Track effective kind - may be resolved from .unknown via stat - let mut effective_kind = current.kind; - 'enqueue: { match current.kind { // a symlink might be a directory or might not be @@ -6756,24 +6734,6 @@ impl NodeFS { owned.push(0); stack.push_back(owned); } - // Some filesystems (e.g., Docker bind mounts, FUSE, NFS) return - // DT_UNKNOWN for d_type. Use lstatat to determine the actual type. - sys::FileKind::Unknown => { - if utf8_name.len() + 1 + name_to_copy.len() > paths::MAX_PATH_BYTES { break 'enqueue; } - match sys::lstatat(fd, current.name_assume_z()) { - Ok(st) => { - let real_kind = sys::kind_from_mode(st.st_mode as Mode); - effective_kind = real_kind; - if matches!(real_kind, sys::FileKind::Directory | sys::FileKind::SymLink) { - let mut owned = Vec::with_capacity(name_to_copy.len() + 1); - owned.extend_from_slice(name_to_copy); - owned.push(0); - stack.push_back(owned); - } - } - Err(_) => {} // Skip entries we can't stat - } - } _ => {} } } @@ -6798,7 +6758,7 @@ impl NodeFS { utf8_name, name_to_copy, &dirent_path_prev, - effective_kind, + current.kind, args.encoding, true, ); @@ -8381,6 +8341,7 @@ impl NodeFS { let mut iterator = DirIterator::WrappedIteratorW::init(fd); #[cfg(not(windows))] let mut iterator = DirIterator::WrappedIterator::init(fd); + iterator.resolve_unknown_entry_types = true; loop { let current = match iterator.next() { diff --git a/src/runtime/node/path_watcher.rs b/src/runtime/node/path_watcher.rs index 715150032bac..edf88f37021f 100644 --- a/src/runtime/node/path_watcher.rs +++ b/src/runtime/node/path_watcher.rs @@ -605,6 +605,7 @@ fn walk_subtree( }; let _close = sys::CloseOnDrop::new(dfd); let mut it = sys::dir_iterator::iterate(dfd); + it.resolve_unknown_entry_types = true; let mut abs_buf = path::path_buffer_pool::get(); let mut rel_buf = path::path_buffer_pool::get(); loop { diff --git a/src/runtime/shell/builtin/ls.rs b/src/runtime/shell/builtin/ls.rs index 8f6213a5dcad..e428c2dfa457 100644 --- a/src/runtime/shell/builtin/ls.rs +++ b/src/runtime/shell/builtin/ls.rs @@ -492,6 +492,8 @@ impl ShellLsTask { } let mut iterator = dir_iterator::iterate(fd); + // The kind is only looked at to decide what to recurse into. + iterator.resolve_unknown_entry_types = this.opts.recursive; // If `-a` is used, "." and ".." should show up as results. However, // our `DirIterator` abstraction skips them, so add them now. diff --git a/src/runtime/webview/ChromeProcess.rs b/src/runtime/webview/ChromeProcess.rs index 3b9235f9531a..5200bfb928a6 100644 --- a/src/runtime/webview/ChromeProcess.rs +++ b/src/runtime/webview/ChromeProcess.rs @@ -315,6 +315,7 @@ fn find_playwright_shell() -> Option { const PREFIX: &[u8] = b"chromium_headless_shell-"; let mut iter = bun_sys::iterate_dir(fd); + iter.resolve_unknown_entry_types = true; loop { let entry = match iter.next() { Ok(Some(e)) => e, diff --git a/src/sys/lib.rs b/src/sys/lib.rs index e563f9bb54b8..73ca151e423f 100644 --- a/src/sys/lib.rs +++ b/src/sys/lib.rs @@ -122,6 +122,25 @@ pub mod dir_iterator { pub kind: EntryKind, } + impl IteratorResult { + /// See `WrappedIterator::resolve_unknown_entry_types`. + fn resolve_unknown_kind(&mut self, dir: Fd) { + #[cfg(not(windows))] + { + if self.kind == EntryKind::Unknown { + if let Ok(stat) = super::lstatat(dir, self.name.as_zstr()) { + self.kind = super::kind_from_mode(stat.st_mode as super::Mode); + } + } + } + #[cfg(windows)] + { + // The Windows iterator always knows the kind. + let _ = dir; + } + } + } + /// Length-known, NUL-terminated entry name in OS-native encoding. /// /// **POSIX**: lifetime-erased borrow (raw pointer + length) into the @@ -265,8 +284,7 @@ pub mod dir_iterator { // literal matches . #[cfg(any(target_os = "macos", target_os = "freebsd"))] 14 /* DT_WHT */ => EntryKind::Whiteout, - // DT_UNKNOWN: some filesystems (bind mounts, FUSE, NFS) don't - // provide d_type. Callers should lstatat() to resolve when needed. + // DT_UNKNOWN: see `WrappedIterator::resolve_unknown_entry_types`. _ => EntryKind::Unknown, } } @@ -731,6 +749,10 @@ pub mod dir_iterator { #[cfg(not(windows))] name_filter: Option>, state: State, + /// `lstat` entries whose kind the filesystem did not report (`Unknown`: + /// FUSE, NFS, XFS with `ftype=0`), so that, as with `d_type`, a symlink is + /// still a symlink. Entries that cannot be stat'ed stay `Unknown`. + pub resolve_unknown_entry_types: bool, } impl WrappedIterator { #[inline] @@ -759,7 +781,13 @@ pub mod dir_iterator { /// Copy it out before pushing the iterator into a `Vec` etc. #[inline] pub fn next(&mut self) -> Result> { - self.state.next(self.dir) + let mut entry = self.state.next(self.dir)?; + if self.resolve_unknown_entry_types { + if let Some(entry) = entry.as_mut() { + entry.resolve_unknown_kind(self.dir); + } + } + Ok(entry) } } @@ -770,6 +798,7 @@ pub mod dir_iterator { dir, name_filter: None, state: State::new(), + resolve_unknown_entry_types: false, } } #[cfg(windows)] @@ -777,6 +806,7 @@ pub mod dir_iterator { WrappedIterator { dir, state: State::new(), + resolve_unknown_entry_types: false, } } } diff --git a/src/sys/walker_skippable.rs b/src/sys/walker_skippable.rs index d4d7d854b72f..44993ddc7093 100644 --- a/src/sys/walker_skippable.rs +++ b/src/sys/walker_skippable.rs @@ -26,7 +26,6 @@ pub struct Walker { skip_dirnames: Range, skip_all: Box<[u64]>, seed: u64, - pub resolve_unknown_entry_types: bool, } /// The directory a walk starts from. The walker never closes a borrowed @@ -53,6 +52,7 @@ pub struct WalkerEntry<'a> { pub dir: Fd, pub basename: &'a OSPathSliceZ, pub path: &'a OSPathSliceZ, + /// Resolved with `lstat` when readdir did not report it; `Unknown` only if that failed too. pub kind: sys::EntryKind, } @@ -61,6 +61,13 @@ struct StackItem { dirname_len: usize, } +/// Every consumer branches on the kind, and the walk itself descends by it. +fn iterate(dir: Fd) -> WrappedIterator { + let mut iter = dir_iterator::iterate(dir); + iter.resolve_unknown_entry_types = true; + iter +} + impl Walker { /// The directory the walk started from; open for as long as the walker is. pub fn root(&self) -> Fd { @@ -76,127 +83,96 @@ impl Walker { // be invalidated by appending to `self.stack` below. let top_idx = self.stack.len() - 1; let mut dirname_len = self.stack[top_idx].dirname_len; - match self.stack[top_idx].iter.next() { - Err(err) => return Err(err), - Ok(res) => { - if let Some(base) = res { - // Some filesystems (NFS, FUSE, bind mounts) don't provide - // d_type and return DT_UNKNOWN. Optionally resolve via - // fstatat so callers get accurate types for recursion. - // This only affects POSIX; Windows always provides types. - #[cfg(not(windows))] - let kind: sys::EntryKind = if base.kind == sys::EntryKind::Unknown - && self.resolve_unknown_entry_types - { - let dir_fd = self.stack[top_idx].iter.dir(); - match sys::lstatat(dir_fd, base.name.as_zstr()) { - Ok(stat_buf) => sys::kind_from_mode(stat_buf.st_mode as sys::Mode), - Err(_) => continue, // skip entries we can't stat - } + let Some(base) = self.stack[top_idx].iter.next()? else { + let item = self.stack.pop().unwrap(); + if !self.stack.is_empty() { + item.iter.dir().close(); + } + continue; + }; + let kind = base.kind; + + match kind { + sys::EntryKind::Directory => { + let skip = &self.skip_all[self.skip_dirnames.clone()]; + if skip.contains( + // avoid hashing if there will be 0 results + &(if !skip.is_empty() { + hash_with_seed(self.seed, slice_as_bytes(base.name.as_slice())) } else { - base.kind - }; - #[cfg(windows)] - let kind: sys::EntryKind = base.kind; - - match kind { - sys::EntryKind::Directory => { - let skip = &self.skip_all[self.skip_dirnames.clone()]; - if skip.contains( - // avoid hashing if there will be 0 results - &(if !skip.is_empty() { - hash_with_seed( - self.seed, - slice_as_bytes(base.name.as_slice()), - ) - } else { - 0 - }), - ) { - continue; - } - } - sys::EntryKind::File => { - let skip = &self.skip_all[self.skip_filenames.clone()]; - if skip.contains( - // avoid hashing if there will be 0 results - &(if !skip.is_empty() { - hash_with_seed( - self.seed, - slice_as_bytes(base.name.as_slice()), - ) - } else { - 0 - }), - ) { - continue; - } - } - - // we don't know what it is for a symlink - sys::EntryKind::SymLink => { - let skip = &self.skip_all[..]; - if skip.contains( - // avoid hashing if there will be 0 results - &(if !skip.is_empty() { - hash_with_seed( - self.seed, - slice_as_bytes(base.name.as_slice()), - ) - } else { - 0 - }), - ) { - continue; - } - } + 0 + }), + ) { + continue; + } + } + sys::EntryKind::File => { + let skip = &self.skip_all[self.skip_filenames.clone()]; + if skip.contains( + // avoid hashing if there will be 0 results + &(if !skip.is_empty() { + hash_with_seed(self.seed, slice_as_bytes(base.name.as_slice())) + } else { + 0 + }), + ) { + continue; + } + } - _ => {} - } + // we don't know what it is for a symlink + sys::EntryKind::SymLink => { + let skip = &self.skip_all[..]; + if skip.contains( + // avoid hashing if there will be 0 results + &(if !skip.is_empty() { + hash_with_seed(self.seed, slice_as_bytes(base.name.as_slice())) + } else { + 0 + }), + ) { + continue; + } + } - self.name_buffer.truncate(dirname_len); - if !self.name_buffer.is_empty() { - self.name_buffer.push(SEP as OSPathChar); - dirname_len += 1; - } - self.name_buffer.extend_from_slice(base.name.as_slice()); - let cur_len = self.name_buffer.len(); - self.name_buffer.push(0); + _ => {} + } - let mut top_idx = top_idx; - if kind == sys::EntryKind::Directory { - let new_dir = sys::open_dir_for_iteration_os_path( - self.stack[top_idx].iter.dir(), - base.name.as_slice(), - )?; - { - self.stack.push(StackItem { - iter: dir_iterator::iterate(new_dir), - dirname_len: cur_len, - }); - top_idx = self.stack.len() - 1; - } - } - // `name_buffer[cur_len] == 0` was written above; both views end at - // `cur_len` and are NUL-terminated by that sentinel char. `from_buf` - // ties the borrow to `&self.name_buffer` (no raw-pointer reslice). - return Ok(Some(WalkerEntry { - dir: self.stack[top_idx].iter.dir(), - basename: OSPathSliceZ::from_buf( - &self.name_buffer[dirname_len..], - cur_len - dirname_len, - ), - path: OSPathSliceZ::from_buf(&self.name_buffer, cur_len), - kind, - })); - } else { - let item = self.stack.pop().unwrap(); - if !self.stack.is_empty() { - item.iter.dir().close(); - } - } + self.name_buffer.truncate(dirname_len); + if !self.name_buffer.is_empty() { + self.name_buffer.push(SEP as OSPathChar); + dirname_len += 1; + } + self.name_buffer.extend_from_slice(base.name.as_slice()); + let cur_len = self.name_buffer.len(); + self.name_buffer.push(0); + + let mut top_idx = top_idx; + if kind == sys::EntryKind::Directory { + let new_dir = sys::open_dir_for_iteration_os_path( + self.stack[top_idx].iter.dir(), + base.name.as_slice(), + )?; + { + self.stack.push(StackItem { + iter: iterate(new_dir), + dirname_len: cur_len, + }); + top_idx = self.stack.len() - 1; } } + // `name_buffer[cur_len] == 0` was written above; both views end at + // `cur_len` and are NUL-terminated by that sentinel char. `from_buf` + // ties the borrow to `&self.name_buffer` (no raw-pointer reslice). + return Ok(Some(WalkerEntry { + dir: self.stack[top_idx].iter.dir(), + basename: OSPathSliceZ::from_buf( + &self.name_buffer[dirname_len..], + cur_len - dirname_len, + ), + path: OSPathSliceZ::from_buf(&self.name_buffer, cur_len), + kind, + })); } Ok(None) } @@ -263,7 +239,7 @@ fn walk_root( } stack.push(StackItem { - iter: dir_iterator::iterate(root.fd()), + iter: iterate(root.fd()), dirname_len: 0, }); @@ -275,6 +251,5 @@ fn walk_root( seed, skip_filenames: skip_filenames_, skip_dirnames: skip_dirnames_, - resolve_unknown_entry_types: false, }) } diff --git a/test/cli/dt-unknown-readdir.test.ts b/test/cli/dt-unknown-readdir.test.ts new file mode 100644 index 000000000000..3f3e5b2494be --- /dev/null +++ b/test/cli/dt-unknown-readdir.test.ts @@ -0,0 +1,185 @@ +// Commands that branch on the kind of a readdir entry (file, directory, symlink) +// must still work on filesystems whose readdir reports every entry as +// DT_UNKNOWN (FUSE such as sshfs, some NFS servers, XFS with ftype=0), which +// means resolving the kind with lstat instead of skipping the entry. Each test +// runs one command under `dtUnknownReaddir` (harness), which simulates such a +// filesystem with an LD_PRELOAD shim. `bun pm pack`, `bun publish` and the +// package copy done by `bun install` are covered in install/dt-unknown-readdir.test.ts. +import { beforeAll, describe, expect, test } from "bun:test"; +import { bunEnv, bunExe, dtUnknownReaddir, tempDir } from "harness"; +import { chmodSync, existsSync, readdirSync } from "node:fs"; +import { join } from "node:path"; + +let shimEnv: NodeJS.Dict; + +beforeAll(async () => { + if (dtUnknownReaddir.available) shimEnv = await dtUnknownReaddir.env(); +}, 30_000); + +async function run( + cwd: string, + args: string[], + { shim = true, env = {} as Record } = {}, +): Promise<{ stdout: string; stderr: string; exitCode: number }> { + await using proc = Bun.spawn({ + cmd: [bunExe(), ...args], + cwd, + env: { ...(shim ? shimEnv : bunEnv), ...env }, + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + if (shim) expect(stderr).toContain(dtUnknownReaddir.marker); + return { stdout, stderr, exitCode }; +} + +function filesUnder(dir: string): string[] { + return readdirSync(dir, { recursive: true, withFileTypes: true }) + .filter(entry => !entry.isDirectory()) + .map(entry => join(entry.parentPath, entry.name).slice(dir.length + 1)) + .sort(); +} + +function binsIn(binDir: string): string[] { + // Not `existsSync(join(binDir, name))`: it follows symlinks, so it cannot see a dangling one. + return existsSync(binDir) ? readdirSync(binDir).sort() : []; +} + +// A package whose bins come from `directories.bin` (a directory to link every +// file of) rather than from `bin`. `nested/` is a directory, not a bin. +const directoriesBinPackage = (name: string) => ({ + [`${name}/package.json`]: JSON.stringify({ name, version: "1.0.0", directories: { bin: "bins" } }), + [`${name}/bins/${name}-a`]: "#!/bin/sh\necho ran-a\n", + [`${name}/bins/${name}-b`]: "#!/bin/sh\necho ran-b\n", + [`${name}/bins/nested/not-a-bin`]: "", +}); + +const globalDirEnv = (globalDir: string) => ({ + BUN_INSTALL: globalDir, + BUN_INSTALL_GLOBAL_DIR: join(globalDir, "install", "global"), + BUN_INSTALL_BIN: join(globalDir, "bin"), +}); + +describe.skipIf(!dtUnknownReaddir.available)("on a filesystem whose readdir reports DT_UNKNOWN", () => { + test.concurrent("bun install links the bins of a directories.bin package", async () => { + using dir = tempDir("dt-unknown-install", { + "package.json": JSON.stringify({ name: "app", dependencies: { dep: "file:./dep" } }), + ...directoriesBinPackage("dep"), + }); + + const { stderr, exitCode } = await run(String(dir), ["install"], { + env: { BUN_INSTALL_CACHE_DIR: join(String(dir), ".cache") }, + }); + + expect(stderr).not.toContain("error:"); + expect(exitCode).toBe(0); + expect(binsIn(join(String(dir), "node_modules", ".bin"))).toEqual(["dep-a", "dep-b"]); + }); + + test.concurrent("bunx finds the executable of a directories.bin package", async () => { + using dir = tempDir("dt-unknown-bunx", { + "package.json": JSON.stringify({ name: "app", dependencies: { tool: "file:./tool" } }), + "tool/package.json": JSON.stringify({ name: "tool", version: "1.0.0", directories: { bin: "bins" } }), + "tool/bins/tool-cli": "#!/bin/sh\necho tool-cli ran\n", + "tool/bins/nested/not-a-bin": "", + }); + chmodSync(join(String(dir), "tool", "bins", "tool-cli"), 0o755); + const env = { BUN_INSTALL_CACHE_DIR: join(String(dir), ".cache") }; + expect(await run(String(dir), ["install"], { shim: false, env })).toMatchObject({ exitCode: 0 }); + + // The bin is not named after the package, so bunx has to read the package's + // `directories.bin` to learn its name. --no-install: failing to do so must + // not fall through to installing the package from the registry. + expect(await run(String(dir), ["x", "--no-install", "tool"], { env })).toEqual({ + stdout: "tool-cli ran\n", + stderr: `${dtUnknownReaddir.marker}\n`, + exitCode: 0, + }); + }); + + test.concurrent("bun link links the bins of a directories.bin package", async () => { + using dir = tempDir("dt-unknown-link", directoriesBinPackage("linked")); + const globalDir = join(String(dir), "global"); + + const { stderr, exitCode } = await run(join(String(dir), "linked"), ["link"], { env: globalDirEnv(globalDir) }); + + expect(stderr).not.toContain("error:"); + expect(exitCode).toBe(0); + expect(binsIn(join(globalDir, "bin"))).toEqual(["linked-a", "linked-b"]); + }); + + test.concurrent("bun unlink removes the bins of a directories.bin package", async () => { + using dir = tempDir("dt-unknown-unlink", directoriesBinPackage("linked")); + const globalDir = join(String(dir), "global"); + const env = globalDirEnv(globalDir); + expect(await run(join(String(dir), "linked"), ["link"], { shim: false, env })).toMatchObject({ exitCode: 0 }); + expect(binsIn(join(globalDir, "bin"))).toEqual(["linked-a", "linked-b"]); + + const { stderr, exitCode } = await run(join(String(dir), "linked"), ["unlink"], { env }); + + expect(stderr).not.toContain("error:"); + expect(exitCode).toBe(0); + expect(binsIn(join(globalDir, "bin"))).toEqual([]); + }); + + test.concurrent("bun remove deletes the removed package's dangling .bin symlinks", async () => { + using dir = tempDir("dt-unknown-remove", { + "package.json": JSON.stringify({ name: "app", dependencies: { dep: "file:./dep" } }), + "dep/package.json": JSON.stringify({ name: "dep", version: "1.0.0", bin: { "dep-cli": "cli.js" } }), + "dep/cli.js": "", + }); + const env = { BUN_INSTALL_CACHE_DIR: join(String(dir), ".cache") }; + const binDir = join(String(dir), "node_modules", ".bin"); + expect(await run(String(dir), ["install"], { shim: false, env })).toMatchObject({ exitCode: 0 }); + expect(binsIn(binDir)).toEqual(["dep-cli"]); + + const { stderr, exitCode } = await run(String(dir), ["remove", "dep"], { env }); + + expect(stderr).not.toContain("error:"); + expect(exitCode).toBe(0); + expect(binsIn(binDir)).toEqual([]); + }); + + test.concurrent("bun create copies a local template", async () => { + using dir = tempDir("dt-unknown-create", { + "templates/tmpl/package.json": JSON.stringify({ name: "tmpl", version: "1.0.0" }), + "templates/tmpl/index.js": "", + "templates/tmpl/src/lib.js": "", + "templates/tmpl/src/deep/x.js": "", + // Skipped by name, which takes knowing that one is a directory and the other a file. + "templates/tmpl/node_modules/left-behind.js": "", + "templates/tmpl/yarn.lock": "", + }); + + const { stderr, exitCode } = await run(String(dir), ["create", "tmpl", "out", "--no-git", "--no-install"], { + env: { BUN_CREATE_DIR: join(String(dir), "templates") }, + }); + + expect(stderr).not.toContain("error:"); + expect(exitCode).toBe(0); + expect(filesUnder(join(String(dir), "out"))).toEqual(["index.js", "package.json", "src/deep/x.js", "src/lib.js"]); + }); + + test.concurrent("bun init does not add an entry point next to existing source files", async () => { + using dir = tempDir("dt-unknown-init", { "app.ts": "export {};\n" }); + // `bun init` ends by running `bun install`; point it at a registry that + // answers nothing so the test stays offline. init still exits 0 (the + // install's 404s go to stderr), and package.json is written before it runs. + using registry = Bun.serve({ port: 0, fetch: () => new Response(null, { status: 404 }) }); + + const { stdout, exitCode } = await run(String(dir), ["init", "-y"], { + env: { + NPM_CONFIG_REGISTRY: `http://localhost:${registry.port}/`, + BUN_INSTALL_CACHE_DIR: join(String(dir), ".cache"), + BUN_AGENT_RULE_DISABLED: "1", + }, + }); + + expect(stdout).not.toContain("index.ts"); + expect(exitCode).toBe(0); + const pkg = await Bun.file(join(String(dir), "package.json")).json(); + expect(pkg).not.toHaveProperty("module"); + expect(existsSync(join(String(dir), "index.ts"))).toBe(false); + }); +}); diff --git a/test/cli/install/bunx-directories-bin.test.ts b/test/cli/install/bunx-directories-bin.test.ts new file mode 100644 index 000000000000..fd2702b6d456 --- /dev/null +++ b/test/cli/install/bunx-directories-bin.test.ts @@ -0,0 +1,75 @@ +import { expect, test } from "bun:test"; +import { bunEnv, bunExe, isWindows, tempDir } from "harness"; +import { chmodSync, mkdirSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; + +async function run(cwd: string, args: string[], env: Record) { + await using proc = Bun.spawn({ + cmd: [bunExe(), ...args], + cwd, + env: { ...bunEnv, ...env }, + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + return { stdout, stderr, exitCode }; +} + +// The bins are shell scripts, which the bin links cannot run on Windows. +test.concurrent.skipIf(isWindows)("bunx runs the bin of a directories.bin package", async () => { + using dir = tempDir("bunx-directories-bin", { + "package.json": JSON.stringify({ name: "app", dependencies: { tool: "file:./tool" } }), + "tool/package.json": JSON.stringify({ name: "tool", version: "1.0.0", directories: { bin: "bins" } }), + "tool/bins/tool-cli": "#!/bin/sh\necho tool-cli ran\n", + "tool/bins/nested/not-a-bin": "", + }); + chmodSync(join(String(dir), "tool", "bins", "tool-cli"), 0o755); + const env = { BUN_INSTALL_CACHE_DIR: join(String(dir), ".cache") }; + expect(await run(String(dir), ["install"], env)).toMatchObject({ exitCode: 0 }); + + // The bin is not named after the package, so bunx has to read the package's + // `directories.bin` (relative to the package, not to the project) to learn + // its name. --no-install: failing to do so must not fall through to + // installing the package from the registry. + expect(await run(String(dir), ["x", "--no-install", "tool"], env)).toEqual({ + stdout: "tool-cli ran\n", + stderr: "", + exitCode: 0, + }); +}); + +// `bun install` links nothing for these values; bunx must not take a bin name +// from the directory they point at either. `picked` is the entry bunx would +// find there (and then run from node_modules/.bin) if it did. +const rejected: [label: string, value: (dir: string) => string, picked: string][] = [ + ["a relative path that leaves the package", () => "../../outside", "planted"], + ["an absolute path", dir => join(dir, "outside"), "planted"], + ["an empty string (the package directory itself)", () => "", "package.json"], +]; + +for (const [label, value, picked] of rejected) { + test.concurrent.skipIf(isWindows)(`bunx ignores a directories.bin that is ${label}`, async () => { + using dir = tempDir("bunx-directories-bin-rejected", { + "package.json": JSON.stringify({ name: "app", dependencies: { tool: "file:./tool" } }), + "tool/package.json": "", + "outside/planted": "", + }); + // Written afterwards because the value may depend on the directory's path. + writeFileSync( + join(String(dir), "tool", "package.json"), + JSON.stringify({ name: "tool", version: "1.0.0", directories: { bin: value(String(dir)) } }), + ); + const env = { BUN_INSTALL_CACHE_DIR: join(String(dir), ".cache") }; + expect(await run(String(dir), ["install"], env)).toMatchObject({ exitCode: 0 }); + const binDir = join(String(dir), "node_modules", ".bin"); + mkdirSync(binDir, { recursive: true }); + writeFileSync(join(binDir, picked), "#!/bin/sh\necho planted ran\n", { mode: 0o755 }); + + const { stdout, stderr, exitCode } = await run(String(dir), ["x", "--no-install", "tool"], env); + + expect(stdout).toBe(""); + expect(stderr).toContain("could not determine executable to run for package tool"); + expect(exitCode).toBe(1); + }); +} diff --git a/test/cli/install/dt-unknown-readdir.test.ts b/test/cli/install/dt-unknown-readdir.test.ts new file mode 100644 index 000000000000..fa39bb3b8ea3 --- /dev/null +++ b/test/cli/install/dt-unknown-readdir.test.ts @@ -0,0 +1,177 @@ +// Some filesystems (FUSE, NFS, XFS formatted with ftype=0) do not fill in +// d_type, so every readdir entry comes back as DT_UNKNOWN. The package manager +// commands must behave as they do elsewhere; `dtUnknownReaddir` (harness) +// simulates such a filesystem with an LD_PRELOAD shim. +import { readTarball } from "bun:internal-for-testing"; +import { beforeAll, describe, expect, test } from "bun:test"; +import { bunExe, dtUnknownReaddir, tempDir } from "harness"; +import { readdir, symlink, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +let env: NodeJS.Dict; + +beforeAll(async () => { + if (dtUnknownReaddir.available) env = await dtUnknownReaddir.env(); +}, 30_000); + +async function run(cwd: string, ...args: string[]) { + await using proc = Bun.spawn({ + cmd: [bunExe(), ...args], + cwd, + env, + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + expect(stderr).toContain(dtUnknownReaddir.marker); + expect({ stdout, stderr, exitCode }).toMatchObject({ stderr: expect.not.stringContaining("error:"), exitCode: 0 }); +} + +function packedPaths(tarball: string): string[] { + return readTarball(tarball) + .entries.map((entry: { pathname: string }) => entry.pathname) + .sort(); +} + +describe.skipIf(!dtUnknownReaddir.available)("pack on a filesystem whose readdir reports DT_UNKNOWN", () => { + test.concurrent("packs the project tree", async () => { + using dir = tempDir("dt-unknown-tree", { + "package.json": JSON.stringify({ name: "dt-unknown-tree", version: "1.0.0" }), + "index.js": "", + "lib/a.js": "", + "lib/nested/b.js": "", + // `out/` only ignores directories, so it needs the entry's kind: the + // `out` directory is ignored, the `lib/out` file is not. + ".npmignore": "out/\n", + "out/c.js": "", + "lib/out": "", + }); + // Symlinks are never packed; resolving the kind with lstat has to keep that. + await symlink("index.js", join(String(dir), "link.js")); + + await run(String(dir), "pm", "pack"); + + expect(packedPaths(join(String(dir), "dt-unknown-tree-1.0.0.tgz"))).toEqual([ + "package/index.js", + "package/lib/a.js", + "package/lib/nested/b.js", + "package/lib/out", + "package/package.json", + ]); + }); + + test.concurrent('packs what "files" selects', async () => { + using dir = tempDir("dt-unknown-files", { + "package.json": JSON.stringify({ + name: "dt-unknown-files", + version: "1.0.0", + files: ["index.js", "lib", "!lib/internal/"], + }), + "index.js": "", + "excluded.js": "", + "lib/a.js": "", + "lib/nested/b.js": "", + "lib/internal/c.js": "", + }); + await symlink("a.js", join(String(dir), "lib", "link.js")); + + await run(String(dir), "pm", "pack"); + + expect(packedPaths(join(String(dir), "dt-unknown-files-1.0.0.tgz"))).toEqual([ + "package/index.js", + "package/lib/a.js", + "package/lib/nested/b.js", + "package/package.json", + ]); + }); + + test.concurrent("packs bundledDependencies", async () => { + using dir = tempDir("dt-unknown-bundled", { + "package.json": JSON.stringify({ + name: "dt-unknown-bundled", + version: "1.0.0", + dependencies: { "dep": "1.0.0", "@scope/dep": "1.0.0", "not-bundled": "1.0.0" }, + bundledDependencies: ["dep", "@scope/dep"], + }), + "index.js": "", + "node_modules/dep/package.json": JSON.stringify({ name: "dep", version: "1.0.0" }), + "node_modules/dep/lib/index.js": "", + "node_modules/@scope/dep/package.json": JSON.stringify({ name: "@scope/dep", version: "1.0.0" }), + "node_modules/@scope/dep/index.js": "", + "node_modules/not-bundled/package.json": JSON.stringify({ name: "not-bundled", version: "1.0.0" }), + }); + + await run(String(dir), "pm", "pack"); + + expect(packedPaths(join(String(dir), "dt-unknown-bundled-1.0.0.tgz"))).toEqual([ + "package/index.js", + "package/node_modules/@scope/dep/index.js", + "package/node_modules/@scope/dep/package.json", + "package/node_modules/dep/lib/index.js", + "package/node_modules/dep/package.json", + "package/package.json", + ]); + }); + + test.concurrent('publish packs the tree, walks "directories.bin" and finds the readme', async () => { + let captured: any; + using registry = Bun.serve({ + port: 0, + async fetch(req) { + if (req.method === "PUT") captured = await req.json(); + return new Response("OK"); + }, + }); + using dir = tempDir("dt-unknown-publish", { + "bunfig.toml": `[install]\ncache = false\nregistry = { url = "http://localhost:${registry.port}", token = "unused" }\n`, + "package.json": JSON.stringify({ + name: "dt-unknown-publish", + version: "1.0.0", + directories: { bin: "bins" }, + }), + "README.md": "# dt-unknown-publish", + "index.js": "", + "bins/a.js": "", + "bins/more/b.js": "", + }); + + await run(String(dir), "publish"); + + expect(captured.versions["1.0.0"]).toMatchObject({ + bin: { "a.js": "bins/a.js", "more": "bins/more", "b.js": "bins/more/b.js" }, + readme: "# dt-unknown-publish", + readmeFilename: "README.md", + }); + + const attachment: { data: string } = Object.values(captured._attachments)[0] as any; + const tarball = join(String(dir), "published.tgz"); + await writeFile(tarball, Buffer.from(attachment.data, "base64")); + expect(packedPaths(tarball)).toEqual([ + "package/README.md", + "package/bins/a.js", + "package/bins/more/b.js", + "package/index.js", + "package/package.json", + ]); + }); +}); + +describe.skipIf(!dtUnknownReaddir.available)("install on a filesystem whose readdir reports DT_UNKNOWN", () => { + // Folder dependencies are installed by walking the folder (walker_skippable + // with resolve_unknown_entry_types), so subdirectories must still be entered. + test.concurrent("installs every file of a folder dependency", async () => { + using dir = tempDir("dt-unknown-install", { + "dep/package.json": JSON.stringify({ name: "dep", version: "1.0.0" }), + "dep/index.js": "", + "dep/lib/a.js": "", + "dep/lib/nested/b.js": "", + "app/package.json": JSON.stringify({ name: "app", dependencies: { dep: "file:../dep" } }), + }); + + await run(join(String(dir), "app"), "install", "--no-summary"); + + const installed = await readdir(join(String(dir), "app", "node_modules", "dep"), { recursive: true }); + expect(installed.sort()).toEqual(["index.js", "lib", "lib/a.js", "lib/nested", "lib/nested/b.js", "package.json"]); + }); +}); diff --git a/test/fixtures/dt-unknown-readdir-shim.c b/test/fixtures/dt-unknown-readdir-shim.c new file mode 100644 index 000000000000..e6a435a315e0 --- /dev/null +++ b/test/fixtures/dt-unknown-readdir-shim.c @@ -0,0 +1,50 @@ +// LD_PRELOAD shim: every getdents64 record comes back with d_type == DT_UNKNOWN, +// the way FUSE, some NFS servers and XFS formatted with ftype=0 report entries. +// bun issues getdents64 through libc's syscall() wrapper, which this interposes. +// Compiled by `dtUnknownReaddir` in test/harness.ts, which defines MARKER: it is +// written to stderr the first time a record is rewritten so a test can tell the +// shim actually saw bun's readdir calls. +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include + +static long (*real_syscall)(long, long, long, long, long, long, long); +static int announced; + +long syscall(long number, ...) { + va_list ap; + long a, b, c, d, e, f; + va_start(ap, number); + a = va_arg(ap, long); + b = va_arg(ap, long); + c = va_arg(ap, long); + d = va_arg(ap, long); + e = va_arg(ap, long); + f = va_arg(ap, long); + va_end(ap); + if (!real_syscall) { + real_syscall = (long (*)(long, long, long, long, long, long, long))dlsym(RTLD_NEXT, "syscall"); + } + long rc = real_syscall(number, a, b, c, d, e, f); + if (number != SYS_getdents64 || rc <= 0) return rc; + if (!announced) { + announced = 1; + static const char marker[] = MARKER "\n"; + if (write(2, marker, sizeof(marker) - 1) < 0) { + } + } + // struct linux_dirent64 { u64 d_ino; s64 d_off; u16 d_reclen; u8 d_type; char d_name[]; } + unsigned char *buf = (unsigned char *)b; + for (long off = 0; off + 19 <= rc;) { + uint16_t reclen; + memcpy(&reclen, buf + off + 16, sizeof(reclen)); + if (reclen == 0) break; + buf[off + 18] = 0; /* DT_UNKNOWN */ + off += reclen; + } + return rc; +} diff --git a/test/harness.ts b/test/harness.ts index b3167919ec84..c2034a71319c 100644 --- a/test/harness.ts +++ b/test/harness.ts @@ -2275,6 +2275,51 @@ export function compileFixture(sourcePath: string, options: { flags?: string[] } return outPath; } +const dtUnknownReaddirMarker = "dt-unknown-readdir-shim: rewrote getdents64 d_type"; +let dtUnknownReaddirShim: Promise | undefined; + +async function compileDtUnknownReaddirShim(): Promise { + const cc = which("cc") || which("clang") || which("gcc"); + if (!cc) throw new Error("dtUnknownReaddir: no C compiler (cc/clang/gcc) found in $PATH"); + const shim = join(tmpdirSync("dt-unknown-readdir-"), "shim.so"); + const source = join(import.meta.dir, "fixtures", "dt-unknown-readdir-shim.c"); + const proc = Bun.spawn({ + cmd: [cc, "-shared", "-fPIC", "-O2", `-DMARKER="${dtUnknownReaddirMarker}"`, "-o", shim, source, "-ldl"], + env: bunEnv, + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + if (exitCode !== 0) + throw new Error(`dtUnknownReaddir: compiling the shim failed (exit ${exitCode}):\n${stderr || stdout}`); + return shim; +} + +/** + * Runs bun as if on a filesystem whose readdir does not report entry types + * (FUSE, some NFS servers, XFS formatted with `ftype=0`), without needing such a + * mount: `env()` preloads a shim that zeroes `d_type` in every `getdents64` + * record. The shim prints `marker` to stderr the first time it does so; assert + * on it, otherwise a test here passes vacuously if bun ever stops issuing + * `getdents64` through libc's `syscall()` wrapper, which is what the shim hooks. + */ +export const dtUnknownReaddir = { + /** Linux with a C compiler; `skipIf(!dtUnknownReaddir.available)`. */ + get available(): boolean { + return isLinux && !!(which("cc") || which("clang") || which("gcc")); + }, + marker: dtUnknownReaddirMarker, + /** + * Compiles the shim the first time it is called. Call it from `beforeAll` + * (the compiler can take several seconds on a loaded machine) and spawn bun + * with the returned env. + */ + async env(): Promise> { + const shim = await (dtUnknownReaddirShim ??= compileDtUnknownReaddirShim()); + return { ...bunEnv, LD_PRELOAD: bunEnv.LD_PRELOAD ? `${shim}:${bunEnv.LD_PRELOAD}` : shim }; + }, +}; + export const rss: () => number = process.platform === "darwin" && typeof Bun.unsafe.memoryFootprint === "function" ? (Bun.unsafe.memoryFootprint as () => number) diff --git a/test/js/bun/shell/shell-dt-unknown-readdir.test.ts b/test/js/bun/shell/shell-dt-unknown-readdir.test.ts new file mode 100644 index 000000000000..ff82dc3b59f6 --- /dev/null +++ b/test/js/bun/shell/shell-dt-unknown-readdir.test.ts @@ -0,0 +1,76 @@ +// The shell's `ls -R` and `cp -R` builtins decide what to recurse into from the +// kind readdir reports for each entry. On filesystems whose readdir reports +// every entry as DT_UNKNOWN (FUSE, some NFS servers, XFS with ftype=0) they have +// to lstat the entries instead: `ls -R` used to list only the top level and +// `cp -R` used to fail with ENOTSUP when it tried to copy a subdirectory as a +// file. Each test runs a fixture under `dtUnknownReaddir` (harness), which +// simulates such a filesystem with an LD_PRELOAD shim. +import { beforeAll, describe, expect, test } from "bun:test"; +import { bunExe, dtUnknownReaddir, tempDir } from "harness"; + +let shimEnv: NodeJS.Dict; + +beforeAll(async () => { + if (dtUnknownReaddir.available) shimEnv = await dtUnknownReaddir.env(); +}, 30_000); + +const TREE = { + "src/a.txt": "a", + "src/sub/b.txt": "b", +}; + +const LS_FIXTURE = /* js */ ` +import { $ } from "bun"; +const { stdout, stderr, exitCode } = await $\`ls -R src\`.quiet().nothrow(); +console.log(JSON.stringify({ lines: stdout.toString().split("\\n").filter(Boolean).sort(), stderr: stderr.toString(), exitCode })); +`; + +const CP_FIXTURE = /* js */ ` +import { $ } from "bun"; +import { readdirSync } from "node:fs"; +const { stderr, exitCode } = await $\`cp -R src dest\`.quiet().nothrow(); +let copied = []; +try { copied = readdirSync("dest", { recursive: true }).sort(); } catch {} +console.log(JSON.stringify({ copied, stderr: stderr.toString(), exitCode })); +`; + +async function runFixture(dir: string, extraEnv: Record = {}) { + await using proc = Bun.spawn({ + cmd: [bunExe(), "fixture.mjs"], + cwd: dir, + env: { ...shimEnv, ...extraEnv }, + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + let result: unknown = stdout; + try { + result = JSON.parse(stdout); + } catch {} + return { result, stderr, exitCode }; +} + +describe.skipIf(!dtUnknownReaddir.available)("shell builtins on a filesystem whose readdir reports DT_UNKNOWN", () => { + test.concurrent("ls -R recurses into subdirectories", async () => { + using dir = tempDir("shell-ls-dt-unknown", { ...TREE, "fixture.mjs": LS_FIXTURE }); + + expect(await runFixture(String(dir))).toEqual({ + result: { lines: ["a.txt", "b.txt", "src/sub:", "sub"], stderr: "", exitCode: 0 }, + stderr: `${dtUnknownReaddir.marker}\n`, + exitCode: 0, + }); + }); + + test.concurrent("cp -R copies subdirectories", async () => { + using dir = tempDir("shell-cp-dt-unknown", { ...TREE, "fixture.mjs": CP_FIXTURE }); + + // The cp builtin is only enabled on POSIX behind this flag; without it the + // shell would spawn the system cp. + expect(await runFixture(String(dir), { BUN_ENABLE_EXPERIMENTAL_SHELL_BUILTINS: "1" })).toEqual({ + result: { copied: ["a.txt", "sub", "sub/b.txt"], stderr: "", exitCode: 0 }, + stderr: `${dtUnknownReaddir.marker}\n`, + exitCode: 0, + }); + }); +}); diff --git a/test/js/node/fs/dt-unknown-readdir.test.ts b/test/js/node/fs/dt-unknown-readdir.test.ts new file mode 100644 index 000000000000..72f3ec376b70 --- /dev/null +++ b/test/js/node/fs/dt-unknown-readdir.test.ts @@ -0,0 +1,114 @@ +// node:fs on filesystems whose readdir reports every entry as DT_UNKNOWN (FUSE, +// some NFS servers, XFS with ftype=0), simulated with the LD_PRELOAD shim from +// `dtUnknownReaddir` (harness). Anything that needs an entry's kind has to +// lstat it: a recursive fs.watch() used to take every entry for a file and +// never watch anything below the root; readdir() resolved the kinds already and +// is covered here because it now shares the implementation with the rest. +import { beforeAll, describe, expect, test } from "bun:test"; +import { bunExe, dtUnknownReaddir, tempDir } from "harness"; +import { symlinkSync } from "node:fs"; +import { join } from "node:path"; + +let shimEnv: NodeJS.Dict; + +beforeAll(async () => { + if (dtUnknownReaddir.available) shimEnv = await dtUnknownReaddir.env(); +}, 30_000); + +async function runFixture(dir: string): Promise<{ result: unknown; stderr: string; exitCode: number }> { + await using proc = Bun.spawn({ + cmd: [bunExe(), "fixture.mjs"], + cwd: dir, + env: shimEnv, + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + let result: unknown = stdout; + try { + result = JSON.parse(stdout); + } catch {} + return { result, stderr, exitCode }; +} + +// inotify reports events in the order they happened, so once the event for a +// file written after sub/inner.txt has arrived, inner.txt's event either arrived +// before it or was never going to. The second round guards against the first +// event being delivered on its own. +const WATCH_FIXTURE = /* js */ ` +import { watch, writeFileSync } from "node:fs"; +import { join } from "node:path"; + +const root = join(import.meta.dir, "root"); +const seen = new Set(); +let waitingFor = null; +const watcher = watch(root, { recursive: true }, (_event, filename) => { + seen.add(String(filename)); + waitingFor?.(); +}); +function write(name) { + const { promise, resolve } = Promise.withResolvers(); + waitingFor = () => seen.has(name) && resolve(); + writeFileSync(join(root, name), ""); + return promise; +} + +writeFileSync(join(root, "sub", "inner.txt"), ""); +await write("first.txt"); +await write("second.txt"); +watcher.close(); +console.log(JSON.stringify([...seen].sort())); +`; + +const READDIR_FIXTURE = /* js */ ` +import { readdirSync, promises } from "node:fs"; +import { join, relative } from "node:path"; + +const root = join(import.meta.dir, "root"); +const kind = d => (d.isDirectory() ? "dir" : d.isSymbolicLink() ? "symlink" : d.isFile() ? "file" : "unknown"); +const describe = entries => + entries.map(d => relative(root, join(d.parentPath, d.name)) + ":" + kind(d)).sort(); +const listings = async readdir => ({ + withFileTypes: describe(await readdir(root, { withFileTypes: true })), + recursive: (await readdir(root, { recursive: true })).sort(), + recursiveWithFileTypes: describe(await readdir(root, { recursive: true, withFileTypes: true })), +}); +console.log(JSON.stringify({ sync: await listings(readdirSync), async: await listings(promises.readdir) })); +`; + +const EXPECTED_LISTINGS = { + withFileTypes: ["a.txt:file", "link:symlink", "sub:dir"], + recursive: ["a.txt", "link", "sub", "sub/b.txt"], + recursiveWithFileTypes: ["a.txt:file", "link:symlink", "sub/b.txt:file", "sub:dir"], +}; + +describe.skipIf(!dtUnknownReaddir.available)("node:fs on a filesystem whose readdir reports DT_UNKNOWN", () => { + test.concurrent("recursive fs.watch watches the subdirectories", async () => { + using dir = tempDir("fs-watch-dt-unknown", { + "fixture.mjs": WATCH_FIXTURE, + "root/sub/.keep": "", + }); + + expect(await runFixture(String(dir))).toEqual({ + result: ["first.txt", "second.txt", "sub/inner.txt"], + stderr: `${dtUnknownReaddir.marker}\n`, + exitCode: 0, + }); + }); + + test.concurrent("readdir reports the kinds and recurses", async () => { + using dir = tempDir("fs-readdir-dt-unknown", { + "fixture.mjs": READDIR_FIXTURE, + "root/a.txt": "", + "root/sub/b.txt": "", + }); + symlinkSync("a.txt", join(String(dir), "root", "link")); + + expect(await runFixture(String(dir))).toEqual({ + result: { sync: EXPECTED_LISTINGS, async: EXPECTED_LISTINGS }, + stderr: `${dtUnknownReaddir.marker}\n`, + exitCode: 0, + }); + }); +});