From 9478e5063a076d884bdfe3474b73ecb62b1645aa Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Fri, 14 Aug 2026 22:38:57 +0000 Subject: [PATCH 1/8] pack: resolve directory entries whose readdir type is unknown On filesystems that do not fill in d_type (FUSE, NFS, XFS with ftype=0) every readdir entry has kind Unknown. The pack tree walkers only accept File and Directory, so `bun pm pack` and `bun publish` produced a tarball containing package.json and the explicitly listed bins and silently dropped everything else, including bundledDependencies. The publish "directories.bin" walk also stopped recursing into subdirectories there. Resolve Unknown with lstat (not stat, so symlinks still resolve to SymLink and stay out of the tarball) before the kind is looked at, in every pack and publish readdir consumer. --- src/runtime/cli/pack_command.rs | 28 ++- src/runtime/cli/publish_command.rs | 4 +- test/cli/install/bun-pack-dt-unknown.test.ts | 236 +++++++++++++++++++ 3 files changed, 261 insertions(+), 7 deletions(-) create mode 100644 test/cli/install/bun-pack-dt-unknown.test.ts diff --git a/src/runtime/cli/pack_command.rs b/src/runtime/cli/pack_command.rs index 4b767be83280..faec6af3d88f 100644 --- a/src/runtime/cli/pack_command.rs +++ b/src/runtime/cli/pack_command.rs @@ -476,8 +476,9 @@ fn iterate_included_project_tree( }); let mut dir_iter = DirIterator::iterate(Fd::from_std_dir(&dir)); - 'next_entry: while let Some(entry) = dir_iter.next().ok().flatten() { + 'next_entry: while let Some(mut entry) = dir_iter.next().ok().flatten() { // On iterator error, treat as end of iteration. + entry.kind = entry_kind(dir.fd, &entry); if entry.kind != bun_sys::FileKind::File && entry.kind != bun_sys::FileKind::Directory { continue; } @@ -713,7 +714,8 @@ fn add_entire_tree( } let mut iter = DirIterator::iterate(Fd::from_std_dir(&dir)); - 'next_entry: while let Some(entry) = iter.next().ok().flatten() { + 'next_entry: while let Some(mut entry) = iter.next().ok().flatten() { + entry.kind = entry_kind(dir.fd, &entry); if entry.kind != bun_sys::FileKind::File && entry.kind != bun_sys::FileKind::Directory { continue; } @@ -792,6 +794,20 @@ fn add_entire_tree( Ok(()) } +/// Some filesystems (FUSE, NFS, XFS formatted with `ftype=0`) do not report +/// entry types from readdir: every entry comes back as `Unknown`. Ask `lstat` +/// instead, which reports what `d_type` would have (a symlink is still a +/// symlink, so it is still not packed). +pub(crate) fn entry_kind(dir: Fd, entry: &DirIterator::IteratorResult) -> bun_sys::FileKind { + if entry.kind != bun_sys::FileKind::Unknown { + return entry.kind; + } + match bun_sys::lstatat(dir, &ZBox::from_bytes(entry.name.slice_u8())) { + Ok(stat) => bun_sys::kind_from_mode(stat.st_mode as bun_sys::Mode), + Err(_) => bun_sys::FileKind::Unknown, + } +} + fn open_subdir(dir: &Dir, entry_name: &[u8], entry_subpath: &ZStr) -> Dir { match dir_open_dir_z( dir, @@ -885,7 +901,7 @@ fn iterate_bundled_deps( let mut iter = DirIterator::iterate(Fd::from_std_dir(&dir)); while let Some(entry) = iter.next().ok().flatten() { - if entry.kind != bun_sys::FileKind::Directory { + if entry_kind(dir.fd, &entry) != bun_sys::FileKind::Directory { continue; } @@ -1022,7 +1038,8 @@ fn add_bundled_dep( let DirInfo(dir, dir_subpath, dir_depth) = dir_info; let mut iter = DirIterator::iterate(Fd::from_std_dir(&dir)); - while let Some(entry) = iter.next().ok().flatten() { + while let Some(mut entry) = iter.next().ok().flatten() { + entry.kind = entry_kind(dir.fd, &entry); if entry.kind != bun_sys::FileKind::File && entry.kind != bun_sys::FileKind::Directory { continue; } @@ -1284,7 +1301,8 @@ fn iterate_project_tree( } let mut dir_iter = DirIterator::iterate(Fd::from_std_dir(&dir)); - 'next_entry: while let Some(entry) = dir_iter.next().ok().flatten() { + 'next_entry: while let Some(mut entry) = dir_iter.next().ok().flatten() { + entry.kind = entry_kind(dir.fd, &entry); if entry.kind != bun_sys::FileKind::File && entry.kind != bun_sys::FileKind::Directory { continue; } diff --git a/src/runtime/cli/publish_command.rs b/src/runtime/cli/publish_command.rs index ac6e1736eb84..987283e5eb67 100644 --- a/src/runtime/cli/publish_command.rs +++ b/src/runtime/cli/publish_command.rs @@ -1589,7 +1589,7 @@ impl PublishCommand { let mut iter = DirIterator::iterate(workspace_dir); while let Some(entry) = iter.next().ok().flatten() { - if entry.kind == bun_sys::EntryKind::Directory { + if pack::entry_kind(workspace_dir, &entry) == bun_sys::EntryKind::Directory { continue; } // Entry names are UTF-8 on every platform. @@ -1861,7 +1861,7 @@ impl PublishCommand { ..Default::default() }); - if entry.kind == bun_sys::EntryKind::Directory { + if pack::entry_kind(dir, &entry) == bun_sys::EntryKind::Directory { let Ok(subdir) = bun_sys::openat(dir, name, bun_sys::O::DIRECTORY, 0) else { continue; diff --git a/test/cli/install/bun-pack-dt-unknown.test.ts b/test/cli/install/bun-pack-dt-unknown.test.ts new file mode 100644 index 000000000000..541c52ae76fb --- /dev/null +++ b/test/cli/install/bun-pack-dt-unknown.test.ts @@ -0,0 +1,236 @@ +// Some filesystems (FUSE, NFS, XFS formatted with ftype=0) do not fill in +// d_type, so every readdir entry comes back as DT_UNKNOWN. `bun pm pack` and +// `bun publish` must still pack those entries. A FUSE mount needs /dev/fuse, so +// instead an LD_PRELOAD shim zeroes d_type in every getdents64 record: bun +// issues getdents64 through libc's syscall(2) wrapper, which the shim +// interposes. The shim announces itself on stderr the first time it rewrites a +// record so the tests can tell that bun's readdir actually went through it +// (if bun ever stops using the libc wrapper, this file needs a real DT_UNKNOWN +// filesystem instead of silently passing). +import { readTarball } from "bun:internal-for-testing"; +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { bunEnv, bunExe, isLinux, tempDir } from "harness"; +import { symlink, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +const cc = Bun.which("cc") || Bun.which("gcc") || Bun.which("clang"); + +const SHIM_MARKER = "dt-unknown-shim: rewrote getdents64 d_type"; + +const SHIM_C = /* c */ ` +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include + +static long (*real_syscall)(long, long, long, long, long, long, long); +static int announced; + +long syscall(long number, ...) { + va_list ap; + long a, b, c, d, e, f; + va_start(ap, number); + a = va_arg(ap, long); + b = va_arg(ap, long); + c = va_arg(ap, long); + d = va_arg(ap, long); + e = va_arg(ap, long); + f = va_arg(ap, long); + va_end(ap); + if (!real_syscall) { + real_syscall = (long (*)(long, long, long, long, long, long, long))dlsym(RTLD_NEXT, "syscall"); + } + long rc = real_syscall(number, a, b, c, d, e, f); + if (number != SYS_getdents64 || rc <= 0) return rc; + if (!announced) { + announced = 1; + static const char marker[] = "${SHIM_MARKER}\\n"; + if (write(2, marker, sizeof(marker) - 1) < 0) {} + } + // struct linux_dirent64 { u64 d_ino; s64 d_off; u16 d_reclen; u8 d_type; char d_name[]; } + unsigned char *buf = (unsigned char *)b; + for (long off = 0; off + 19 <= rc;) { + uint16_t reclen; + memcpy(&reclen, buf + off + 16, sizeof(reclen)); + if (reclen == 0) break; + buf[off + 18] = 0; /* DT_UNKNOWN */ + off += reclen; + } + return rc; +} +`; + +let shimDir: ReturnType | undefined; +let shimPath: string; + +beforeAll(async () => { + if (!isLinux || !cc) return; + shimDir = tempDir("dt-unknown-shim", { "shim.c": SHIM_C }); + shimPath = join(String(shimDir), "shim.so"); + await using proc = Bun.spawn({ + cmd: [cc, "-shared", "-fPIC", "-o", shimPath, join(String(shimDir), "shim.c"), "-ldl"], + env: bunEnv, + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + if (exitCode !== 0) { + throw new Error(`shim compile failed: ${stderr || stdout}`); + } +}); + +afterAll(() => { + shimDir?.[Symbol.dispose](); +}); + +function shimEnv() { + return { ...bunEnv, LD_PRELOAD: bunEnv.LD_PRELOAD ? `${shimPath}:${bunEnv.LD_PRELOAD}` : shimPath }; +} + +async function run(cwd: string, ...args: string[]) { + await using proc = Bun.spawn({ + cmd: [bunExe(), ...args], + cwd, + env: shimEnv(), + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + expect(stderr).toContain(SHIM_MARKER); + expect({ stdout, stderr, exitCode }).toMatchObject({ stderr: expect.not.stringContaining("error:"), exitCode: 0 }); +} + +function packedPaths(tarball: string): string[] { + return readTarball(tarball) + .entries.map((entry: { pathname: string }) => entry.pathname) + .sort(); +} + +describe.skipIf(!isLinux || !cc)("pack on a filesystem whose readdir reports DT_UNKNOWN", () => { + test.concurrent("packs the project tree", async () => { + using dir = tempDir("dt-unknown-tree", { + "package.json": JSON.stringify({ name: "dt-unknown-tree", version: "1.0.0" }), + "index.js": "", + "lib/a.js": "", + "lib/nested/b.js": "", + // `out/` only ignores directories, so it needs the entry's kind: the + // `out` directory is ignored, the `lib/out` file is not. + ".npmignore": "out/\n", + "out/c.js": "", + "lib/out": "", + }); + // Symlinks are never packed; resolving the kind with lstat has to keep that. + await symlink("index.js", join(String(dir), "link.js")); + + await run(String(dir), "pm", "pack"); + + expect(packedPaths(join(String(dir), "dt-unknown-tree-1.0.0.tgz"))).toEqual([ + "package/index.js", + "package/lib/a.js", + "package/lib/nested/b.js", + "package/lib/out", + "package/package.json", + ]); + }); + + test.concurrent('packs what "files" selects', async () => { + using dir = tempDir("dt-unknown-files", { + "package.json": JSON.stringify({ + name: "dt-unknown-files", + version: "1.0.0", + files: ["index.js", "lib", "!lib/internal/"], + }), + "index.js": "", + "excluded.js": "", + "lib/a.js": "", + "lib/nested/b.js": "", + "lib/internal/c.js": "", + }); + await symlink("a.js", join(String(dir), "lib", "link.js")); + + await run(String(dir), "pm", "pack"); + + expect(packedPaths(join(String(dir), "dt-unknown-files-1.0.0.tgz"))).toEqual([ + "package/index.js", + "package/lib/a.js", + "package/lib/nested/b.js", + "package/package.json", + ]); + }); + + test.concurrent("packs bundledDependencies", async () => { + using dir = tempDir("dt-unknown-bundled", { + "package.json": JSON.stringify({ + name: "dt-unknown-bundled", + version: "1.0.0", + dependencies: { "dep": "1.0.0", "@scope/dep": "1.0.0", "not-bundled": "1.0.0" }, + bundledDependencies: ["dep", "@scope/dep"], + }), + "index.js": "", + "node_modules/dep/package.json": JSON.stringify({ name: "dep", version: "1.0.0" }), + "node_modules/dep/lib/index.js": "", + "node_modules/@scope/dep/package.json": JSON.stringify({ name: "@scope/dep", version: "1.0.0" }), + "node_modules/@scope/dep/index.js": "", + "node_modules/not-bundled/package.json": JSON.stringify({ name: "not-bundled", version: "1.0.0" }), + }); + + await run(String(dir), "pm", "pack"); + + expect(packedPaths(join(String(dir), "dt-unknown-bundled-1.0.0.tgz"))).toEqual([ + "package/index.js", + "package/node_modules/@scope/dep/index.js", + "package/node_modules/@scope/dep/package.json", + "package/node_modules/dep/lib/index.js", + "package/node_modules/dep/package.json", + "package/package.json", + ]); + }); + + test.concurrent('publish packs the tree, walks "directories.bin" and finds the readme', async () => { + let captured: any; + using registry = Bun.serve({ + port: 0, + async fetch(req) { + if (req.method === "PUT") captured = await req.json(); + return new Response("OK"); + }, + }); + using dir = tempDir("dt-unknown-publish", { + "bunfig.toml": Bun.TOML.stringify({ + install: { cache: false, registry: { url: `http://localhost:${registry.port}`, token: "unused" } }, + }), + "package.json": JSON.stringify({ + name: "dt-unknown-publish", + version: "1.0.0", + directories: { bin: "bins" }, + }), + "README.md": "# dt-unknown-publish", + "index.js": "", + "bins/a.js": "", + "bins/more/b.js": "", + }); + + await run(String(dir), "publish"); + + expect(captured.versions["1.0.0"]).toMatchObject({ + bin: { "a.js": "bins/a.js", "more": "bins/more", "b.js": "bins/more/b.js" }, + readme: "# dt-unknown-publish", + readmeFilename: "README.md", + }); + + const attachment: { data: string } = Object.values(captured._attachments)[0] as any; + const tarball = join(String(dir), "published.tgz"); + await writeFile(tarball, Buffer.from(attachment.data, "base64")); + expect(packedPaths(tarball)).toEqual([ + "package/README.md", + "package/bins/a.js", + "package/bins/more/b.js", + "package/index.js", + "package/package.json", + ]); + }); +}); From c9c9c24973c9dae400686bc2b8196dae95cad022 Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Sat, 15 Aug 2026 05:59:48 +0000 Subject: [PATCH 2/8] sys: add IteratorResult::resolve_kind, use it from pack and publish The lstat fallback belongs next to the iterator that hands out Unknown in the first place: it can use the entry's NUL-terminated name directly instead of copying it, and publish no longer imports a readdir helper from pack. Windows never yields Unknown, so the method is the identity there. --- src/runtime/cli/pack_command.rs | 26 ++++++------------------ src/runtime/cli/publish_command.rs | 8 ++++---- src/sys/lib.rs | 32 +++++++++++++++++++++++++++++- 3 files changed, 41 insertions(+), 25 deletions(-) diff --git a/src/runtime/cli/pack_command.rs b/src/runtime/cli/pack_command.rs index faec6af3d88f..f7edef8c19ed 100644 --- a/src/runtime/cli/pack_command.rs +++ b/src/runtime/cli/pack_command.rs @@ -478,7 +478,7 @@ fn iterate_included_project_tree( let mut dir_iter = DirIterator::iterate(Fd::from_std_dir(&dir)); 'next_entry: while let Some(mut entry) = dir_iter.next().ok().flatten() { // On iterator error, treat as end of iteration. - entry.kind = entry_kind(dir.fd, &entry); + entry.resolve_kind(dir.fd); if entry.kind != bun_sys::FileKind::File && entry.kind != bun_sys::FileKind::Directory { continue; } @@ -715,7 +715,7 @@ fn add_entire_tree( let mut iter = DirIterator::iterate(Fd::from_std_dir(&dir)); 'next_entry: while let Some(mut entry) = iter.next().ok().flatten() { - entry.kind = entry_kind(dir.fd, &entry); + entry.resolve_kind(dir.fd); if entry.kind != bun_sys::FileKind::File && entry.kind != bun_sys::FileKind::Directory { continue; } @@ -794,20 +794,6 @@ fn add_entire_tree( Ok(()) } -/// Some filesystems (FUSE, NFS, XFS formatted with `ftype=0`) do not report -/// entry types from readdir: every entry comes back as `Unknown`. Ask `lstat` -/// instead, which reports what `d_type` would have (a symlink is still a -/// symlink, so it is still not packed). -pub(crate) fn entry_kind(dir: Fd, entry: &DirIterator::IteratorResult) -> bun_sys::FileKind { - if entry.kind != bun_sys::FileKind::Unknown { - return entry.kind; - } - match bun_sys::lstatat(dir, &ZBox::from_bytes(entry.name.slice_u8())) { - Ok(stat) => bun_sys::kind_from_mode(stat.st_mode as bun_sys::Mode), - Err(_) => bun_sys::FileKind::Unknown, - } -} - fn open_subdir(dir: &Dir, entry_name: &[u8], entry_subpath: &ZStr) -> Dir { match dir_open_dir_z( dir, @@ -900,8 +886,8 @@ fn iterate_bundled_deps( let mut additional_bundled_deps: Vec = Vec::new(); let mut iter = DirIterator::iterate(Fd::from_std_dir(&dir)); - while let Some(entry) = iter.next().ok().flatten() { - if entry_kind(dir.fd, &entry) != bun_sys::FileKind::Directory { + while let Some(mut entry) = iter.next().ok().flatten() { + if entry.resolve_kind(dir.fd) != bun_sys::FileKind::Directory { continue; } @@ -1039,7 +1025,7 @@ fn add_bundled_dep( let mut iter = DirIterator::iterate(Fd::from_std_dir(&dir)); while let Some(mut entry) = iter.next().ok().flatten() { - entry.kind = entry_kind(dir.fd, &entry); + entry.resolve_kind(dir.fd); if entry.kind != bun_sys::FileKind::File && entry.kind != bun_sys::FileKind::Directory { continue; } @@ -1302,7 +1288,7 @@ fn iterate_project_tree( let mut dir_iter = DirIterator::iterate(Fd::from_std_dir(&dir)); 'next_entry: while let Some(mut entry) = dir_iter.next().ok().flatten() { - entry.kind = entry_kind(dir.fd, &entry); + entry.resolve_kind(dir.fd); if entry.kind != bun_sys::FileKind::File && entry.kind != bun_sys::FileKind::Directory { continue; } diff --git a/src/runtime/cli/publish_command.rs b/src/runtime/cli/publish_command.rs index 987283e5eb67..59e22c4a0d6a 100644 --- a/src/runtime/cli/publish_command.rs +++ b/src/runtime/cli/publish_command.rs @@ -1588,8 +1588,8 @@ impl PublishCommand { }); let mut iter = DirIterator::iterate(workspace_dir); - while let Some(entry) = iter.next().ok().flatten() { - if pack::entry_kind(workspace_dir, &entry) == bun_sys::EntryKind::Directory { + while let Some(mut entry) = iter.next().ok().flatten() { + if entry.resolve_kind(workspace_dir) == bun_sys::EntryKind::Directory { continue; } // Entry names are UTF-8 on every platform. @@ -1810,7 +1810,7 @@ impl PublishCommand { }); let mut iter = DirIterator::iterate(dir); - while let Some(entry) = iter.next().ok().flatten() { + while let Some(mut entry) = iter.next().ok().flatten() { let (name, subpath): (&'static ZStr, &'static ZStr) = { // Entry names are UTF-8 on every platform. let name = entry.name.slice_u8(); @@ -1861,7 +1861,7 @@ impl PublishCommand { ..Default::default() }); - if pack::entry_kind(dir, &entry) == bun_sys::EntryKind::Directory { + if entry.resolve_kind(dir) == bun_sys::EntryKind::Directory { let Ok(subdir) = bun_sys::openat(dir, name, bun_sys::O::DIRECTORY, 0) else { continue; diff --git a/src/sys/lib.rs b/src/sys/lib.rs index e563f9bb54b8..52a3822f9bdc 100644 --- a/src/sys/lib.rs +++ b/src/sys/lib.rs @@ -119,9 +119,38 @@ pub mod dir_iterator { /// step. pub struct IteratorResult { pub name: Name, + /// `Unknown` on filesystems that do not fill in `d_type` (FUSE, NFS, + /// XFS formatted with `ftype=0`); see [`IteratorResult::resolve_kind`]. pub kind: EntryKind, } + impl IteratorResult { + /// Resolves an `Unknown` kind in place and returns the kind. + /// + /// Uses `lstat`, which reports the same thing `d_type` would have (a + /// symlink is reported as a symlink, not as its target), so callers + /// behave the same on filesystems with and without `d_type`. `dir` is + /// the directory the entry was read from. The kind stays `Unknown` if + /// the entry cannot be stat'ed (for example it was removed since). + pub fn resolve_kind(&mut self, dir: Fd) -> EntryKind { + #[cfg(not(windows))] + { + if self.kind == EntryKind::Unknown { + if let Ok(stat) = super::lstatat(dir, self.name.as_zstr()) { + self.kind = super::kind_from_mode(stat.st_mode as super::Mode); + } + } + } + #[cfg(windows)] + { + // `NtQueryDirectoryFile` always reports attributes, so the + // Windows iterator never yields `Unknown`. + let _ = dir; + } + self.kind + } + } + /// Length-known, NUL-terminated entry name in OS-native encoding. /// /// **POSIX**: lifetime-erased borrow (raw pointer + length) into the @@ -266,7 +295,8 @@ pub mod dir_iterator { #[cfg(any(target_os = "macos", target_os = "freebsd"))] 14 /* DT_WHT */ => EntryKind::Whiteout, // DT_UNKNOWN: some filesystems (bind mounts, FUSE, NFS) don't - // provide d_type. Callers should lstatat() to resolve when needed. + // provide d_type. Callers that need the type call + // `IteratorResult::resolve_kind()`. _ => EntryKind::Unknown, } } From d635214175e34c3e19b73fbb2cac02c1728e3f3c Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Sat, 15 Aug 2026 06:04:05 +0000 Subject: [PATCH 3/8] sys: shorten the resolve_kind comments --- src/sys/lib.rs | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/src/sys/lib.rs b/src/sys/lib.rs index 52a3822f9bdc..9a7d94206380 100644 --- a/src/sys/lib.rs +++ b/src/sys/lib.rs @@ -119,19 +119,14 @@ pub mod dir_iterator { /// step. pub struct IteratorResult { pub name: Name, - /// `Unknown` on filesystems that do not fill in `d_type` (FUSE, NFS, - /// XFS formatted with `ftype=0`); see [`IteratorResult::resolve_kind`]. pub kind: EntryKind, } impl IteratorResult { - /// Resolves an `Unknown` kind in place and returns the kind. - /// - /// Uses `lstat`, which reports the same thing `d_type` would have (a - /// symlink is reported as a symlink, not as its target), so callers - /// behave the same on filesystems with and without `d_type`. `dir` is - /// the directory the entry was read from. The kind stays `Unknown` if - /// the entry cannot be stat'ed (for example it was removed since). + /// Replaces an `Unknown` kind (the filesystem did not fill in `d_type`) + /// with what `lstat` reports for the entry in `dir`, the directory it was + /// read from, and returns the kind. `lstat` so that a symlink is reported + /// the way `d_type` reports it, not as its target. pub fn resolve_kind(&mut self, dir: Fd) -> EntryKind { #[cfg(not(windows))] { @@ -143,8 +138,7 @@ pub mod dir_iterator { } #[cfg(windows)] { - // `NtQueryDirectoryFile` always reports attributes, so the - // Windows iterator never yields `Unknown`. + // The Windows iterator always knows the kind. let _ = dir; } self.kind @@ -294,9 +288,7 @@ pub mod dir_iterator { // literal matches . #[cfg(any(target_os = "macos", target_os = "freebsd"))] 14 /* DT_WHT */ => EntryKind::Whiteout, - // DT_UNKNOWN: some filesystems (bind mounts, FUSE, NFS) don't - // provide d_type. Callers that need the type call - // `IteratorResult::resolve_kind()`. + // DT_UNKNOWN (FUSE, NFS, ...): see `IteratorResult::resolve_kind`. _ => EntryKind::Unknown, } } From 093ab3fcb6595918832fe173ea9deb56f8a74dc0 Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Sat, 15 Aug 2026 06:08:02 +0000 Subject: [PATCH 4/8] sys: tighten the resolve_kind doc comment --- src/sys/lib.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/sys/lib.rs b/src/sys/lib.rs index 9a7d94206380..6ffea61468ae 100644 --- a/src/sys/lib.rs +++ b/src/sys/lib.rs @@ -123,10 +123,9 @@ pub mod dir_iterator { } impl IteratorResult { - /// Replaces an `Unknown` kind (the filesystem did not fill in `d_type`) - /// with what `lstat` reports for the entry in `dir`, the directory it was - /// read from, and returns the kind. `lstat` so that a symlink is reported - /// the way `d_type` reports it, not as its target. + /// Resolves an `Unknown` kind (no `d_type` from the filesystem) by `lstat`ing + /// the entry in `dir`, the directory it was read from. `lstat`, not `stat`, + /// so a symlink stays a symlink as it does with `d_type`. pub fn resolve_kind(&mut self, dir: Fd) -> EntryKind { #[cfg(not(windows))] { From a02e49a101c28bcf45f5fcd3d3fd4f374d9aa71f Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Sat, 15 Aug 2026 07:53:57 +0000 Subject: [PATCH 5/8] sys: resolve unknown entry types inside the directory iterator Replace the per-entry resolve_kind() with an opt-in flag on WrappedIterator, applied in next(). walker_skippable and prune drop their own copies of the lstat fallback and set the flag instead; pack and publish set it on each iterator they create. The getdents64 d_type-zeroing shim moves out of the test into test/fixtures, with a harness helper so other tests can use it; the test gains an install-from-folder case covering the walker. --- src/install/prune.rs | 16 +- src/runtime/cli/pack_command.rs | 21 +- src/runtime/cli/publish_command.rs | 10 +- src/sys/lib.rs | 25 ++- src/sys/walker_skippable.rs | 203 ++++++++---------- ...own.test.ts => dt-unknown-readdir.test.ts} | 119 +++------- test/fixtures/dt-unknown-readdir-shim.c | 50 +++++ test/harness.ts | 23 ++ 8 files changed, 229 insertions(+), 238 deletions(-) rename test/cli/install/{bun-pack-dt-unknown.test.ts => dt-unknown-readdir.test.ts} (61%) create mode 100644 test/fixtures/dt-unknown-readdir-shim.c diff --git a/src/install/prune.rs b/src/install/prune.rs index 01aa08b4607c..488295d72f56 100644 --- a/src/install/prune.rs +++ b/src/install/prune.rs @@ -1343,24 +1343,17 @@ fn lstat_kind(dir: &Dir, name: &[u8]) -> EntryKind { } } -fn entry_kind(dir: &Dir, name: &[u8], kind: EntryKind) -> EntryKind { - if kind != EntryKind::Unknown { - return kind; - } - lstat_kind(dir, name) -} - fn read_entries(dir: &Dir) -> Vec<(Box<[u8]>, EntryKind)> { let mut out = Vec::new(); let mut iter = sys::iterate_dir(dir.fd()); + iter.resolve_unknown_entry_types = true; while let Ok(Some(entry)) = iter.next() { let name = entry.name.slice_u8(); if name.first() == Some(&b'.') { continue; } - let kind = entry_kind(dir, name, entry.kind); - if kind == EntryKind::Directory || kind == EntryKind::SymLink { - out.push((name.into(), kind)); + if entry.kind == EntryKind::Directory || entry.kind == EntryKind::SymLink { + out.push((name.into(), entry.kind)); } } out @@ -1780,9 +1773,10 @@ fn prune_bins(dir: &Dir) { }; let mut dangling: Vec> = Vec::new(); let mut iter = sys::iterate_dir(bin.fd()); + iter.resolve_unknown_entry_types = true; while let Ok(Some(entry)) = iter.next() { let name = entry.name.slice_u8(); - if entry_kind(&bin, name, entry.kind) == EntryKind::SymLink && is_dangling(&bin, name) { + if entry.kind == EntryKind::SymLink && is_dangling(&bin, name) { dangling.push(name.into()); } } diff --git a/src/runtime/cli/pack_command.rs b/src/runtime/cli/pack_command.rs index f7edef8c19ed..b0e02568b97e 100644 --- a/src/runtime/cli/pack_command.rs +++ b/src/runtime/cli/pack_command.rs @@ -476,9 +476,9 @@ fn iterate_included_project_tree( }); let mut dir_iter = DirIterator::iterate(Fd::from_std_dir(&dir)); - 'next_entry: while let Some(mut entry) = dir_iter.next().ok().flatten() { + dir_iter.resolve_unknown_entry_types = true; + 'next_entry: while let Some(entry) = dir_iter.next().ok().flatten() { // On iterator error, treat as end of iteration. - entry.resolve_kind(dir.fd); if entry.kind != bun_sys::FileKind::File && entry.kind != bun_sys::FileKind::Directory { continue; } @@ -714,8 +714,8 @@ fn add_entire_tree( } let mut iter = DirIterator::iterate(Fd::from_std_dir(&dir)); - 'next_entry: while let Some(mut entry) = iter.next().ok().flatten() { - entry.resolve_kind(dir.fd); + iter.resolve_unknown_entry_types = true; + 'next_entry: while let Some(entry) = iter.next().ok().flatten() { if entry.kind != bun_sys::FileKind::File && entry.kind != bun_sys::FileKind::Directory { continue; } @@ -886,8 +886,9 @@ fn iterate_bundled_deps( let mut additional_bundled_deps: Vec = Vec::new(); let mut iter = DirIterator::iterate(Fd::from_std_dir(&dir)); - while let Some(mut entry) = iter.next().ok().flatten() { - if entry.resolve_kind(dir.fd) != bun_sys::FileKind::Directory { + iter.resolve_unknown_entry_types = true; + while let Some(entry) = iter.next().ok().flatten() { + if entry.kind != bun_sys::FileKind::Directory { continue; } @@ -1024,8 +1025,8 @@ fn add_bundled_dep( let DirInfo(dir, dir_subpath, dir_depth) = dir_info; let mut iter = DirIterator::iterate(Fd::from_std_dir(&dir)); - while let Some(mut entry) = iter.next().ok().flatten() { - entry.resolve_kind(dir.fd); + iter.resolve_unknown_entry_types = true; + while let Some(entry) = iter.next().ok().flatten() { if entry.kind != bun_sys::FileKind::File && entry.kind != bun_sys::FileKind::Directory { continue; } @@ -1287,8 +1288,8 @@ fn iterate_project_tree( } let mut dir_iter = DirIterator::iterate(Fd::from_std_dir(&dir)); - 'next_entry: while let Some(mut entry) = dir_iter.next().ok().flatten() { - entry.resolve_kind(dir.fd); + dir_iter.resolve_unknown_entry_types = true; + 'next_entry: while let Some(entry) = dir_iter.next().ok().flatten() { if entry.kind != bun_sys::FileKind::File && entry.kind != bun_sys::FileKind::Directory { continue; } diff --git a/src/runtime/cli/publish_command.rs b/src/runtime/cli/publish_command.rs index 59e22c4a0d6a..b8c8dab974f6 100644 --- a/src/runtime/cli/publish_command.rs +++ b/src/runtime/cli/publish_command.rs @@ -1588,8 +1588,9 @@ impl PublishCommand { }); let mut iter = DirIterator::iterate(workspace_dir); - while let Some(mut entry) = iter.next().ok().flatten() { - if entry.resolve_kind(workspace_dir) == bun_sys::EntryKind::Directory { + iter.resolve_unknown_entry_types = true; + while let Some(entry) = iter.next().ok().flatten() { + if entry.kind == bun_sys::EntryKind::Directory { continue; } // Entry names are UTF-8 on every platform. @@ -1810,7 +1811,8 @@ impl PublishCommand { }); let mut iter = DirIterator::iterate(dir); - while let Some(mut entry) = iter.next().ok().flatten() { + iter.resolve_unknown_entry_types = true; + while let Some(entry) = iter.next().ok().flatten() { let (name, subpath): (&'static ZStr, &'static ZStr) = { // Entry names are UTF-8 on every platform. let name = entry.name.slice_u8(); @@ -1861,7 +1863,7 @@ impl PublishCommand { ..Default::default() }); - if entry.resolve_kind(dir) == bun_sys::EntryKind::Directory { + if entry.kind == bun_sys::EntryKind::Directory { let Ok(subdir) = bun_sys::openat(dir, name, bun_sys::O::DIRECTORY, 0) else { continue; diff --git a/src/sys/lib.rs b/src/sys/lib.rs index 6ffea61468ae..8370157d7c79 100644 --- a/src/sys/lib.rs +++ b/src/sys/lib.rs @@ -123,10 +123,8 @@ pub mod dir_iterator { } impl IteratorResult { - /// Resolves an `Unknown` kind (no `d_type` from the filesystem) by `lstat`ing - /// the entry in `dir`, the directory it was read from. `lstat`, not `stat`, - /// so a symlink stays a symlink as it does with `d_type`. - pub fn resolve_kind(&mut self, dir: Fd) -> EntryKind { + /// See `WrappedIterator::resolve_unknown_entry_types`. + fn resolve_unknown_kind(&mut self, dir: Fd) { #[cfg(not(windows))] { if self.kind == EntryKind::Unknown { @@ -140,7 +138,6 @@ pub mod dir_iterator { // The Windows iterator always knows the kind. let _ = dir; } - self.kind } } @@ -287,7 +284,7 @@ pub mod dir_iterator { // literal matches . #[cfg(any(target_os = "macos", target_os = "freebsd"))] 14 /* DT_WHT */ => EntryKind::Whiteout, - // DT_UNKNOWN (FUSE, NFS, ...): see `IteratorResult::resolve_kind`. + // DT_UNKNOWN: see `WrappedIterator::resolve_unknown_entry_types`. _ => EntryKind::Unknown, } } @@ -752,6 +749,12 @@ pub mod dir_iterator { #[cfg(not(windows))] name_filter: Option>, state: State, + /// Filesystems that do not fill in `d_type` (FUSE, NFS, XFS with `ftype=0`) + /// report every entry as `Unknown`. When set, `next()` resolves those with + /// `lstat` (a symlink stays a symlink, as with `d_type`); an entry that + /// cannot be stat'ed stays `Unknown`. Off by default: the resolver and glob + /// only want the kind of the few entries they end up using. + pub resolve_unknown_entry_types: bool, } impl WrappedIterator { #[inline] @@ -780,7 +783,13 @@ pub mod dir_iterator { /// Copy it out before pushing the iterator into a `Vec` etc. #[inline] pub fn next(&mut self) -> Result> { - self.state.next(self.dir) + let mut entry = self.state.next(self.dir)?; + if self.resolve_unknown_entry_types { + if let Some(entry) = entry.as_mut() { + entry.resolve_unknown_kind(self.dir); + } + } + Ok(entry) } } @@ -791,6 +800,7 @@ pub mod dir_iterator { dir, name_filter: None, state: State::new(), + resolve_unknown_entry_types: false, } } #[cfg(windows)] @@ -798,6 +808,7 @@ pub mod dir_iterator { WrappedIterator { dir, state: State::new(), + resolve_unknown_entry_types: false, } } } diff --git a/src/sys/walker_skippable.rs b/src/sys/walker_skippable.rs index d4d7d854b72f..849c6d5766ad 100644 --- a/src/sys/walker_skippable.rs +++ b/src/sys/walker_skippable.rs @@ -26,6 +26,7 @@ pub struct Walker { skip_dirnames: Range, skip_all: Box<[u64]>, seed: u64, + /// See `dir_iterator::WrappedIterator::resolve_unknown_entry_types`. pub resolve_unknown_entry_types: bool, } @@ -76,127 +77,99 @@ impl Walker { // be invalidated by appending to `self.stack` below. let top_idx = self.stack.len() - 1; let mut dirname_len = self.stack[top_idx].dirname_len; - match self.stack[top_idx].iter.next() { - Err(err) => return Err(err), - Ok(res) => { - if let Some(base) = res { - // Some filesystems (NFS, FUSE, bind mounts) don't provide - // d_type and return DT_UNKNOWN. Optionally resolve via - // fstatat so callers get accurate types for recursion. - // This only affects POSIX; Windows always provides types. - #[cfg(not(windows))] - let kind: sys::EntryKind = if base.kind == sys::EntryKind::Unknown - && self.resolve_unknown_entry_types - { - let dir_fd = self.stack[top_idx].iter.dir(); - match sys::lstatat(dir_fd, base.name.as_zstr()) { - Ok(stat_buf) => sys::kind_from_mode(stat_buf.st_mode as sys::Mode), - Err(_) => continue, // skip entries we can't stat - } + // Callers set the flag after `walk()` has already created the root + // iterator, so pass it down per call rather than at construction. + self.stack[top_idx].iter.resolve_unknown_entry_types = self.resolve_unknown_entry_types; + let Some(base) = self.stack[top_idx].iter.next()? else { + let item = self.stack.pop().unwrap(); + if !self.stack.is_empty() { + item.iter.dir().close(); + } + continue; + }; + let kind = base.kind; + + match kind { + sys::EntryKind::Directory => { + let skip = &self.skip_all[self.skip_dirnames.clone()]; + if skip.contains( + // avoid hashing if there will be 0 results + &(if !skip.is_empty() { + hash_with_seed(self.seed, slice_as_bytes(base.name.as_slice())) } else { - base.kind - }; - #[cfg(windows)] - let kind: sys::EntryKind = base.kind; - - match kind { - sys::EntryKind::Directory => { - let skip = &self.skip_all[self.skip_dirnames.clone()]; - if skip.contains( - // avoid hashing if there will be 0 results - &(if !skip.is_empty() { - hash_with_seed( - self.seed, - slice_as_bytes(base.name.as_slice()), - ) - } else { - 0 - }), - ) { - continue; - } - } - sys::EntryKind::File => { - let skip = &self.skip_all[self.skip_filenames.clone()]; - if skip.contains( - // avoid hashing if there will be 0 results - &(if !skip.is_empty() { - hash_with_seed( - self.seed, - slice_as_bytes(base.name.as_slice()), - ) - } else { - 0 - }), - ) { - continue; - } - } - - // we don't know what it is for a symlink - sys::EntryKind::SymLink => { - let skip = &self.skip_all[..]; - if skip.contains( - // avoid hashing if there will be 0 results - &(if !skip.is_empty() { - hash_with_seed( - self.seed, - slice_as_bytes(base.name.as_slice()), - ) - } else { - 0 - }), - ) { - continue; - } - } + 0 + }), + ) { + continue; + } + } + sys::EntryKind::File => { + let skip = &self.skip_all[self.skip_filenames.clone()]; + if skip.contains( + // avoid hashing if there will be 0 results + &(if !skip.is_empty() { + hash_with_seed(self.seed, slice_as_bytes(base.name.as_slice())) + } else { + 0 + }), + ) { + continue; + } + } - _ => {} - } + // we don't know what it is for a symlink + sys::EntryKind::SymLink => { + let skip = &self.skip_all[..]; + if skip.contains( + // avoid hashing if there will be 0 results + &(if !skip.is_empty() { + hash_with_seed(self.seed, slice_as_bytes(base.name.as_slice())) + } else { + 0 + }), + ) { + continue; + } + } - self.name_buffer.truncate(dirname_len); - if !self.name_buffer.is_empty() { - self.name_buffer.push(SEP as OSPathChar); - dirname_len += 1; - } - self.name_buffer.extend_from_slice(base.name.as_slice()); - let cur_len = self.name_buffer.len(); - self.name_buffer.push(0); + _ => {} + } - let mut top_idx = top_idx; - if kind == sys::EntryKind::Directory { - let new_dir = sys::open_dir_for_iteration_os_path( - self.stack[top_idx].iter.dir(), - base.name.as_slice(), - )?; - { - self.stack.push(StackItem { - iter: dir_iterator::iterate(new_dir), - dirname_len: cur_len, - }); - top_idx = self.stack.len() - 1; - } - } - // `name_buffer[cur_len] == 0` was written above; both views end at - // `cur_len` and are NUL-terminated by that sentinel char. `from_buf` - // ties the borrow to `&self.name_buffer` (no raw-pointer reslice). - return Ok(Some(WalkerEntry { - dir: self.stack[top_idx].iter.dir(), - basename: OSPathSliceZ::from_buf( - &self.name_buffer[dirname_len..], - cur_len - dirname_len, - ), - path: OSPathSliceZ::from_buf(&self.name_buffer, cur_len), - kind, - })); - } else { - let item = self.stack.pop().unwrap(); - if !self.stack.is_empty() { - item.iter.dir().close(); - } - } + self.name_buffer.truncate(dirname_len); + if !self.name_buffer.is_empty() { + self.name_buffer.push(SEP as OSPathChar); + dirname_len += 1; + } + self.name_buffer.extend_from_slice(base.name.as_slice()); + let cur_len = self.name_buffer.len(); + self.name_buffer.push(0); + + let mut top_idx = top_idx; + if kind == sys::EntryKind::Directory { + let new_dir = sys::open_dir_for_iteration_os_path( + self.stack[top_idx].iter.dir(), + base.name.as_slice(), + )?; + { + self.stack.push(StackItem { + iter: dir_iterator::iterate(new_dir), + dirname_len: cur_len, + }); + top_idx = self.stack.len() - 1; } } + // `name_buffer[cur_len] == 0` was written above; both views end at + // `cur_len` and are NUL-terminated by that sentinel char. `from_buf` + // ties the borrow to `&self.name_buffer` (no raw-pointer reslice). + return Ok(Some(WalkerEntry { + dir: self.stack[top_idx].iter.dir(), + basename: OSPathSliceZ::from_buf( + &self.name_buffer[dirname_len..], + cur_len - dirname_len, + ), + path: OSPathSliceZ::from_buf(&self.name_buffer, cur_len), + kind, + })); } Ok(None) } diff --git a/test/cli/install/bun-pack-dt-unknown.test.ts b/test/cli/install/dt-unknown-readdir.test.ts similarity index 61% rename from test/cli/install/bun-pack-dt-unknown.test.ts rename to test/cli/install/dt-unknown-readdir.test.ts index 541c52ae76fb..9ca3bd1c8ed4 100644 --- a/test/cli/install/bun-pack-dt-unknown.test.ts +++ b/test/cli/install/dt-unknown-readdir.test.ts @@ -1,106 +1,24 @@ // Some filesystems (FUSE, NFS, XFS formatted with ftype=0) do not fill in -// d_type, so every readdir entry comes back as DT_UNKNOWN. `bun pm pack` and -// `bun publish` must still pack those entries. A FUSE mount needs /dev/fuse, so -// instead an LD_PRELOAD shim zeroes d_type in every getdents64 record: bun -// issues getdents64 through libc's syscall(2) wrapper, which the shim -// interposes. The shim announces itself on stderr the first time it rewrites a -// record so the tests can tell that bun's readdir actually went through it -// (if bun ever stops using the libc wrapper, this file needs a real DT_UNKNOWN -// filesystem instead of silently passing). +// d_type, so every readdir entry comes back as DT_UNKNOWN. The package manager +// commands must behave as they do elsewhere; `dtUnknownReaddir` (harness) +// simulates such a filesystem with an LD_PRELOAD shim. import { readTarball } from "bun:internal-for-testing"; -import { afterAll, beforeAll, describe, expect, test } from "bun:test"; -import { bunEnv, bunExe, isLinux, tempDir } from "harness"; -import { symlink, writeFile } from "node:fs/promises"; +import { describe, expect, test } from "bun:test"; +import { bunExe, dtUnknownReaddir, tempDir } from "harness"; +import { readdir, symlink, writeFile } from "node:fs/promises"; import { join } from "node:path"; -const cc = Bun.which("cc") || Bun.which("gcc") || Bun.which("clang"); - -const SHIM_MARKER = "dt-unknown-shim: rewrote getdents64 d_type"; - -const SHIM_C = /* c */ ` -#define _GNU_SOURCE -#include -#include -#include -#include -#include -#include - -static long (*real_syscall)(long, long, long, long, long, long, long); -static int announced; - -long syscall(long number, ...) { - va_list ap; - long a, b, c, d, e, f; - va_start(ap, number); - a = va_arg(ap, long); - b = va_arg(ap, long); - c = va_arg(ap, long); - d = va_arg(ap, long); - e = va_arg(ap, long); - f = va_arg(ap, long); - va_end(ap); - if (!real_syscall) { - real_syscall = (long (*)(long, long, long, long, long, long, long))dlsym(RTLD_NEXT, "syscall"); - } - long rc = real_syscall(number, a, b, c, d, e, f); - if (number != SYS_getdents64 || rc <= 0) return rc; - if (!announced) { - announced = 1; - static const char marker[] = "${SHIM_MARKER}\\n"; - if (write(2, marker, sizeof(marker) - 1) < 0) {} - } - // struct linux_dirent64 { u64 d_ino; s64 d_off; u16 d_reclen; u8 d_type; char d_name[]; } - unsigned char *buf = (unsigned char *)b; - for (long off = 0; off + 19 <= rc;) { - uint16_t reclen; - memcpy(&reclen, buf + off + 16, sizeof(reclen)); - if (reclen == 0) break; - buf[off + 18] = 0; /* DT_UNKNOWN */ - off += reclen; - } - return rc; -} -`; - -let shimDir: ReturnType | undefined; -let shimPath: string; - -beforeAll(async () => { - if (!isLinux || !cc) return; - shimDir = tempDir("dt-unknown-shim", { "shim.c": SHIM_C }); - shimPath = join(String(shimDir), "shim.so"); - await using proc = Bun.spawn({ - cmd: [cc, "-shared", "-fPIC", "-o", shimPath, join(String(shimDir), "shim.c"), "-ldl"], - env: bunEnv, - stdout: "pipe", - stderr: "pipe", - }); - const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); - if (exitCode !== 0) { - throw new Error(`shim compile failed: ${stderr || stdout}`); - } -}); - -afterAll(() => { - shimDir?.[Symbol.dispose](); -}); - -function shimEnv() { - return { ...bunEnv, LD_PRELOAD: bunEnv.LD_PRELOAD ? `${shimPath}:${bunEnv.LD_PRELOAD}` : shimPath }; -} - async function run(cwd: string, ...args: string[]) { await using proc = Bun.spawn({ cmd: [bunExe(), ...args], cwd, - env: shimEnv(), + env: dtUnknownReaddir.env(), stdin: "ignore", stdout: "pipe", stderr: "pipe", }); const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); - expect(stderr).toContain(SHIM_MARKER); + expect(stderr).toContain(dtUnknownReaddir.marker); expect({ stdout, stderr, exitCode }).toMatchObject({ stderr: expect.not.stringContaining("error:"), exitCode: 0 }); } @@ -110,7 +28,7 @@ function packedPaths(tarball: string): string[] { .sort(); } -describe.skipIf(!isLinux || !cc)("pack on a filesystem whose readdir reports DT_UNKNOWN", () => { +describe.skipIf(!dtUnknownReaddir.available)("pack on a filesystem whose readdir reports DT_UNKNOWN", () => { test.concurrent("packs the project tree", async () => { using dir = tempDir("dt-unknown-tree", { "package.json": JSON.stringify({ name: "dt-unknown-tree", version: "1.0.0" }), @@ -234,3 +152,22 @@ describe.skipIf(!isLinux || !cc)("pack on a filesystem whose readdir reports DT_ ]); }); }); + +describe.skipIf(!dtUnknownReaddir.available)("install on a filesystem whose readdir reports DT_UNKNOWN", () => { + // Folder dependencies are installed by walking the folder (walker_skippable + // with resolve_unknown_entry_types), so subdirectories must still be entered. + test.concurrent("installs every file of a folder dependency", async () => { + using dir = tempDir("dt-unknown-install", { + "dep/package.json": JSON.stringify({ name: "dep", version: "1.0.0" }), + "dep/index.js": "", + "dep/lib/a.js": "", + "dep/lib/nested/b.js": "", + "app/package.json": JSON.stringify({ name: "app", dependencies: { dep: "file:../dep" } }), + }); + + await run(join(String(dir), "app"), "install", "--no-summary"); + + const installed = await readdir(join(String(dir), "app", "node_modules", "dep"), { recursive: true }); + expect(installed.sort()).toEqual(["index.js", "lib", "lib/a.js", "lib/nested", "lib/nested/b.js", "package.json"]); + }); +}); diff --git a/test/fixtures/dt-unknown-readdir-shim.c b/test/fixtures/dt-unknown-readdir-shim.c new file mode 100644 index 000000000000..e6a435a315e0 --- /dev/null +++ b/test/fixtures/dt-unknown-readdir-shim.c @@ -0,0 +1,50 @@ +// LD_PRELOAD shim: every getdents64 record comes back with d_type == DT_UNKNOWN, +// the way FUSE, some NFS servers and XFS formatted with ftype=0 report entries. +// bun issues getdents64 through libc's syscall() wrapper, which this interposes. +// Compiled by `dtUnknownReaddir` in test/harness.ts, which defines MARKER: it is +// written to stderr the first time a record is rewritten so a test can tell the +// shim actually saw bun's readdir calls. +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include + +static long (*real_syscall)(long, long, long, long, long, long, long); +static int announced; + +long syscall(long number, ...) { + va_list ap; + long a, b, c, d, e, f; + va_start(ap, number); + a = va_arg(ap, long); + b = va_arg(ap, long); + c = va_arg(ap, long); + d = va_arg(ap, long); + e = va_arg(ap, long); + f = va_arg(ap, long); + va_end(ap); + if (!real_syscall) { + real_syscall = (long (*)(long, long, long, long, long, long, long))dlsym(RTLD_NEXT, "syscall"); + } + long rc = real_syscall(number, a, b, c, d, e, f); + if (number != SYS_getdents64 || rc <= 0) return rc; + if (!announced) { + announced = 1; + static const char marker[] = MARKER "\n"; + if (write(2, marker, sizeof(marker) - 1) < 0) { + } + } + // struct linux_dirent64 { u64 d_ino; s64 d_off; u16 d_reclen; u8 d_type; char d_name[]; } + unsigned char *buf = (unsigned char *)b; + for (long off = 0; off + 19 <= rc;) { + uint16_t reclen; + memcpy(&reclen, buf + off + 16, sizeof(reclen)); + if (reclen == 0) break; + buf[off + 18] = 0; /* DT_UNKNOWN */ + off += reclen; + } + return rc; +} diff --git a/test/harness.ts b/test/harness.ts index b3167919ec84..157dc2ce26f4 100644 --- a/test/harness.ts +++ b/test/harness.ts @@ -2275,6 +2275,29 @@ export function compileFixture(sourcePath: string, options: { flags?: string[] } return outPath; } +/** + * Runs bun as if on a filesystem whose readdir does not report entry types + * (FUSE, some NFS servers, XFS formatted with `ftype=0`), without needing such a + * mount: `env()` preloads a shim that zeroes `d_type` in every `getdents64` + * record. The shim prints `marker` to stderr the first time it does so; assert + * on it, otherwise a test here passes vacuously if bun ever stops issuing + * `getdents64` through libc's `syscall()` wrapper, which is what the shim hooks. + */ +const dtUnknownReaddirMarker = "dt-unknown-readdir-shim: rewrote getdents64 d_type"; +export const dtUnknownReaddir = { + /** Linux with a C compiler; `skipIf(!dtUnknownReaddir.available)`. */ + get available(): boolean { + return isLinux && !!(which("cc") || which("clang") || which("gcc")); + }, + marker: dtUnknownReaddirMarker, + env(): NodeJS.Dict { + const shim = compileFixture(join(import.meta.dir, "fixtures", "dt-unknown-readdir-shim.c"), { + flags: [`-DMARKER="${dtUnknownReaddirMarker}"`, "-ldl"], + }); + return { ...bunEnv, LD_PRELOAD: bunEnv.LD_PRELOAD ? `${shim}:${bunEnv.LD_PRELOAD}` : shim }; + }, +}; + export const rss: () => number = process.platform === "darwin" && typeof Bun.unsafe.memoryFootprint === "function" ? (Bun.unsafe.memoryFootprint as () => number) From 3e3eb934f910d57d5ef3fc442d672f23f4426236 Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Sat, 15 Aug 2026 08:01:31 +0000 Subject: [PATCH 6/8] sys: shorter comments on the resolve flag --- src/sys/lib.rs | 8 +++----- src/sys/walker_skippable.rs | 3 +-- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/src/sys/lib.rs b/src/sys/lib.rs index 8370157d7c79..73ca151e423f 100644 --- a/src/sys/lib.rs +++ b/src/sys/lib.rs @@ -749,11 +749,9 @@ pub mod dir_iterator { #[cfg(not(windows))] name_filter: Option>, state: State, - /// Filesystems that do not fill in `d_type` (FUSE, NFS, XFS with `ftype=0`) - /// report every entry as `Unknown`. When set, `next()` resolves those with - /// `lstat` (a symlink stays a symlink, as with `d_type`); an entry that - /// cannot be stat'ed stays `Unknown`. Off by default: the resolver and glob - /// only want the kind of the few entries they end up using. + /// `lstat` entries whose kind the filesystem did not report (`Unknown`: + /// FUSE, NFS, XFS with `ftype=0`), so that, as with `d_type`, a symlink is + /// still a symlink. Entries that cannot be stat'ed stay `Unknown`. pub resolve_unknown_entry_types: bool, } impl WrappedIterator { diff --git a/src/sys/walker_skippable.rs b/src/sys/walker_skippable.rs index 849c6d5766ad..e40af88c7d88 100644 --- a/src/sys/walker_skippable.rs +++ b/src/sys/walker_skippable.rs @@ -77,8 +77,7 @@ impl Walker { // be invalidated by appending to `self.stack` below. let top_idx = self.stack.len() - 1; let mut dirname_len = self.stack[top_idx].dirname_len; - // Callers set the flag after `walk()` has already created the root - // iterator, so pass it down per call rather than at construction. + // Per call: callers set the flag after `walk()` built the root iterator. self.stack[top_idx].iter.resolve_unknown_entry_types = self.resolve_unknown_entry_types; let Some(base) = self.stack[top_idx].iter.next()? else { let item = self.stack.pop().unwrap(); From 8166de6e539bd6bd6beaf5832af0a6a600bc10c8 Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Sat, 15 Aug 2026 08:08:06 +0000 Subject: [PATCH 7/8] test: compile the DT_UNKNOWN shim in beforeAll, not inside the tests Compiling it synchronously from the first test body blocked the runner for as long as the compiler took (several seconds on a loaded machine), which counted against every concurrently started test and timed them out. --- test/cli/install/dt-unknown-readdir.test.ts | 14 +++++---- test/harness.ts | 32 +++++++++++++++++---- 2 files changed, 36 insertions(+), 10 deletions(-) diff --git a/test/cli/install/dt-unknown-readdir.test.ts b/test/cli/install/dt-unknown-readdir.test.ts index 9ca3bd1c8ed4..fa39bb3b8ea3 100644 --- a/test/cli/install/dt-unknown-readdir.test.ts +++ b/test/cli/install/dt-unknown-readdir.test.ts @@ -3,16 +3,22 @@ // commands must behave as they do elsewhere; `dtUnknownReaddir` (harness) // simulates such a filesystem with an LD_PRELOAD shim. import { readTarball } from "bun:internal-for-testing"; -import { describe, expect, test } from "bun:test"; +import { beforeAll, describe, expect, test } from "bun:test"; import { bunExe, dtUnknownReaddir, tempDir } from "harness"; import { readdir, symlink, writeFile } from "node:fs/promises"; import { join } from "node:path"; +let env: NodeJS.Dict; + +beforeAll(async () => { + if (dtUnknownReaddir.available) env = await dtUnknownReaddir.env(); +}, 30_000); + async function run(cwd: string, ...args: string[]) { await using proc = Bun.spawn({ cmd: [bunExe(), ...args], cwd, - env: dtUnknownReaddir.env(), + env, stdin: "ignore", stdout: "pipe", stderr: "pipe", @@ -118,9 +124,7 @@ describe.skipIf(!dtUnknownReaddir.available)("pack on a filesystem whose readdir }, }); using dir = tempDir("dt-unknown-publish", { - "bunfig.toml": Bun.TOML.stringify({ - install: { cache: false, registry: { url: `http://localhost:${registry.port}`, token: "unused" } }, - }), + "bunfig.toml": `[install]\ncache = false\nregistry = { url = "http://localhost:${registry.port}", token = "unused" }\n`, "package.json": JSON.stringify({ name: "dt-unknown-publish", version: "1.0.0", diff --git a/test/harness.ts b/test/harness.ts index 157dc2ce26f4..c2034a71319c 100644 --- a/test/harness.ts +++ b/test/harness.ts @@ -2275,6 +2275,26 @@ export function compileFixture(sourcePath: string, options: { flags?: string[] } return outPath; } +const dtUnknownReaddirMarker = "dt-unknown-readdir-shim: rewrote getdents64 d_type"; +let dtUnknownReaddirShim: Promise | undefined; + +async function compileDtUnknownReaddirShim(): Promise { + const cc = which("cc") || which("clang") || which("gcc"); + if (!cc) throw new Error("dtUnknownReaddir: no C compiler (cc/clang/gcc) found in $PATH"); + const shim = join(tmpdirSync("dt-unknown-readdir-"), "shim.so"); + const source = join(import.meta.dir, "fixtures", "dt-unknown-readdir-shim.c"); + const proc = Bun.spawn({ + cmd: [cc, "-shared", "-fPIC", "-O2", `-DMARKER="${dtUnknownReaddirMarker}"`, "-o", shim, source, "-ldl"], + env: bunEnv, + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + if (exitCode !== 0) + throw new Error(`dtUnknownReaddir: compiling the shim failed (exit ${exitCode}):\n${stderr || stdout}`); + return shim; +} + /** * Runs bun as if on a filesystem whose readdir does not report entry types * (FUSE, some NFS servers, XFS formatted with `ftype=0`), without needing such a @@ -2283,17 +2303,19 @@ export function compileFixture(sourcePath: string, options: { flags?: string[] } * on it, otherwise a test here passes vacuously if bun ever stops issuing * `getdents64` through libc's `syscall()` wrapper, which is what the shim hooks. */ -const dtUnknownReaddirMarker = "dt-unknown-readdir-shim: rewrote getdents64 d_type"; export const dtUnknownReaddir = { /** Linux with a C compiler; `skipIf(!dtUnknownReaddir.available)`. */ get available(): boolean { return isLinux && !!(which("cc") || which("clang") || which("gcc")); }, marker: dtUnknownReaddirMarker, - env(): NodeJS.Dict { - const shim = compileFixture(join(import.meta.dir, "fixtures", "dt-unknown-readdir-shim.c"), { - flags: [`-DMARKER="${dtUnknownReaddirMarker}"`, "-ldl"], - }); + /** + * Compiles the shim the first time it is called. Call it from `beforeAll` + * (the compiler can take several seconds on a loaded machine) and spawn bun + * with the returned env. + */ + async env(): Promise> { + const shim = await (dtUnknownReaddirShim ??= compileDtUnknownReaddirShim()); return { ...bunEnv, LD_PRELOAD: bunEnv.LD_PRELOAD ? `${shim}:${bunEnv.LD_PRELOAD}` : shim }; }, }; From 2c6552fd1a663a54176bbabb414d428ba7b33752 Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Sat, 15 Aug 2026 13:27:39 +0000 Subject: [PATCH 8/8] Leave walker_skippable and prune to the follow-up sweep The sweep of the remaining readdir consumers (#38961) is where the walker's own resolution policy is being decided, so moving those two onto the iterator flag belongs there; this PR is the flag plus pack and publish. The install test case went with it. --- src/install/prune.rs | 16 +- src/sys/walker_skippable.rs | 202 +++++++++++--------- test/cli/install/dt-unknown-readdir.test.ts | 23 +-- 3 files changed, 129 insertions(+), 112 deletions(-) diff --git a/src/install/prune.rs b/src/install/prune.rs index 488295d72f56..01aa08b4607c 100644 --- a/src/install/prune.rs +++ b/src/install/prune.rs @@ -1343,17 +1343,24 @@ fn lstat_kind(dir: &Dir, name: &[u8]) -> EntryKind { } } +fn entry_kind(dir: &Dir, name: &[u8], kind: EntryKind) -> EntryKind { + if kind != EntryKind::Unknown { + return kind; + } + lstat_kind(dir, name) +} + fn read_entries(dir: &Dir) -> Vec<(Box<[u8]>, EntryKind)> { let mut out = Vec::new(); let mut iter = sys::iterate_dir(dir.fd()); - iter.resolve_unknown_entry_types = true; while let Ok(Some(entry)) = iter.next() { let name = entry.name.slice_u8(); if name.first() == Some(&b'.') { continue; } - if entry.kind == EntryKind::Directory || entry.kind == EntryKind::SymLink { - out.push((name.into(), entry.kind)); + let kind = entry_kind(dir, name, entry.kind); + if kind == EntryKind::Directory || kind == EntryKind::SymLink { + out.push((name.into(), kind)); } } out @@ -1773,10 +1780,9 @@ fn prune_bins(dir: &Dir) { }; let mut dangling: Vec> = Vec::new(); let mut iter = sys::iterate_dir(bin.fd()); - iter.resolve_unknown_entry_types = true; while let Ok(Some(entry)) = iter.next() { let name = entry.name.slice_u8(); - if entry.kind == EntryKind::SymLink && is_dangling(&bin, name) { + if entry_kind(&bin, name, entry.kind) == EntryKind::SymLink && is_dangling(&bin, name) { dangling.push(name.into()); } } diff --git a/src/sys/walker_skippable.rs b/src/sys/walker_skippable.rs index e40af88c7d88..d4d7d854b72f 100644 --- a/src/sys/walker_skippable.rs +++ b/src/sys/walker_skippable.rs @@ -26,7 +26,6 @@ pub struct Walker { skip_dirnames: Range, skip_all: Box<[u64]>, seed: u64, - /// See `dir_iterator::WrappedIterator::resolve_unknown_entry_types`. pub resolve_unknown_entry_types: bool, } @@ -77,98 +76,127 @@ impl Walker { // be invalidated by appending to `self.stack` below. let top_idx = self.stack.len() - 1; let mut dirname_len = self.stack[top_idx].dirname_len; - // Per call: callers set the flag after `walk()` built the root iterator. - self.stack[top_idx].iter.resolve_unknown_entry_types = self.resolve_unknown_entry_types; - let Some(base) = self.stack[top_idx].iter.next()? else { - let item = self.stack.pop().unwrap(); - if !self.stack.is_empty() { - item.iter.dir().close(); - } - continue; - }; - let kind = base.kind; - - match kind { - sys::EntryKind::Directory => { - let skip = &self.skip_all[self.skip_dirnames.clone()]; - if skip.contains( - // avoid hashing if there will be 0 results - &(if !skip.is_empty() { - hash_with_seed(self.seed, slice_as_bytes(base.name.as_slice())) - } else { - 0 - }), - ) { - continue; - } - } - sys::EntryKind::File => { - let skip = &self.skip_all[self.skip_filenames.clone()]; - if skip.contains( - // avoid hashing if there will be 0 results - &(if !skip.is_empty() { - hash_with_seed(self.seed, slice_as_bytes(base.name.as_slice())) + match self.stack[top_idx].iter.next() { + Err(err) => return Err(err), + Ok(res) => { + if let Some(base) = res { + // Some filesystems (NFS, FUSE, bind mounts) don't provide + // d_type and return DT_UNKNOWN. Optionally resolve via + // fstatat so callers get accurate types for recursion. + // This only affects POSIX; Windows always provides types. + #[cfg(not(windows))] + let kind: sys::EntryKind = if base.kind == sys::EntryKind::Unknown + && self.resolve_unknown_entry_types + { + let dir_fd = self.stack[top_idx].iter.dir(); + match sys::lstatat(dir_fd, base.name.as_zstr()) { + Ok(stat_buf) => sys::kind_from_mode(stat_buf.st_mode as sys::Mode), + Err(_) => continue, // skip entries we can't stat + } } else { - 0 - }), - ) { - continue; - } - } + base.kind + }; + #[cfg(windows)] + let kind: sys::EntryKind = base.kind; - // we don't know what it is for a symlink - sys::EntryKind::SymLink => { - let skip = &self.skip_all[..]; - if skip.contains( - // avoid hashing if there will be 0 results - &(if !skip.is_empty() { - hash_with_seed(self.seed, slice_as_bytes(base.name.as_slice())) - } else { - 0 - }), - ) { - continue; - } - } + match kind { + sys::EntryKind::Directory => { + let skip = &self.skip_all[self.skip_dirnames.clone()]; + if skip.contains( + // avoid hashing if there will be 0 results + &(if !skip.is_empty() { + hash_with_seed( + self.seed, + slice_as_bytes(base.name.as_slice()), + ) + } else { + 0 + }), + ) { + continue; + } + } + sys::EntryKind::File => { + let skip = &self.skip_all[self.skip_filenames.clone()]; + if skip.contains( + // avoid hashing if there will be 0 results + &(if !skip.is_empty() { + hash_with_seed( + self.seed, + slice_as_bytes(base.name.as_slice()), + ) + } else { + 0 + }), + ) { + continue; + } + } - _ => {} - } + // we don't know what it is for a symlink + sys::EntryKind::SymLink => { + let skip = &self.skip_all[..]; + if skip.contains( + // avoid hashing if there will be 0 results + &(if !skip.is_empty() { + hash_with_seed( + self.seed, + slice_as_bytes(base.name.as_slice()), + ) + } else { + 0 + }), + ) { + continue; + } + } - self.name_buffer.truncate(dirname_len); - if !self.name_buffer.is_empty() { - self.name_buffer.push(SEP as OSPathChar); - dirname_len += 1; - } - self.name_buffer.extend_from_slice(base.name.as_slice()); - let cur_len = self.name_buffer.len(); - self.name_buffer.push(0); - - let mut top_idx = top_idx; - if kind == sys::EntryKind::Directory { - let new_dir = sys::open_dir_for_iteration_os_path( - self.stack[top_idx].iter.dir(), - base.name.as_slice(), - )?; - { - self.stack.push(StackItem { - iter: dir_iterator::iterate(new_dir), - dirname_len: cur_len, - }); - top_idx = self.stack.len() - 1; + _ => {} + } + + self.name_buffer.truncate(dirname_len); + if !self.name_buffer.is_empty() { + self.name_buffer.push(SEP as OSPathChar); + dirname_len += 1; + } + self.name_buffer.extend_from_slice(base.name.as_slice()); + let cur_len = self.name_buffer.len(); + self.name_buffer.push(0); + + let mut top_idx = top_idx; + if kind == sys::EntryKind::Directory { + let new_dir = sys::open_dir_for_iteration_os_path( + self.stack[top_idx].iter.dir(), + base.name.as_slice(), + )?; + { + self.stack.push(StackItem { + iter: dir_iterator::iterate(new_dir), + dirname_len: cur_len, + }); + top_idx = self.stack.len() - 1; + } + } + // `name_buffer[cur_len] == 0` was written above; both views end at + // `cur_len` and are NUL-terminated by that sentinel char. `from_buf` + // ties the borrow to `&self.name_buffer` (no raw-pointer reslice). + return Ok(Some(WalkerEntry { + dir: self.stack[top_idx].iter.dir(), + basename: OSPathSliceZ::from_buf( + &self.name_buffer[dirname_len..], + cur_len - dirname_len, + ), + path: OSPathSliceZ::from_buf(&self.name_buffer, cur_len), + kind, + })); + } else { + let item = self.stack.pop().unwrap(); + if !self.stack.is_empty() { + item.iter.dir().close(); + } + } } } - // `name_buffer[cur_len] == 0` was written above; both views end at - // `cur_len` and are NUL-terminated by that sentinel char. `from_buf` - // ties the borrow to `&self.name_buffer` (no raw-pointer reslice). - return Ok(Some(WalkerEntry { - dir: self.stack[top_idx].iter.dir(), - basename: OSPathSliceZ::from_buf( - &self.name_buffer[dirname_len..], - cur_len - dirname_len, - ), - path: OSPathSliceZ::from_buf(&self.name_buffer, cur_len), - kind, - })); } Ok(None) } diff --git a/test/cli/install/dt-unknown-readdir.test.ts b/test/cli/install/dt-unknown-readdir.test.ts index fa39bb3b8ea3..56fa2d784240 100644 --- a/test/cli/install/dt-unknown-readdir.test.ts +++ b/test/cli/install/dt-unknown-readdir.test.ts @@ -5,11 +5,13 @@ import { readTarball } from "bun:internal-for-testing"; import { beforeAll, describe, expect, test } from "bun:test"; import { bunExe, dtUnknownReaddir, tempDir } from "harness"; -import { readdir, symlink, writeFile } from "node:fs/promises"; +import { symlink, writeFile } from "node:fs/promises"; import { join } from "node:path"; let env: NodeJS.Dict; +// Compiles the shim; a C compiler on a busy CI machine can take longer than the +// default hook timeout. beforeAll(async () => { if (dtUnknownReaddir.available) env = await dtUnknownReaddir.env(); }, 30_000); @@ -156,22 +158,3 @@ describe.skipIf(!dtUnknownReaddir.available)("pack on a filesystem whose readdir ]); }); }); - -describe.skipIf(!dtUnknownReaddir.available)("install on a filesystem whose readdir reports DT_UNKNOWN", () => { - // Folder dependencies are installed by walking the folder (walker_skippable - // with resolve_unknown_entry_types), so subdirectories must still be entered. - test.concurrent("installs every file of a folder dependency", async () => { - using dir = tempDir("dt-unknown-install", { - "dep/package.json": JSON.stringify({ name: "dep", version: "1.0.0" }), - "dep/index.js": "", - "dep/lib/a.js": "", - "dep/lib/nested/b.js": "", - "app/package.json": JSON.stringify({ name: "app", dependencies: { dep: "file:../dep" } }), - }); - - await run(join(String(dir), "app"), "install", "--no-summary"); - - const installed = await readdir(join(String(dir), "app", "node_modules", "dep"), { recursive: true }); - expect(installed.sort()).toEqual(["index.js", "lib", "lib/a.js", "lib/nested", "lib/nested/b.js", "package.json"]); - }); -});