pack, publish: include entries on filesystems whose readdir reports DT_UNKNOWN

src/runtime/cli/pack_command.rs

@@ -476,6 +476,7 @@ fn iterate_included_project_tree(
         });
 
         let mut dir_iter = DirIterator::iterate(Fd::from_std_dir(&dir));
+        dir_iter.resolve_unknown_entry_types = true;
         'next_entry: while let Some(entry) = dir_iter.next().ok().flatten() {
             // On iterator error, treat as end of iteration.
             if entry.kind != bun_sys::FileKind::File && entry.kind != bun_sys::FileKind::Directory {
@@ -713,6 +714,7 @@ fn add_entire_tree(
         }
 
         let mut iter = DirIterator::iterate(Fd::from_std_dir(&dir));
+        iter.resolve_unknown_entry_types = true;
         'next_entry: while let Some(entry) = iter.next().ok().flatten() {
             if entry.kind != bun_sys::FileKind::File && entry.kind != bun_sys::FileKind::Directory {
                 continue;
@@ -884,6 +886,7 @@ fn iterate_bundled_deps(
     let mut additional_bundled_deps: Vec<DirInfo> = Vec::new();
 
     let mut iter = DirIterator::iterate(Fd::from_std_dir(&dir));
+    iter.resolve_unknown_entry_types = true;
     while let Some(entry) = iter.next().ok().flatten() {
         if entry.kind != bun_sys::FileKind::Directory {
             continue;
@@ -1022,6 +1025,7 @@ fn add_bundled_dep(
         let DirInfo(dir, dir_subpath, dir_depth) = dir_info;
 
         let mut iter = DirIterator::iterate(Fd::from_std_dir(&dir));
+        iter.resolve_unknown_entry_types = true;
         while let Some(entry) = iter.next().ok().flatten() {
             if entry.kind != bun_sys::FileKind::File && entry.kind != bun_sys::FileKind::Directory {
                 continue;
@@ -1284,6 +1288,7 @@ fn iterate_project_tree(
         }
 
         let mut dir_iter = DirIterator::iterate(Fd::from_std_dir(&dir));
+        dir_iter.resolve_unknown_entry_types = true;
         'next_entry: while let Some(entry) = dir_iter.next().ok().flatten() {
             if entry.kind != bun_sys::FileKind::File && entry.kind != bun_sys::FileKind::Directory {
                 continue;

src/runtime/cli/publish_command.rs

@@ -1588,6 +1588,7 @@ impl PublishCommand {
         });
 
         let mut iter = DirIterator::iterate(workspace_dir);
+        iter.resolve_unknown_entry_types = true;
         while let Some(entry) = iter.next().ok().flatten() {
             if entry.kind == bun_sys::EntryKind::Directory {
                 continue;
@@ -1810,6 +1811,7 @@ impl PublishCommand {
                     });
 
                     let mut iter = DirIterator::iterate(dir);
+                    iter.resolve_unknown_entry_types = true;
                     while let Some(entry) = iter.next().ok().flatten() {
                         let (name, subpath): (&'static ZStr, &'static ZStr) = {
                             // Entry names are UTF-8 on every platform.

src/sys/lib.rs

@@ -122,6 +122,25 @@ pub mod dir_iterator {
         pub kind: EntryKind,
     }
 
+    impl IteratorResult {
+        /// See `WrappedIterator::resolve_unknown_entry_types`.
+        fn resolve_unknown_kind(&mut self, dir: Fd) {
+            #[cfg(not(windows))]
+            {
+                if self.kind == EntryKind::Unknown {
+                    if let Ok(stat) = super::lstatat(dir, self.name.as_zstr()) {
+                        self.kind = super::kind_from_mode(stat.st_mode as super::Mode);
+                    }
+                }
+            }
+            #[cfg(windows)]
+            {
+                // The Windows iterator always knows the kind.
+                let _ = dir;
+            }
+        }
+    }
+
     /// Length-known, NUL-terminated entry name in OS-native encoding.
     ///
     /// **POSIX**: lifetime-erased borrow (raw pointer + length) into the
@@ -265,8 +284,7 @@ pub mod dir_iterator {
             // literal matches <sys/dirent.h>.
             #[cfg(any(target_os = "macos", target_os = "freebsd"))]
             14 /* DT_WHT */ => EntryKind::Whiteout,
-            // DT_UNKNOWN: some filesystems (bind mounts, FUSE, NFS) don't
-            // provide d_type. Callers should lstatat() to resolve when needed.
+            // DT_UNKNOWN: see `WrappedIterator::resolve_unknown_entry_types`.
             _ => EntryKind::Unknown,
         }
     }
@@ -731,6 +749,10 @@ pub mod dir_iterator {
         #[cfg(not(windows))]
         name_filter: Option<Vec<u16>>,
         state: State,
+        /// `lstat` entries whose kind the filesystem did not report (`Unknown`:
+        /// FUSE, NFS, XFS with `ftype=0`), so that, as with `d_type`, a symlink is
+        /// still a symlink. Entries that cannot be stat'ed stay `Unknown`.
+        pub resolve_unknown_entry_types: bool,
     }
     impl WrappedIterator {
         #[inline]
@@ -759,7 +781,13 @@ pub mod dir_iterator {
         /// Copy it out before pushing the iterator into a `Vec` etc.
         #[inline]
         pub fn next(&mut self) -> Result<Option<IteratorResult>> {
-            self.state.next(self.dir)
+            let mut entry = self.state.next(self.dir)?;
+            if self.resolve_unknown_entry_types {
+                if let Some(entry) = entry.as_mut() {
+                    entry.resolve_unknown_kind(self.dir);
+                }
+            }
+            Ok(entry)
         }
     }
 
@@ -770,13 +798,15 @@ pub mod dir_iterator {
                 dir,
                 name_filter: None,
                 state: State::new(),
+                resolve_unknown_entry_types: false,
             }
         }
         #[cfg(windows)]
         {
             WrappedIterator {
                 dir,
                 state: State::new(),
+                resolve_unknown_entry_types: false,
             }
         }
     }

test/cli/install/dt-unknown-readdir.test.ts

@@ -0,0 +1,160 @@
+// Some filesystems (FUSE, NFS, XFS formatted with ftype=0) do not fill in
+// d_type, so every readdir entry comes back as DT_UNKNOWN. The package manager
+// commands must behave as they do elsewhere; `dtUnknownReaddir` (harness)
+// simulates such a filesystem with an LD_PRELOAD shim.
+import { readTarball } from "bun:internal-for-testing";
+import { beforeAll, describe, expect, test } from "bun:test";
+import { bunExe, dtUnknownReaddir, tempDir } from "harness";
+import { symlink, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+
+let env: NodeJS.Dict<string>;
+
+// Compiles the shim; a C compiler on a busy CI machine can take longer than the
+// default hook timeout.
+beforeAll(async () => {
+  if (dtUnknownReaddir.available) env = await dtUnknownReaddir.env();
+}, 30_000);
+
+async function run(cwd: string, ...args: string[]) {
+  await using proc = Bun.spawn({
+    cmd: [bunExe(), ...args],
+    cwd,
+    env,
+    stdin: "ignore",
+    stdout: "pipe",
+    stderr: "pipe",
+  });
+  const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]);
+  expect(stderr).toContain(dtUnknownReaddir.marker);
+  expect({ stdout, stderr, exitCode }).toMatchObject({ stderr: expect.not.stringContaining("error:"), exitCode: 0 });
+}
+
+function packedPaths(tarball: string): string[] {
+  return readTarball(tarball)
+    .entries.map((entry: { pathname: string }) => entry.pathname)
+    .sort();
+}
+
+describe.skipIf(!dtUnknownReaddir.available)("pack on a filesystem whose readdir reports DT_UNKNOWN", () => {
+  test.concurrent("packs the project tree", async () => {
+    using dir = tempDir("dt-unknown-tree", {
+      "package.json": JSON.stringify({ name: "dt-unknown-tree", version: "1.0.0" }),
+      "index.js": "",
+      "lib/a.js": "",
+      "lib/nested/b.js": "",
+      // `out/` only ignores directories, so it needs the entry's kind: the
+      // `out` directory is ignored, the `lib/out` file is not.
+      ".npmignore": "out/\n",
+      "out/c.js": "",
+      "lib/out": "",
+    });
+    // Symlinks are never packed; resolving the kind with lstat has to keep that.
+    await symlink("index.js", join(String(dir), "link.js"));
+
+    await run(String(dir), "pm", "pack");
+
+    expect(packedPaths(join(String(dir), "dt-unknown-tree-1.0.0.tgz"))).toEqual([
+      "package/index.js",
+      "package/lib/a.js",
+      "package/lib/nested/b.js",
+      "package/lib/out",
+      "package/package.json",
+    ]);
+  });
+
+  test.concurrent('packs what "files" selects', async () => {
+    using dir = tempDir("dt-unknown-files", {
+      "package.json": JSON.stringify({
+        name: "dt-unknown-files",
+        version: "1.0.0",
+        files: ["index.js", "lib", "!lib/internal/"],
+      }),
+      "index.js": "",
+      "excluded.js": "",
+      "lib/a.js": "",
+      "lib/nested/b.js": "",
+      "lib/internal/c.js": "",
+    });
+    await symlink("a.js", join(String(dir), "lib", "link.js"));
+
+    await run(String(dir), "pm", "pack");
+
+    expect(packedPaths(join(String(dir), "dt-unknown-files-1.0.0.tgz"))).toEqual([
+      "package/index.js",
+      "package/lib/a.js",
+      "package/lib/nested/b.js",
+      "package/package.json",
+    ]);
+  });
+
+  test.concurrent("packs bundledDependencies", async () => {
+    using dir = tempDir("dt-unknown-bundled", {
+      "package.json": JSON.stringify({
+        name: "dt-unknown-bundled",
+        version: "1.0.0",
+        dependencies: { "dep": "1.0.0", "@scope/dep": "1.0.0", "not-bundled": "1.0.0" },
+        bundledDependencies: ["dep", "@scope/dep"],
+      }),
+      "index.js": "",
+      "node_modules/dep/package.json": JSON.stringify({ name: "dep", version: "1.0.0" }),
+      "node_modules/dep/lib/index.js": "",
+      "node_modules/@scope/dep/package.json": JSON.stringify({ name: "@scope/dep", version: "1.0.0" }),
+      "node_modules/@scope/dep/index.js": "",
+      "node_modules/not-bundled/package.json": JSON.stringify({ name: "not-bundled", version: "1.0.0" }),
+    });
+
+    await run(String(dir), "pm", "pack");
+
+    expect(packedPaths(join(String(dir), "dt-unknown-bundled-1.0.0.tgz"))).toEqual([
+      "package/index.js",
+      "package/node_modules/@scope/dep/index.js",
+      "package/node_modules/@scope/dep/package.json",
+      "package/node_modules/dep/lib/index.js",
+      "package/node_modules/dep/package.json",
+      "package/package.json",
+    ]);
+  });
+
+  test.concurrent('publish packs the tree, walks "directories.bin" and finds the readme', async () => {
+    let captured: any;
+    using registry = Bun.serve({
+      port: 0,
+      async fetch(req) {
+        if (req.method === "PUT") captured = await req.json();
+        return new Response("OK");
+      },
+    });
+    using dir = tempDir("dt-unknown-publish", {
+      "bunfig.toml": `[install]\ncache = false\nregistry = { url = "http://localhost:${registry.port}", token = "unused" }\n`,
+      "package.json": JSON.stringify({
+        name: "dt-unknown-publish",
+        version: "1.0.0",
+        directories: { bin: "bins" },
+      }),
+      "README.md": "# dt-unknown-publish",
+      "index.js": "",
+      "bins/a.js": "",
+      "bins/more/b.js": "",
+    });
+
+    await run(String(dir), "publish");
+
+    expect(captured.versions["1.0.0"]).toMatchObject({
+      bin: { "a.js": "bins/a.js", "more": "bins/more", "b.js": "bins/more/b.js" },
+      readme: "# dt-unknown-publish",
+      readmeFilename: "README.md",
+    });
+
+    const attachment: { data: string } = Object.values(captured._attachments)[0] as any;
+    const tarball = join(String(dir), "published.tgz");
+    await writeFile(tarball, Buffer.from(attachment.data, "base64"));
+    expect(packedPaths(tarball)).toEqual([
+      "package/README.md",
+      "package/bins/a.js",
+      "package/bins/more/b.js",
+      "package/index.js",
+      "package/package.json",
+    ]);
+  });
+});

test/fixtures/dt-unknown-readdir-shim.c

@@ -0,0 +1,50 @@
+// LD_PRELOAD shim: every getdents64 record comes back with d_type == DT_UNKNOWN,
+// the way FUSE, some NFS servers and XFS formatted with ftype=0 report entries.
+// bun issues getdents64 through libc's syscall() wrapper, which this interposes.
+// Compiled by `dtUnknownReaddir` in test/harness.ts, which defines MARKER: it is
+// written to stderr the first time a record is rewritten so a test can tell the
+// shim actually saw bun's readdir calls.
+#define _GNU_SOURCE
+#include <dlfcn.h>
+#include <stdarg.h>
+#include <stdint.h>
+#include <string.h>
+#include <sys/syscall.h>
+#include <unistd.h>
+
+static long (*real_syscall)(long, long, long, long, long, long, long);
+static int announced;
+
+long syscall(long number, ...) {
+  va_list ap;
+  long a, b, c, d, e, f;
+  va_start(ap, number);
+  a = va_arg(ap, long);
+  b = va_arg(ap, long);
+  c = va_arg(ap, long);
+  d = va_arg(ap, long);
+  e = va_arg(ap, long);
+  f = va_arg(ap, long);
+  va_end(ap);
+  if (!real_syscall) {
+    real_syscall = (long (*)(long, long, long, long, long, long, long))dlsym(RTLD_NEXT, "syscall");
+  }
+  long rc = real_syscall(number, a, b, c, d, e, f);
+  if (number != SYS_getdents64 || rc <= 0) return rc;
+  if (!announced) {
+    announced = 1;
+    static const char marker[] = MARKER "\n";
+    if (write(2, marker, sizeof(marker) - 1) < 0) {
+    }
+  }
+  // struct linux_dirent64 { u64 d_ino; s64 d_off; u16 d_reclen; u8 d_type; char d_name[]; }
+  unsigned char *buf = (unsigned char *)b;
+  for (long off = 0; off + 19 <= rc;) {
+    uint16_t reclen;
+    memcpy(&reclen, buf + off + 16, sizeof(reclen));
+    if (reclen == 0) break;
+    buf[off + 18] = 0; /* DT_UNKNOWN */
+    off += reclen;
+  }
+  return rc;
+}

test/harness.ts

@@ -2275,6 +2275,51 @@ export function compileFixture(sourcePath: string, options: { flags?: string[] }
   return outPath;
 }
 
+const dtUnknownReaddirMarker = "dt-unknown-readdir-shim: rewrote getdents64 d_type";
+let dtUnknownReaddirShim: Promise<string> | undefined;
+
+async function compileDtUnknownReaddirShim(): Promise<string> {
+  const cc = which("cc") || which("clang") || which("gcc");
+  if (!cc) throw new Error("dtUnknownReaddir: no C compiler (cc/clang/gcc) found in $PATH");
+  const shim = join(tmpdirSync("dt-unknown-readdir-"), "shim.so");
+  const source = join(import.meta.dir, "fixtures", "dt-unknown-readdir-shim.c");
+  const proc = Bun.spawn({
+    cmd: [cc, "-shared", "-fPIC", "-O2", `-DMARKER="${dtUnknownReaddirMarker}"`, "-o", shim, source, "-ldl"],
+    env: bunEnv,
+    stdout: "pipe",
+    stderr: "pipe",
+  });
+  const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]);
+  if (exitCode !== 0)
+    throw new Error(`dtUnknownReaddir: compiling the shim failed (exit ${exitCode}):\n${stderr || stdout}`);
+  return shim;
+}
+
+/**
+ * Runs bun as if on a filesystem whose readdir does not report entry types
+ * (FUSE, some NFS servers, XFS formatted with `ftype=0`), without needing such a
+ * mount: `env()` preloads a shim that zeroes `d_type` in every `getdents64`
+ * record. The shim prints `marker` to stderr the first time it does so; assert
+ * on it, otherwise a test here passes vacuously if bun ever stops issuing
+ * `getdents64` through libc's `syscall()` wrapper, which is what the shim hooks.
+ */
+export const dtUnknownReaddir = {
+  /** Linux with a C compiler; `skipIf(!dtUnknownReaddir.available)`. */
+  get available(): boolean {
+    return isLinux && !!(which("cc") || which("clang") || which("gcc"));
+  },
+  marker: dtUnknownReaddirMarker,
+  /**
+   * Compiles the shim the first time it is called. Call it from `beforeAll`
+   * (the compiler can take several seconds on a loaded machine) and spawn bun
+   * with the returned env.
+   */
+  async env(): Promise<NodeJS.Dict<string>> {
+    const shim = await (dtUnknownReaddirShim ??= compileDtUnknownReaddirShim());
+    return { ...bunEnv, LD_PRELOAD: bunEnv.LD_PRELOAD ? `${shim}:${bunEnv.LD_PRELOAD}` : shim };
+  },
+};
+
 export const rss: () => number =
   process.platform === "darwin" && typeof Bun.unsafe.memoryFootprint === "function"
     ? (Bun.unsafe.memoryFootprint as () => number)