diff --git a/src/runtime/cli/pack_command.rs b/src/runtime/cli/pack_command.rs index 4b767be83280..b0e02568b97e 100644 --- a/src/runtime/cli/pack_command.rs +++ b/src/runtime/cli/pack_command.rs @@ -476,6 +476,7 @@ fn iterate_included_project_tree( }); let mut dir_iter = DirIterator::iterate(Fd::from_std_dir(&dir)); + dir_iter.resolve_unknown_entry_types = true; 'next_entry: while let Some(entry) = dir_iter.next().ok().flatten() { // On iterator error, treat as end of iteration. if entry.kind != bun_sys::FileKind::File && entry.kind != bun_sys::FileKind::Directory { @@ -713,6 +714,7 @@ fn add_entire_tree( } let mut iter = DirIterator::iterate(Fd::from_std_dir(&dir)); + iter.resolve_unknown_entry_types = true; 'next_entry: while let Some(entry) = iter.next().ok().flatten() { if entry.kind != bun_sys::FileKind::File && entry.kind != bun_sys::FileKind::Directory { continue; @@ -884,6 +886,7 @@ fn iterate_bundled_deps( let mut additional_bundled_deps: Vec = Vec::new(); let mut iter = DirIterator::iterate(Fd::from_std_dir(&dir)); + iter.resolve_unknown_entry_types = true; while let Some(entry) = iter.next().ok().flatten() { if entry.kind != bun_sys::FileKind::Directory { continue; @@ -1022,6 +1025,7 @@ fn add_bundled_dep( let DirInfo(dir, dir_subpath, dir_depth) = dir_info; let mut iter = DirIterator::iterate(Fd::from_std_dir(&dir)); + iter.resolve_unknown_entry_types = true; while let Some(entry) = iter.next().ok().flatten() { if entry.kind != bun_sys::FileKind::File && entry.kind != bun_sys::FileKind::Directory { continue; @@ -1284,6 +1288,7 @@ fn iterate_project_tree( } let mut dir_iter = DirIterator::iterate(Fd::from_std_dir(&dir)); + dir_iter.resolve_unknown_entry_types = true; 'next_entry: while let Some(entry) = dir_iter.next().ok().flatten() { if entry.kind != bun_sys::FileKind::File && entry.kind != bun_sys::FileKind::Directory { continue; diff --git a/src/runtime/cli/publish_command.rs b/src/runtime/cli/publish_command.rs index ac6e1736eb84..b8c8dab974f6 100644 --- a/src/runtime/cli/publish_command.rs +++ b/src/runtime/cli/publish_command.rs @@ -1588,6 +1588,7 @@ impl PublishCommand { }); let mut iter = DirIterator::iterate(workspace_dir); + iter.resolve_unknown_entry_types = true; while let Some(entry) = iter.next().ok().flatten() { if entry.kind == bun_sys::EntryKind::Directory { continue; @@ -1810,6 +1811,7 @@ impl PublishCommand { }); let mut iter = DirIterator::iterate(dir); + iter.resolve_unknown_entry_types = true; while let Some(entry) = iter.next().ok().flatten() { let (name, subpath): (&'static ZStr, &'static ZStr) = { // Entry names are UTF-8 on every platform. diff --git a/src/sys/lib.rs b/src/sys/lib.rs index e563f9bb54b8..73ca151e423f 100644 --- a/src/sys/lib.rs +++ b/src/sys/lib.rs @@ -122,6 +122,25 @@ pub mod dir_iterator { pub kind: EntryKind, } + impl IteratorResult { + /// See `WrappedIterator::resolve_unknown_entry_types`. + fn resolve_unknown_kind(&mut self, dir: Fd) { + #[cfg(not(windows))] + { + if self.kind == EntryKind::Unknown { + if let Ok(stat) = super::lstatat(dir, self.name.as_zstr()) { + self.kind = super::kind_from_mode(stat.st_mode as super::Mode); + } + } + } + #[cfg(windows)] + { + // The Windows iterator always knows the kind. + let _ = dir; + } + } + } + /// Length-known, NUL-terminated entry name in OS-native encoding. /// /// **POSIX**: lifetime-erased borrow (raw pointer + length) into the @@ -265,8 +284,7 @@ pub mod dir_iterator { // literal matches . #[cfg(any(target_os = "macos", target_os = "freebsd"))] 14 /* DT_WHT */ => EntryKind::Whiteout, - // DT_UNKNOWN: some filesystems (bind mounts, FUSE, NFS) don't - // provide d_type. Callers should lstatat() to resolve when needed. + // DT_UNKNOWN: see `WrappedIterator::resolve_unknown_entry_types`. _ => EntryKind::Unknown, } } @@ -731,6 +749,10 @@ pub mod dir_iterator { #[cfg(not(windows))] name_filter: Option>, state: State, + /// `lstat` entries whose kind the filesystem did not report (`Unknown`: + /// FUSE, NFS, XFS with `ftype=0`), so that, as with `d_type`, a symlink is + /// still a symlink. Entries that cannot be stat'ed stay `Unknown`. + pub resolve_unknown_entry_types: bool, } impl WrappedIterator { #[inline] @@ -759,7 +781,13 @@ pub mod dir_iterator { /// Copy it out before pushing the iterator into a `Vec` etc. #[inline] pub fn next(&mut self) -> Result> { - self.state.next(self.dir) + let mut entry = self.state.next(self.dir)?; + if self.resolve_unknown_entry_types { + if let Some(entry) = entry.as_mut() { + entry.resolve_unknown_kind(self.dir); + } + } + Ok(entry) } } @@ -770,6 +798,7 @@ pub mod dir_iterator { dir, name_filter: None, state: State::new(), + resolve_unknown_entry_types: false, } } #[cfg(windows)] @@ -777,6 +806,7 @@ pub mod dir_iterator { WrappedIterator { dir, state: State::new(), + resolve_unknown_entry_types: false, } } } diff --git a/test/cli/install/dt-unknown-readdir.test.ts b/test/cli/install/dt-unknown-readdir.test.ts new file mode 100644 index 000000000000..56fa2d784240 --- /dev/null +++ b/test/cli/install/dt-unknown-readdir.test.ts @@ -0,0 +1,160 @@ +// Some filesystems (FUSE, NFS, XFS formatted with ftype=0) do not fill in +// d_type, so every readdir entry comes back as DT_UNKNOWN. The package manager +// commands must behave as they do elsewhere; `dtUnknownReaddir` (harness) +// simulates such a filesystem with an LD_PRELOAD shim. +import { readTarball } from "bun:internal-for-testing"; +import { beforeAll, describe, expect, test } from "bun:test"; +import { bunExe, dtUnknownReaddir, tempDir } from "harness"; +import { symlink, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +let env: NodeJS.Dict; + +// Compiles the shim; a C compiler on a busy CI machine can take longer than the +// default hook timeout. +beforeAll(async () => { + if (dtUnknownReaddir.available) env = await dtUnknownReaddir.env(); +}, 30_000); + +async function run(cwd: string, ...args: string[]) { + await using proc = Bun.spawn({ + cmd: [bunExe(), ...args], + cwd, + env, + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + expect(stderr).toContain(dtUnknownReaddir.marker); + expect({ stdout, stderr, exitCode }).toMatchObject({ stderr: expect.not.stringContaining("error:"), exitCode: 0 }); +} + +function packedPaths(tarball: string): string[] { + return readTarball(tarball) + .entries.map((entry: { pathname: string }) => entry.pathname) + .sort(); +} + +describe.skipIf(!dtUnknownReaddir.available)("pack on a filesystem whose readdir reports DT_UNKNOWN", () => { + test.concurrent("packs the project tree", async () => { + using dir = tempDir("dt-unknown-tree", { + "package.json": JSON.stringify({ name: "dt-unknown-tree", version: "1.0.0" }), + "index.js": "", + "lib/a.js": "", + "lib/nested/b.js": "", + // `out/` only ignores directories, so it needs the entry's kind: the + // `out` directory is ignored, the `lib/out` file is not. + ".npmignore": "out/\n", + "out/c.js": "", + "lib/out": "", + }); + // Symlinks are never packed; resolving the kind with lstat has to keep that. + await symlink("index.js", join(String(dir), "link.js")); + + await run(String(dir), "pm", "pack"); + + expect(packedPaths(join(String(dir), "dt-unknown-tree-1.0.0.tgz"))).toEqual([ + "package/index.js", + "package/lib/a.js", + "package/lib/nested/b.js", + "package/lib/out", + "package/package.json", + ]); + }); + + test.concurrent('packs what "files" selects', async () => { + using dir = tempDir("dt-unknown-files", { + "package.json": JSON.stringify({ + name: "dt-unknown-files", + version: "1.0.0", + files: ["index.js", "lib", "!lib/internal/"], + }), + "index.js": "", + "excluded.js": "", + "lib/a.js": "", + "lib/nested/b.js": "", + "lib/internal/c.js": "", + }); + await symlink("a.js", join(String(dir), "lib", "link.js")); + + await run(String(dir), "pm", "pack"); + + expect(packedPaths(join(String(dir), "dt-unknown-files-1.0.0.tgz"))).toEqual([ + "package/index.js", + "package/lib/a.js", + "package/lib/nested/b.js", + "package/package.json", + ]); + }); + + test.concurrent("packs bundledDependencies", async () => { + using dir = tempDir("dt-unknown-bundled", { + "package.json": JSON.stringify({ + name: "dt-unknown-bundled", + version: "1.0.0", + dependencies: { "dep": "1.0.0", "@scope/dep": "1.0.0", "not-bundled": "1.0.0" }, + bundledDependencies: ["dep", "@scope/dep"], + }), + "index.js": "", + "node_modules/dep/package.json": JSON.stringify({ name: "dep", version: "1.0.0" }), + "node_modules/dep/lib/index.js": "", + "node_modules/@scope/dep/package.json": JSON.stringify({ name: "@scope/dep", version: "1.0.0" }), + "node_modules/@scope/dep/index.js": "", + "node_modules/not-bundled/package.json": JSON.stringify({ name: "not-bundled", version: "1.0.0" }), + }); + + await run(String(dir), "pm", "pack"); + + expect(packedPaths(join(String(dir), "dt-unknown-bundled-1.0.0.tgz"))).toEqual([ + "package/index.js", + "package/node_modules/@scope/dep/index.js", + "package/node_modules/@scope/dep/package.json", + "package/node_modules/dep/lib/index.js", + "package/node_modules/dep/package.json", + "package/package.json", + ]); + }); + + test.concurrent('publish packs the tree, walks "directories.bin" and finds the readme', async () => { + let captured: any; + using registry = Bun.serve({ + port: 0, + async fetch(req) { + if (req.method === "PUT") captured = await req.json(); + return new Response("OK"); + }, + }); + using dir = tempDir("dt-unknown-publish", { + "bunfig.toml": `[install]\ncache = false\nregistry = { url = "http://localhost:${registry.port}", token = "unused" }\n`, + "package.json": JSON.stringify({ + name: "dt-unknown-publish", + version: "1.0.0", + directories: { bin: "bins" }, + }), + "README.md": "# dt-unknown-publish", + "index.js": "", + "bins/a.js": "", + "bins/more/b.js": "", + }); + + await run(String(dir), "publish"); + + expect(captured.versions["1.0.0"]).toMatchObject({ + bin: { "a.js": "bins/a.js", "more": "bins/more", "b.js": "bins/more/b.js" }, + readme: "# dt-unknown-publish", + readmeFilename: "README.md", + }); + + const attachment: { data: string } = Object.values(captured._attachments)[0] as any; + const tarball = join(String(dir), "published.tgz"); + await writeFile(tarball, Buffer.from(attachment.data, "base64")); + expect(packedPaths(tarball)).toEqual([ + "package/README.md", + "package/bins/a.js", + "package/bins/more/b.js", + "package/index.js", + "package/package.json", + ]); + }); +}); diff --git a/test/fixtures/dt-unknown-readdir-shim.c b/test/fixtures/dt-unknown-readdir-shim.c new file mode 100644 index 000000000000..e6a435a315e0 --- /dev/null +++ b/test/fixtures/dt-unknown-readdir-shim.c @@ -0,0 +1,50 @@ +// LD_PRELOAD shim: every getdents64 record comes back with d_type == DT_UNKNOWN, +// the way FUSE, some NFS servers and XFS formatted with ftype=0 report entries. +// bun issues getdents64 through libc's syscall() wrapper, which this interposes. +// Compiled by `dtUnknownReaddir` in test/harness.ts, which defines MARKER: it is +// written to stderr the first time a record is rewritten so a test can tell the +// shim actually saw bun's readdir calls. +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include + +static long (*real_syscall)(long, long, long, long, long, long, long); +static int announced; + +long syscall(long number, ...) { + va_list ap; + long a, b, c, d, e, f; + va_start(ap, number); + a = va_arg(ap, long); + b = va_arg(ap, long); + c = va_arg(ap, long); + d = va_arg(ap, long); + e = va_arg(ap, long); + f = va_arg(ap, long); + va_end(ap); + if (!real_syscall) { + real_syscall = (long (*)(long, long, long, long, long, long, long))dlsym(RTLD_NEXT, "syscall"); + } + long rc = real_syscall(number, a, b, c, d, e, f); + if (number != SYS_getdents64 || rc <= 0) return rc; + if (!announced) { + announced = 1; + static const char marker[] = MARKER "\n"; + if (write(2, marker, sizeof(marker) - 1) < 0) { + } + } + // struct linux_dirent64 { u64 d_ino; s64 d_off; u16 d_reclen; u8 d_type; char d_name[]; } + unsigned char *buf = (unsigned char *)b; + for (long off = 0; off + 19 <= rc;) { + uint16_t reclen; + memcpy(&reclen, buf + off + 16, sizeof(reclen)); + if (reclen == 0) break; + buf[off + 18] = 0; /* DT_UNKNOWN */ + off += reclen; + } + return rc; +} diff --git a/test/harness.ts b/test/harness.ts index b3167919ec84..c2034a71319c 100644 --- a/test/harness.ts +++ b/test/harness.ts @@ -2275,6 +2275,51 @@ export function compileFixture(sourcePath: string, options: { flags?: string[] } return outPath; } +const dtUnknownReaddirMarker = "dt-unknown-readdir-shim: rewrote getdents64 d_type"; +let dtUnknownReaddirShim: Promise | undefined; + +async function compileDtUnknownReaddirShim(): Promise { + const cc = which("cc") || which("clang") || which("gcc"); + if (!cc) throw new Error("dtUnknownReaddir: no C compiler (cc/clang/gcc) found in $PATH"); + const shim = join(tmpdirSync("dt-unknown-readdir-"), "shim.so"); + const source = join(import.meta.dir, "fixtures", "dt-unknown-readdir-shim.c"); + const proc = Bun.spawn({ + cmd: [cc, "-shared", "-fPIC", "-O2", `-DMARKER="${dtUnknownReaddirMarker}"`, "-o", shim, source, "-ldl"], + env: bunEnv, + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + if (exitCode !== 0) + throw new Error(`dtUnknownReaddir: compiling the shim failed (exit ${exitCode}):\n${stderr || stdout}`); + return shim; +} + +/** + * Runs bun as if on a filesystem whose readdir does not report entry types + * (FUSE, some NFS servers, XFS formatted with `ftype=0`), without needing such a + * mount: `env()` preloads a shim that zeroes `d_type` in every `getdents64` + * record. The shim prints `marker` to stderr the first time it does so; assert + * on it, otherwise a test here passes vacuously if bun ever stops issuing + * `getdents64` through libc's `syscall()` wrapper, which is what the shim hooks. + */ +export const dtUnknownReaddir = { + /** Linux with a C compiler; `skipIf(!dtUnknownReaddir.available)`. */ + get available(): boolean { + return isLinux && !!(which("cc") || which("clang") || which("gcc")); + }, + marker: dtUnknownReaddirMarker, + /** + * Compiles the shim the first time it is called. Call it from `beforeAll` + * (the compiler can take several seconds on a loaded machine) and spawn bun + * with the returned env. + */ + async env(): Promise> { + const shim = await (dtUnknownReaddirShim ??= compileDtUnknownReaddirShim()); + return { ...bunEnv, LD_PRELOAD: bunEnv.LD_PRELOAD ? `${shim}:${bunEnv.LD_PRELOAD}` : shim }; + }, +}; + export const rss: () => number = process.platform === "darwin" && typeof Bun.unsafe.memoryFootprint === "function" ? (Bun.unsafe.memoryFootprint as () => number)