diff --git a/scripts/build/CLAUDE.md b/scripts/build/CLAUDE.md index d387f153bddb..de8bbc8f2c5b 100644 --- a/scripts/build/CLAUDE.md +++ b/scripts/build/CLAUDE.md @@ -188,6 +188,7 @@ Split CI modes: `rust-only` (lolhtml+codegen+cargo → libbun_rust.a), `cpp-only | `source.ts` | `Dependency` types, `resolveDep()`, fetch/configure/build emission | | `codegen.ts` | Code generation steps, `emitCodegen()`, `CodegenOutputs` | | `rust.ts` | `cargo build` step, `emitRust()`, `rustLibPath()`, cross-compile matrix | +| `rustc-metadata-shim.rs` | `RUSTC_WORKSPACE_WRAPPER` — pins `-C metadata` so our crates' symbol names survive a dependency-graph edit | | `cargo-config.ts` | Generates the git-ignored `.cargo/config.toml` (per-target `linker` from `cfg.hostCxx`) | | `bun.ts` | `emitBun()` — assembles deps+codegen+rust+compile+link | | `shims.ts` | Platform/toolchain workaround dylibs, `emitShims()` | diff --git a/scripts/build/rust.ts b/scripts/build/rust.ts index 73ef19932385..c78c264fcdc0 100644 --- a/scripts/build/rust.ts +++ b/scripts/build/rust.ts @@ -180,6 +180,14 @@ export function rustLibPath(cfg: Config): string { return resolve(rustTargetDir(cfg), rustTarget(cfg), subdir, `${cfg.libPrefix}bun_rust${cfg.libSuffix}`); } +/** The `RUSTC_WORKSPACE_WRAPPER` source — see its module doc for why it exists. */ +const metadataShimSource = resolve(import.meta.dirname, "rustc-metadata-shim.rs"); + +/** Where the compiled wrapper lands. A HOST executable — cargo spawns it. */ +function metadataShimPath(cfg: Config): string { + return resolve(cfg.buildDir, `rustc-metadata-shim${cfg.host.exeSuffix}`); +} + // ─────────────────────────────────────────────────────────────────────────── // Ninja rules // ─────────────────────────────────────────────────────────────────────────── @@ -203,6 +211,34 @@ export function registerRustRules(n: Ninja, cfg: Config): void { if (cfg.cargo === undefined) return; // emitRust() asserts with a hint const stream = `${cfg.jsRuntime} ${q(streamPath)} rust`; + const rustup = findRustup(cfg); + // Toolchain self-heal, prepended to anything that invokes a rustup proxy — + // see the rust_build_cross comment below for what it repairs and why it's a + // ~70ms no-op otherwise. + const repair = + rustup !== undefined && cfg.rustToolchain !== undefined + ? `${stream} --console $env ${q(rustup)} toolchain install ${cfg.rustToolchain} --force --component rust-src $rust_target_arg && ` + : ""; + + // `RUSTC_WORKSPACE_WRAPPER` — pins `-C metadata` so our crates keep the same + // v0 symbol names across builds (rustc-metadata-shim.rs has the why). A bare + // `rustc` builds it: it must exist before cargo runs, so it can't be a + // workspace member. That makes it the build's first rustup-proxy call, hence + // the repair above: a partially installed toolchain has no host `rust-std` + // for it to link against. `-Clinker`: rustc's default `cc` isn't on every CI + // image, so use the clang tools.ts resolved (MSVC's link.exe on a Windows + // host), same as the cargo env picks for the dep graph's host artifacts. + const rustc = join(dirname(cfg.cargo), `rustc${cfg.host.exeSuffix}`); + const shimLink = hostWin + ? [`-Clinker=${cfg.msvcLinker ?? cfg.ld}`] + : [`-Clinker=${cfg.hostCc}`, "-Clink-arg=-fuse-ld=lld"]; + const shimCmd = `${repair}${q(rustc)} --edition 2024 -Copt-level=2 ${quoteArgs(shimLink, hostWin)} -o $out $in`; + n.rule("rustc_metadata_shim", { + // ninja spawns via CreateProcess, so `&&` needs a shell — same as rust_build_cross. + command: hostWin && repair !== "" ? `cmd /c "${shimCmd}"` : shimCmd, + description: "rustc → $out", + }); + // Cargo build for `bun_bin`. Runs from repo root (workspace `Cargo.toml` // lives there). Env passed via stream.ts `--env=K=V`. // @@ -295,11 +331,8 @@ export function registerRustRules(n: Ninja, cfg: Config): void { }); } - const rustup = findRustup(cfg); - if (rustup !== undefined && cfg.rustToolchain !== undefined) { - const chain = - `${stream} --console $env ${q(rustup)} toolchain install ${cfg.rustToolchain} --force --component rust-src $rust_target_arg && ` + - `${stream} --console --cwd=$cwd $env ${q(cfg.cargo)} build $args`; + if (repair !== "") { + const chain = `${repair}${stream} --console --cwd=$cwd $env ${q(cfg.cargo)} build $args`; n.rule("rust_build_cross", { command: hostWin ? `cmd /c "${chain}"` : chain, description: "cargo bun_bin → $label ($rust_target_arg)", @@ -365,6 +398,11 @@ export function emitRust(n: Ninja, cfg: Config, inputs: RustBuildInputs): string const profile = cargoProfile(cfg); const lib = rustLibPath(cfg); + // Implicit input of every cargo edge below, so it's on disk before cargo + // tries to spawn it. Emitted after `env` is built (the rule's toolchain + // repair wants it). + const metadataShim = metadataShimPath(cfg); + // ─── Build args ─── const args: string[] = [ "-p", @@ -653,6 +691,12 @@ export function emitRust(n: Ninja, cfg: Config, inputs: RustBuildInputs): string // ─── Environment ─── const env: Record = { CARGO_TERM_COLOR: "always", + // Pins `-C metadata` for our own crates so their v0 symbol names survive a + // dependency-graph edit — rustc-metadata-shim.rs has the why. `_WORKSPACE_` + // rather than plain `RUSTC_WRAPPER`: cargo applies it to workspace members + // only, leaving registry crates on its collision-proof hash. `cargo clippy` + // sets this variable itself, so it keeps working. + RUSTC_WORKSPACE_WRAPPER: metadataShim, // `include!(concat!(env!("BUN_CODEGEN_DIR"), "/generated_*.rs"))` and // `include_bytes!` in `bun_js_parser`/`bun_runtime` resolve against this. // Set in cargo's env so it reaches every crate's `rustc` invocation @@ -764,6 +808,18 @@ export function emitRust(n: Ninja, cfg: Config, inputs: RustBuildInputs): string } if (rustflags.length > 0) env.CARGO_ENCODED_RUSTFLAGS = rustflags.join("\x1f"); + const envArgs = Object.entries(env) + .map(([k, v]) => `--env=${k}=${quote(v, hostWin)}`) + .join(" "); + + // ─── `-C metadata` pin ─── + n.build({ + outputs: [metadataShim], + rule: "rustc_metadata_shim", + inputs: [metadataShimSource], + vars: { env: envArgs }, + }); + // ─── Windows .bin/ shim PE ─── // Builds `src/install/windows-shim/bun_shim_impl.rs` as a freestanding release PE and wires the artifact into `include_bytes!`. Without this step `include_bytes!` embeds the // 0-byte placeholder and `bun install` writes empty `.exe`s into @@ -861,7 +917,7 @@ export function emitRust(n: Ninja, cfg: Config, inputs: RustBuildInputs): string // workspace manifest if any path-dep's `Cargo.toml` is missing. // shimDest: rebuilt when a sibling build dir (other arch/profile) // overwrote the shared exe. - implicitInputs: [cfg.cargo, ...inputs.rustSources, ...inputs.vendorStamps, shimDest], + implicitInputs: [cfg.cargo, metadataShim, ...inputs.rustSources, ...inputs.vendorStamps, shimDest], vars: { cwd: cfg.cwd, args: quoteArgs(shimArgs, hostWin), @@ -898,16 +954,21 @@ export function emitRust(n: Ninja, cfg: Config, inputs: RustBuildInputs): string // so depending on those orders the codegen step before cargo without // ninja needing to know the `.rs` paths. vendorStamps orders the // lol-html source fetch before cargo resolves the path dep. - implicitInputs: [cfg.cargo, ...inputs.rustSources, ...inputs.codegenInputs, ...inputs.vendorStamps, ...shimInputs], + implicitInputs: [ + cfg.cargo, + metadataShim, + ...inputs.rustSources, + ...inputs.codegenInputs, + ...inputs.vendorStamps, + ...shimInputs, + ], orderOnlyInputs: inputs.codegenOrderOnly, vars: { cwd: cfg.cwd, args: quoteArgs(args, hostWin), ...(useCrossRule ? { rust_target_arg: tier3 ? "" : `--target ${triple}` } : {}), label: `${cfg.libPrefix}bun_rust${cfg.libSuffix}`, - env: Object.entries(env) - .map(([k, v]) => `--env=${k}=${quote(v, hostWin)}`) - .join(" "), + env: envArgs, }, }); n.phony("bun-rust", [lib]); diff --git a/scripts/build/rustc-metadata-shim.rs b/scripts/build/rustc-metadata-shim.rs new file mode 100644 index 000000000000..bcb5c3cd6a94 --- /dev/null +++ b/scripts/build/rustc-metadata-shim.rs @@ -0,0 +1,72 @@ +//! `RUSTC_WORKSPACE_WRAPPER`: replace cargo's `-C metadata` with the package +//! name, so bun's crates keep the same v0 symbol names from build to build. +//! +//! Cargo mixes the `-C metadata` of every dependency into a unit's own, and +//! rustc hashes that into the `StableCrateId` every v0 symbol carries — the +//! `Cs…` in `_RNvNtNtCs4EG9u9StnXu_11bun_runtime3cli7command15boot_standalone`. +//! A dependency-edge edit *anywhere below a crate* therefore renames every +//! symbol that crate defines. `bun_runtime` has ~100 direct workspace deps, so +//! it was renamed by nearly every commit, and anything matching symbol names +//! across two builds silently lost it: lld `--symbol-ordering-file` reuse, +//! sccache hits, symbolicating an old profile against a new binary. +//! +//! rustc hashes *every* `-C metadata` it is handed, so an extra one from +//! RUSTFLAGS can't shadow cargo's — rewriting it here is the only lever. +//! +//! The package name alone keeps symbols unique: rustc mixes the crate name into +//! `StableCrateId` on top of this, no two workspace packages share a name, and +//! cargo applies this wrapper to workspace members only — registry crates, where +//! two versions of one name can coexist, keep cargo's hash. + +use std::env; +use std::ffi::OsString; +use std::process::Command; + +fn main() { + let mut argv = env::args_os().skip(1); + let Some(rustc) = argv.next() else { + eprintln!("rustc-metadata-shim: expected rustc's path as the first argument"); + std::process::exit(1); + }; + + // Cargo emits `-C` and `metadata=` as two arguments, and nothing else + // it passes starts with `metadata=`. Invocations with none at all (cargo's + // `-vV` / `--print` probes) fall through untouched. + let pin = format!( + "metadata=bun.{}", + env::var("CARGO_PKG_NAME").unwrap_or_default() + ); + let args: Vec = argv + .map(|arg| match arg.to_str() { + Some(a) if a.starts_with("metadata=") => OsString::from(pin.as_str()), + _ => arg, + }) + .collect(); + + run(&rustc, &args) +} + +#[cfg(unix)] +fn run(rustc: &OsString, args: &[OsString]) -> ! { + use std::os::unix::process::CommandExt; + let err = Command::new(rustc).args(args).exec(); + eprintln!( + "rustc-metadata-shim: cannot exec {}: {err}", + rustc.to_string_lossy() + ); + std::process::exit(1) +} + +#[cfg(not(unix))] +fn run(rustc: &OsString, args: &[OsString]) -> ! { + match Command::new(rustc).args(args).status() { + Ok(status) => std::process::exit(status.code().unwrap_or(1)), + Err(err) => { + eprintln!( + "rustc-metadata-shim: cannot run {}: {err}", + rustc.to_string_lossy() + ); + std::process::exit(1) + } + } +} diff --git a/test/internal/rustc-metadata-shim.test.ts b/test/internal/rustc-metadata-shim.test.ts new file mode 100644 index 000000000000..349e5ab3afe8 --- /dev/null +++ b/test/internal/rustc-metadata-shim.test.ts @@ -0,0 +1,136 @@ +/** + * Regression tests for the `-C metadata` pin — scripts/build/rustc-metadata-shim.rs + * and its wiring in scripts/build/rust.ts. + * + * Cargo folds the `-C metadata` of every dependency into a unit's own, and rustc + * hashes that into the `Cs…` disambiguator every v0 symbol carries. Without the + * wrapper, a dependency-edge edit anywhere below a crate renames every symbol + * that crate defines — `bun_runtime`, with ~100 direct workspace deps, was + * renamed by nearly every commit. + * + * The graph assertion is configure-time only and runs everywhere. The + * behavioural ones drive the wrapper `bun bd` already built, so they need a + * build directory; CI's test lanes run a downloaded binary and skip them. (The + * build-rust lanes are what prove the wrapper compiles and cargo runs through + * it on every platform — no need to rebuild it from a test.) + */ +import { beforeAll, describe, expect, test } from "bun:test"; +import { bunEnv, bunExe, isWindows, tempDir } from "harness"; +import { existsSync } from "node:fs"; +import { dirname, join } from "node:path"; + +import { resolveConfig, type Toolchain } from "../../scripts/build/config.ts"; +import { Ninja } from "../../scripts/build/ninja.ts"; +import { emitRust, registerRustRules } from "../../scripts/build/rust.ts"; + +test("the wrapper is built before cargo and reaches its env", () => { + /** A fully-populated fake toolchain — resolveConfig never spawns any of these. */ + const toolchain: Toolchain = { + cc: "/fake/llvm/bin/clang", + cxx: "/fake/llvm/bin/clang++", + clangVersion: "21.1.8", + clangResourceDir: "/fake/llvm/lib/clang/21", + ar: "/fake/llvm/bin/llvm-ar", + ranlib: "/fake/llvm/bin/llvm-ranlib", + ld: "/fake/llvm/bin/ld.lld", + ld64Lld: "/fake/llvm/bin/ld64.lld", + rustLld: undefined, + rustLlvmVersion: "22.1.4", + rustSysroot: undefined, + rustHostTriple: undefined, + strip: "/fake/bin/strip", + llvmStrip: "/fake/llvm/bin/llvm-strip", + dsymutil: "/fake/llvm/bin/dsymutil", + bun: "/fake/bin/bun", + jsRuntime: "/fake/bin/bun", + esbuild: "/fake/bin/esbuild", + ccache: undefined, + cmake: "/fake/bin/cmake", + cargo: "/fake/rust/bin/cargo", + cargoHome: "/fake/.cargo", + rustupHome: "/fake/.rustup", + msvcLinker: undefined, + rc: undefined, + mt: undefined, + nasm: undefined, + }; + const cfg = resolveConfig({ os: "linux", arch: "x64", buildType: "Debug" }, toolchain); + const n = new Ninja({ buildDir: cfg.buildDir }); + registerRustRules(n, cfg); + emitRust(n, cfg, { codegenInputs: [], codegenOrderOnly: [], rustSources: [], vendorStamps: [] }); + // ninja wraps long build lines with a trailing `$`; join them back up. + const graph = n.toString().replace(/\$\n\s+/g, " "); + + // `.exe` on a Windows host: the wrapper is a host executable, and the host + // isn't the linux target asked for above. + expect(graph).toMatch(/^build rustc-metadata-shim(\.exe)?: rustc_metadata_shim \S*rustc-metadata-shim\.rs\b/m); + // `_WORKSPACE_`, not plain RUSTC_WRAPPER: registry crates keep cargo's + // collision-proof hash, only our own crates get a pinned one. + expect(graph).toContain("--env=RUSTC_WORKSPACE_WRAPPER="); + expect(graph).not.toContain("--env=RUSTC_WRAPPER="); + + const cargoEdge = graph.split("\n").find(line => line.startsWith("build ") && line.includes("libbun_rust.a:")); + expect(cargoEdge).toBeDefined(); + // Implicit inputs come after the `|` — cargo can't run before the wrapper exists. + expect(cargoEdge!.split("|")[1]).toContain("rustc-metadata-shim"); +}); + +// emitRust() drops the wrapper in the build directory, which is where the bun +// under test lives when it came from `bun bd`. A downloaded binary has no build +// directory next to it, so CI's test lanes skip: they have no business invoking +// a rust toolchain, and on a rustup proxy the first call downloads a channel. +const shim = join(dirname(process.execPath), `rustc-metadata-shim${isWindows ? ".exe" : ""}`); + +describe.skipIf(!existsSync(shim))("rustc metadata shim", () => { + let fakeRustc = ""; + + beforeAll(() => { + // Not disposed: bun:test has no `using` for suite-scoped fixtures, and the + // OS reaps its own temp dir. + const dir = tempDir("rustc-metadata-shim", { "fake-rustc.ts": `console.log(process.argv.slice(2).join("\\n"))` }); + fakeRustc = join(String(dir), "fake-rustc.ts"); + }); + + /** Run the wrapper; returns the argv it would have handed the real rustc. */ + async function argvFor(args: string[], pkg = "bun_runtime") { + await using proc = Bun.spawn({ + // The wrapper's first argument is always rustc's path — that's cargo's contract. + cmd: [shim, bunExe(), fakeRustc, ...args], + env: { ...bunEnv, CARGO_PKG_NAME: pkg }, + stderr: "pipe", + }); + // stderr is drained so the pipe can't fill, but not asserted empty — a debug + // or ASAN bun writes benign noise there. It does carry the wrapper's own + // failure message, so surface it when the exit code is wrong. + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + if (exitCode !== 0) throw new Error(`wrapper exited ${exitCode}\n${stderr}`); + return stdout.split("\n").filter(Boolean); + } + + const unit = ["--crate-name", "bun_runtime", "--crate-type", "lib", "--target", "x86_64-unknown-linux-gnu"]; + + test.concurrent("two dependency-graph states collapse to the same metadata", async () => { + // The only difference is cargo's dependency hash — exactly what moves when a + // crate anywhere below this one gains or loses a dependency. `extra-filename` + // must survive: it keys cargo's on-disk artifact names. + const before = [...unit, "-C", "extra-filename=-4e6e51e9e0da5e6b", "-C", "metadata=4e6e51e9e0da5e6b", "lib.rs"]; + const after = [...unit, "-C", "extra-filename=-9c1d0ab7735ffd12", "-C", "metadata=9c1d0ab7735ffd12", "lib.rs"]; + + expect(await argvFor(before)).toEqual( + before.map(arg => (arg.startsWith("metadata=") ? "metadata=bun.bun_runtime" : arg)), + ); + expect(await argvFor(after)).toEqual( + after.map(arg => (arg.startsWith("metadata=") ? "metadata=bun.bun_runtime" : arg)), + ); + }); + + test.concurrent("distinct packages keep distinct metadata", async () => { + expect(await argvFor([...unit, "-C", "metadata=aaaa"], "bun_core")).toContain("metadata=bun.bun_core"); + }); + + test.concurrent("an invocation without -C metadata passes through untouched", async () => { + // cargo probes the wrapper with `-vV` / `--print` before it compiles anything. + expect(await argvFor(["-vV"])).toEqual(["-vV"]); + expect(await argvFor(["--print", "cfg"])).toEqual(["--print", "cfg"]); + }); +});