diff --git a/Cargo.lock b/Cargo.lock index 5c8ed1083cc9..dbe6c928028d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1962,7 +1962,6 @@ dependencies = [ "bun_bundler", "bun_collections", "bun_core", - "bun_dotenv", "bun_errno", "bun_exe_format", "bun_http", diff --git a/src/bundler/BundleThread.rs b/src/bundler/BundleThread.rs index c5a405b1549a..cdb812f72a63 100644 --- a/src/bundler/BundleThread.rs +++ b/src/bundler/BundleThread.rs @@ -150,6 +150,8 @@ impl BundleThread { let ptr = SendPtr(instance); let thread = std::thread::Builder::new() .name("Bundler".into()) + // `Bun.build({ compile })` writes its executable here; use bun's usual stack. + .stack_size(bun_threading::thread_pool::DEFAULT_THREAD_STACK_SIZE as usize) .spawn(move || { let ptr = ptr; // SAFETY: caller guarantees `instance` is valid for 'static; `thread_main` diff --git a/src/options_types/compile_target.rs b/src/options_types/compile_target.rs index 1875cab88df8..347e26319803 100644 --- a/src/options_types/compile_target.rs +++ b/src/options_types/compile_target.rs @@ -98,7 +98,40 @@ pub enum ParseError { InvalidTarget, } +/// Env-derived settings for downloading a target's executable; see [`CompileTarget::download_options`]. +pub struct DownloadOptions { + pub reject_unauthorized: bool, + /// Proxy for the registry URL, with `NO_PROXY` already applied. + pub http_proxy: Option>, +} + +impl Default for DownloadOptions { + /// Placeholder until [`CompileTarget::download_options`] fills it in; verifies TLS. + fn default() -> Self { + Self { + reject_unauthorized: true, + http_proxy: None, + } + } +} + impl CompileTarget { + /// Buffer size callers hand to [`Self::to_npm_registry_url`]. + pub const REGISTRY_URL_BUF_LEN: usize = 2048; + + pub fn download_options(&self, env: &bun_dotenv::Loader) -> DownloadOptions { + let mut url_buf = [0u8; Self::REGISTRY_URL_BUF_LEN]; + // A URL that cannot be built fails the download itself, which reports it. + let http_proxy = self.to_npm_registry_url(&mut url_buf).ok().and_then(|url| { + env.get_http_proxy_for(&bun_url::URL::parse(url)) + .map(|proxy| Box::from(proxy.href)) + }); + DownloadOptions { + reject_unauthorized: env.get_tls_reject_unauthorized(), + http_proxy, + } + } + pub(crate) fn eql(&self, other: &CompileTarget) -> bool { self.os == other.os && self.arch == other.arch @@ -185,7 +218,6 @@ impl CompileTarget { &self, buf: &'a mut PathBuffer, version_str: &'a ZStr, - _env: &mut bun_dotenv::Loader, needs_download: &mut bool, ) -> &'a ZStr { if self.is_default() { @@ -206,7 +238,7 @@ impl CompileTarget { return version_str; } - // T1 fallback ignores `_env` (full env-override chain lives in bun_install). + // The full cache-dir override chain lives in bun_install; this is its fallback. let cache_dir = bun_sys::fetch_cache_directory_path(); let dest = path::resolve_path::join_abs_string_buf_z::( path::fs::FileSystem::instance().top_level_dir(), diff --git a/src/runtime/api/JSBundler.rs b/src/runtime/api/JSBundler.rs index f70ee676d02e..3f1b9ab3fbfc 100644 --- a/src/runtime/api/JSBundler.rs +++ b/src/runtime/api/JSBundler.rs @@ -14,7 +14,7 @@ use bun_core::Output; use bun_core::{String as BunString, ZigString}; use bun_jsc::ConcurrentTask::ConcurrentTask; use bun_jsc::{self as jsc, CallFrame, JSGlobalObject, JSValue, JsError, JsResult}; -use bun_options_types::compile_target::CompileTarget; +use bun_options_types::compile_target::{CompileTarget, DownloadOptions}; use bun_options_types::schema::api; // bun.schema.api use bun_standalone_graph::StandaloneModuleGraph; @@ -223,6 +223,8 @@ pub mod js_bundler { pub struct CompileOptions { pub(crate) compile_target: CompileTarget, + /// Taken from the VM's env on the calling thread; the download runs on the bundle thread. + pub(crate) download: DownloadOptions, pub(crate) exec_argv: OwnedString, pub(crate) executable_path: OwnedString, pub(crate) windows_hide_console: bool, @@ -244,6 +246,7 @@ pub mod js_bundler { fn default() -> Self { Self { compile_target: CompileTarget::default(), + download: DownloadOptions::default(), exec_argv: OwnedString::default(), executable_path: OwnedString::default(), windows_hide_console: false, @@ -1197,6 +1200,9 @@ pub mod js_bundler { let is_standalone_html = this.target == Target::Browser && has_all_html_entrypoints; if !is_standalone_html { this.target = Target::Bun; + compile.download = compile + .compile_target + .download_options(global_this.bun_vm().env_loader()); let define_keys = compile.compile_target.define_keys(); let define_values = compile.compile_target.define_values(); diff --git a/src/runtime/api/js_bundle_completion_task.rs b/src/runtime/api/js_bundle_completion_task.rs index 27657f0da198..f1a37ea3c515 100644 --- a/src/runtime/api/js_bundle_completion_task.rs +++ b/src/runtime/api/js_bundle_completion_task.rs @@ -287,8 +287,8 @@ impl JSBundleCompletionTask { Ok(()) } - /// Port of `JSBundleCompletionTask.doCompilation`. - fn do_compilation(&mut self, output_files: &mut Vec) -> CompileResult { + /// Port of `JSBundleCompletionTask.doCompilation`. Bundle thread: nothing in here may touch JS. + fn do_compilation(&self, output_files: &mut Vec) -> CompileResult { let compile_options = self .config .compile @@ -426,10 +426,7 @@ impl JSBundleCompletionTask { root_dir.fd, module_prefix, outfile_for_executable, - // SAFETY: `self.env` is the per-VM `DotEnv.Loader` stashed at - // construction; valid for the lifetime of the VirtualMachine, and - // nothing inside `to_executable` reaches it otherwise. - unsafe { &mut *self.env }, + &compile_options.download, self.config.format, &WindowsOptions { hide_console: compile_options.windows_hide_console, @@ -559,6 +556,36 @@ impl JSBundleCompletionTask { result } + /// `Bun.build({ compile })`: the bundle becomes the executable before `on_complete` gets it. + fn compile_on_bundle_thread(&mut self) { + if self.config.compile.is_none() + || self.cancelled.load(core::sync::atomic::Ordering::Acquire) + { + return; + } + let mut build = match core::mem::replace(&mut self.result, BundleV2Result::Pending) { + BundleV2Result::Value(build) => build, + not_bundled => { + self.result = not_bundled; + return; + } + }; + let compile_result = self.do_compilation(&mut build.output_files); + // Nothing else flushes this thread's buffered stderr, which the above reports failures to. + bun_core::Output::flush(); + self.result = match compile_result { + CompileResult::Success => BundleV2Result::Value(build), + CompileResult::Err(err) => { + self.log.add_error_fmt( + None, + bun_ast::Loc::EMPTY, + format_args!("{}", bstr::BStr::new(err.slice())), + ); + BundleV2Result::Err(bun_bundler::Error::CompilationFailed) + } + }; + } + pub(crate) fn on_complete_anytask(ctx: *mut Self) -> bun_event_loop::JsResult<()> { crate::jsc_hooks::ActiveHandle::Bundle(NonNull::new(ctx).expect("completion")).unregister(); // For the +1 taken by `complete_on_bundle_thread` enqueue. @@ -644,7 +671,7 @@ impl JSBundleCompletionTask { } // Copy the BackRef out (it is `Copy`) so `global_this` borrows a local - // instead of `*this` — `do_compilation`/`to_js_error` below need `&mut *this`. + // instead of `*this` — `to_js_error` below needs `&mut *this`. let global_this_ref = this.global_this; let global_this = global_this_ref.get(); // `Strong::swap` ties the returned `&mut JSPromise` to @@ -654,40 +681,6 @@ impl JSBundleCompletionTask { let promise: *mut JSPromise = this.promise.swap(); let promise = JSPromise::opaque_mut(promise); - // `do_compilation` borrows `&mut self` while needing - // `&mut output_files` from inside `self.result`. Temporarily move the - // Vec out via `take` so the method gets a disjoint `&mut self`. - if matches!(this.result, BundleV2Result::Value(_)) && this.config.compile.is_some() { - let mut output_files = match &mut this.result { - BundleV2Result::Value(build) => core::mem::take(&mut build.output_files), - // SAFETY: arm checked above. - _ => unsafe { core::hint::unreachable_unchecked() }, - }; - let compile_result = this.do_compilation(&mut output_files); - // `defer compile_result.deinit()` — `CompileResult` is a Rust enum - // with owned `Vec` payloads; drops at end of scope. - - if let CompileResult::Err(err) = &compile_result { - // `bun.handleOom(log.addError(..., bun.handleOom(dupe(..))))` - this.log.add_error_fmt( - None, - bun_ast::Loc::EMPTY, - format_args!("{}", bstr::BStr::new(err.slice())), - ); - // `this.result.value.deinit()` — owned fields drop with the - // overwrite below; `output_files` (moved out above) drops here. - drop(output_files); - this.result = BundleV2Result::Err(bun_bundler::Error::CompilationFailed); - } else { - // Put the compacted output_files back. - match &mut this.result { - BundleV2Result::Value(build) => build.output_files = output_files, - // SAFETY: arm checked above. - _ => unsafe { core::hint::unreachable_unchecked() }, - } - } - } - // `to_js_error` borrows `&mut self`, which would overlap a // `&mut this.result` match scrutinee. Dispatch the pending/err arms // first, then take a fresh `&mut` for Value. @@ -1115,6 +1108,7 @@ impl CompletionStruct for JSBundleCompletionTask { } fn complete_on_bundle_thread(&mut self) { + self.compile_on_bundle_thread(); // The bundle thread's last touch of this task and of the VM's memory: // hand it back (always queued — the VM waits for it) and stop counting. self.bundle_loop diff --git a/src/runtime/cli/build_command.rs b/src/runtime/cli/build_command.rs index 92003b6f7c64..13fb226cc817 100644 --- a/src/runtime/cli/build_command.rs +++ b/src/runtime/cli/build_command.rs @@ -889,7 +889,7 @@ impl BuildCommand { &opt_public_path, outfile, // SAFETY: `env` is a process-lifetime singleton. - unsafe { &mut *env_ptr }, + &compile_target.download_options(unsafe { &*env_ptr }), opt_output_format, &ctx.bundler_options.windows, ctx.bundler_options diff --git a/src/standalone_graph/Cargo.toml b/src/standalone_graph/Cargo.toml index 72d2572e8333..00e656add3ef 100644 --- a/src/standalone_graph/Cargo.toml +++ b/src/standalone_graph/Cargo.toml @@ -25,7 +25,6 @@ bun_alloc.workspace = true bun_core.workspace = true bun_bundler.workspace = true bun_collections.workspace = true -bun_dotenv.workspace = true bun_exe_format.workspace = true bun_http.workspace = true bun_parsers.workspace = true diff --git a/src/standalone_graph/StandaloneModuleGraph.rs b/src/standalone_graph/StandaloneModuleGraph.rs index 20e2851fcfa8..80a44f309308 100644 --- a/src/standalone_graph/StandaloneModuleGraph.rs +++ b/src/standalone_graph/StandaloneModuleGraph.rs @@ -1646,7 +1646,7 @@ pub(crate) fn inject( } use bun_core::Environment::OperatingSystem as CompileTargetOs; -pub use bun_options_types::compile_target::CompileTarget; +pub use bun_options_types::compile_target::{CompileTarget, DownloadOptions}; /// Moved up from `bun_options_types` (T3) so it can name /// `bun_http::AsyncHTTP` directly @@ -1654,7 +1654,7 @@ pub use bun_options_types::compile_target::CompileTarget; /// two `download*` fns below in this crate. pub(crate) fn download_to_path( target: &CompileTarget, - env: &mut bun_dotenv::Loader, + download: &DownloadOptions, dest_z: &ZStr, ) -> crate::Result<()> { bun_http::http_thread::init(&Default::default()); @@ -1666,7 +1666,7 @@ pub(crate) fn download_to_path( // TODO: This is way too much code necessary to send a single HTTP request... let mut compressed_archive_bytes = Box::new(bun_core::MutableString::init(24 * 1024 * 1024)?); - let mut url_buffer = [0u8; 2048]; + let mut url_buffer = [0u8; CompileTarget::REGISTRY_URL_BUF_LEN]; let url_str = match target.to_npm_registry_url(&mut url_buffer) { Ok(s) => s, Err(err) => { @@ -1681,10 +1681,6 @@ pub(crate) fn download_to_path( // `progress.end()` below is sufficient: no fallible call sits between // `refresher.start` and it, so every exit path (including the // error returns after it) ends the node exactly once. - // Note: reshaped for borrowck — `get_http_proxy_for` borrows - // `env` for the proxy URL lifetime; read the bool first. - let reject_unauthorized = env.get_tls_reject_unauthorized(); - let http_proxy: Option> = env.get_http_proxy_for(&url); let progress = refresher.start(b"Downloading", 0); let mut async_http = Box::new(bun_http::AsyncHTTP::init_sync( @@ -1693,13 +1689,13 @@ pub(crate) fn download_to_path( Default::default(), b"", b"", - http_proxy, + download.http_proxy.as_deref().map(bun_url::URL::parse), None, bun_http::FetchRedirect::Follow, )); async_http.client.progress_node = core::ptr::NonNull::new(core::ptr::from_mut(progress)); - async_http.client.flags.reject_unauthorized = reject_unauthorized; + async_http.client.flags.reject_unauthorized = download.reject_unauthorized; let send_result = async_http.send_sync(&mut compressed_archive_bytes); progress.end(); @@ -1818,7 +1814,7 @@ pub fn to_executable( root_dir: Fd, module_prefix: &[u8], outfile: &[u8], - env: &mut bun_dotenv::Loader, + download: &DownloadOptions, output_format: Format, windows_options: &WindowsOptions, compile_exec_argv: &[u8], @@ -1870,10 +1866,10 @@ pub fn to_executable( let version_zstr = ZStr::from_slice_with_nul(&version_str[..]); let mut needs_download: bool = true; - let dest_z = target.exe_path(&mut exe_path_buf, version_zstr, env, &mut needs_download); + let dest_z = target.exe_path(&mut exe_path_buf, version_zstr, &mut needs_download); if needs_download { - if let Err(e) = download_to_path(target, env, dest_z) { + if let Err(e) = download_to_path(target, download, dest_z) { return Ok(match e { crate::Error::TargetNotFound => CompileResult::fail_fmt(format_args!( "Target platform '{}' is not available for download. Check if this version of Bun supports this target.", diff --git a/test/bundler/bun-build-compile.test.ts b/test/bundler/bun-build-compile.test.ts index 3020baa43de3..401b1fcfae90 100644 --- a/test/bundler/bun-build-compile.test.ts +++ b/test/bundler/bun-build-compile.test.ts @@ -72,6 +72,187 @@ describe("Bun.build compile", () => { }, ); + // These never copy the bun binary: the target download is answered with a 404 by a server + // in this test, or the template handed in via executablePath is rejected up front. + describe.concurrent("producing the executable", () => { + // Any OS other than the host's has to be downloaded; aarch64 never carries a "-baseline" suffix. + const target = `bun-${isMacOS ? "linux" : "darwin"}-aarch64`; + const [version] = Bun.version.match(/^\d+\.\d+\.\d+/)!; + const notAvailable = + JSON.stringify({ + success: false, + logs: [ + `Target platform '${target}-v${version}' is not available for download. Check if this version of Bun supports this target.`, + ], + }) + "\n"; + // A fresh cache, so the target is never already on disk, and no inherited proxy + // configuration, so the download goes where the test points it. + const downloadEnv = (dir: string, tarballUrl: string) => ({ + ...bunEnv, + BUN_COMPILE_TARGET_TARBALL_URL: tarballUrl, + BUN_INSTALL_CACHE_DIR: join(dir, "install-cache"), + HTTP_PROXY: undefined, + HTTPS_PROXY: undefined, + NO_PROXY: undefined, + http_proxy: undefined, + https_proxy: undefined, + no_proxy: undefined, + }); + const notFound = () => new Response("no such tarball", { status: 404 }); + + test("the event loop keeps running meanwhile", async () => { + // Producing the executable (downloading the target binary for a cross target, then + // copying and rewriting it) used to happen on the JS thread between bundling and the + // promise settling, so nothing else in that process ran meanwhile. Withhold the + // download's response until the building process has answered an IPC ping, which a + // process whose JS thread is inside the compile step cannot do. + using dir = tempDir("build-compile-event-loop", { + "app.js": `console.log("hi");`, + "build.js": ` + process.on("message", message => process.send(message)); + const result = await Bun.build({ + entrypoints: [import.meta.dir + "/app.js"], + compile: { target: process.argv[2], outfile: "app" }, + throw: false, + }); + console.log(JSON.stringify({ success: result.success, logs: result.logs.map(log => log.message) })); + process.exit(0); + `, + }); + + type Outcome = "never requested the target" | "answered the ping" | "did not answer the ping"; + let outcome: Outcome = "never requested the target"; + const pong = Promise.withResolvers(); + await using registry = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + async fetch() { + // A process stuck inside the compile step emits nothing, so that state can only be + // observed by giving up on the ping. The unblocked process answers within + // milliseconds (about 50ms under a debug build). + const gaveUp = Promise.withResolvers(); + const giveUp = setTimeout(gaveUp.resolve, 4_000, "did not answer the ping"); + try { + proc.send("ping"); + outcome = await Promise.race([ + pong.promise, + gaveUp.promise, + proc.exited.then((): Outcome => "did not answer the ping"), + ]); + } finally { + clearTimeout(giveUp); + } + return notFound(); + }, + }); + + await using proc = Bun.spawn({ + cmd: [bunExe(), "build.js", target], + cwd: String(dir), + env: downloadEnv(String(dir), `${registry.url}bun.tgz`), + stdout: "pipe", + stderr: "pipe", + ipc(message) { + if (message === "ping") pong.resolve("answered the ping"); + }, + }); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + + // stderr is only diagnostic: the download reports progress there once the response + // has been outstanding for 500ms, so it is empty or not depending on the ping's speed. + expect({ outcome, stdout, exitCode }, `stderr: ${stderr}`).toEqual({ + outcome: "answered the ping", + stdout: notAvailable, + exitCode: 0, + }); + }); + + test.each(["before", "after"])( + "the download uses the proxy settings from when Bun.build() was called (HTTP_PROXY set %s the call)", + async when => { + // Like fetch(), the proxy decision is made on the calling thread when Bun.build() is + // called, not read from the environment later by the thread doing the download. + using dir = tempDir("build-compile-download-proxy", { + "app.js": `console.log("hi");`, + "build.js": ` + const [when, target, proxy] = process.argv.slice(2); + if (when === "before") process.env.HTTP_PROXY = proxy; + const build = Bun.build({ + entrypoints: [import.meta.dir + "/app.js"], + compile: { target, outfile: "app" }, + throw: false, + }); + if (when === "after") process.env.HTTP_PROXY = proxy; + const result = await build; + console.log(JSON.stringify({ success: result.success, logs: result.logs.map(log => log.message) })); + `, + }); + const requests: string[] = []; + const serve = (name: string) => + Bun.serve({ + hostname: "127.0.0.1", + port: 0, + fetch(req) { + requests.push(`${name} received ${req.url}`); + return notFound(); + }, + }); + await using registry = serve("proxy-less registry"); + await using proxy = serve("proxy"); + const tarballUrl = `${registry.url}bun.tgz`; + + await using proc = Bun.spawn({ + cmd: [bunExe(), "build.js", when, target, String(proxy.url)], + cwd: String(dir), + env: downloadEnv(String(dir), tarballUrl), + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + + expect({ requests, stdout, exitCode }, `stderr: ${stderr}`).toEqual({ + requests: [`${when === "before" ? "proxy" : "proxy-less registry"} received ${tarballUrl}`], + stdout: notAvailable, + exitCode: 0, + }); + }, + ); + + test("a template that cannot be patched reports the cause on stderr", async () => { + // The specific failure is printed by the thread producing the executable, whose + // buffered stderr nothing else flushes; the build result only carries a generic error. + using dir = tempDir("build-compile-bad-template", { + "app.js": `console.log("hi");`, + "not-bun": "definitely not an executable\n", + "build.js": ` + const result = await Bun.build({ + entrypoints: [import.meta.dir + "/app.js"], + // A Linux target on every host, so it is always the ELF code that rejects the template. + compile: { target: "bun-linux-x64", executablePath: import.meta.dir + "/not-bun", outfile: "app" }, + throw: false, + }); + console.log(JSON.stringify({ success: result.success, errors: result.logs.length })); + `, + }); + + await using proc = Bun.spawn({ + cmd: [bunExe(), "build.js"], + cwd: String(dir), + env: bunEnv, + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + + expect({ stdout, stderr, exitCode, wroteOutfile: existsSync(join(String(dir), "app")) }).toEqual({ + stdout: '{"success":false,"errors":1}\n', + stderr: "Error initializing ELF file: InvalidElfFile\n", + exitCode: 0, + wroteOutfile: false, + }); + }); + }); + test("compile with embedded resources uses correct module prefix", async () => { using dir = tempDir("build-compile-embedded-resources", { "app.js": `