From c493822c47c44ff864788b3e1ca08177df48bb30 Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Fri, 5 Jun 2026 01:50:39 +0000 Subject: [PATCH] install: refresh URL/local tarballs under --force and explicit re-add/update URL and local tarballs are cached under a folder named from the URL/path hash (cached_tarball_folder_name), not from the tarball content. When the bytes behind the same URL/path changed, `bun install --force` copied the stale extraction into node_modules and never re-downloaded, so the code never updated. A forced re-download would also have tripped the lockfile-pinned integrity, which is why the only known workaround was clearing the cache and the lockfile by hand. Explicitly re-adding the URL (`bun i `) also no longer refreshed it: Bun 1.3 did, as a side effect of the duplicate-package.json-entry bug fixed for #30499, and users rely on that re-add-to-refresh behavior. For these content-mutable tarballs only (npm/git/github are content-addressed by version/commit): - treat the package as missing from the cache under --force, and also when the dependency was explicitly named on the command line (matched against update_requests), so a download/read task is enqueued (package_missing_from_cache, and the isolated installer's inline equivalent). Gated on the initial install-phase pass, on the preinstall state not already being Done, and on no tarball task for the same URL having been enqueued this run (bun update re-downloads during the resolve phase; re-enqueueing would push a callback into the already-drained task and the package would silently never install). - drop the lockfile-pinned integrity when building the ExtractTarball so the fresh bytes are accepted and the hash is recomputed from them. - persist the recomputed hash over the stale one after extraction, for both the hoisted and isolated installers. Tests in bun-install-tarball-integrity.test.ts cover both linkers for --force (url + local + first-install-no-lockfile), re-add via `bun i `, and `bun update `; they fail on the unfixed build. Fixes #31864 --- src/install/PackageInstall.rs | 17 + src/install/PackageInstaller.rs | 29 +- .../PackageManager/PackageManagerEnqueue.rs | 12 +- .../PackageManager/PackageManagerLifecycle.rs | 37 +- src/install/PackageManager/runTasks.rs | 47 ++- src/install/isolated_install.rs | 96 +++-- src/install/resolution.rs | 9 + .../bun-install-tarball-integrity.test.ts | 378 ++++++++++++++++-- 8 files changed, 569 insertions(+), 56 deletions(-) diff --git a/src/install/PackageInstall.rs b/src/install/PackageInstall.rs index 723488062e31..bef0df897b7a 100644 --- a/src/install/PackageInstall.rs +++ b/src/install/PackageInstall.rs @@ -2337,8 +2337,25 @@ impl<'a> PackageInstall<'a> { manager: &mut PackageManager, package_id: PackageID, resolution_tag: resolution::Tag, + force_refresh_tarball: bool, ) -> bool { let state = manager.get_preinstall_state(package_id); + // `force_refresh_tarball` is set by the caller when a URL/local tarball + // must re-fetch its bytes (`--force`, or the dependency was explicitly + // named on the command line): the cache folder is keyed by the URL/path + // hash, so an unchanged key hides changed content behind a stale + // extraction. Report it as missing so a download/read task is enqueued. + // + // Gated on the state not already being `Done`: once this run has + // downloaded and extracted the tarball, its state is `Done` (set after + // extraction) and the cache is fresh, so the package must install from + // cache rather than re-enqueue into an already-drained/deduped task. + // Without the `Done` guard, a first-time `--force` install (no lockfile) + // where the resolve phase fetched the tarball would re-enqueue and + // silently skip installing it. + if force_refresh_tarball && state != crate::PreinstallState::Done { + return true; + } match state { crate::PreinstallState::Done => false, _ => 'brk: { diff --git a/src/install/PackageInstaller.rs b/src/install/PackageInstaller.rs index d96dc00ba4d7..7818656cea1a 100644 --- a/src/install/PackageInstaller.rs +++ b/src/install/PackageInstaller.rs @@ -1492,7 +1492,33 @@ impl<'a> PackageInstaller<'a> { } } - let needs_install = self.force_install + // A URL/local tarball is re-fetched under `--force`, and also when the + // dependency was explicitly named on the command line (`bun i `, + // `bun update `): the bytes behind the same cache key may have + // changed. Only on the initial install-phase pass (`NEEDS_VERIFY`); the + // post-extraction callback installs from the freshly refreshed cache. + // Skipped when this run already fetched the tarball (the resolve phase + // re-downloads it when `bun update` invalidates the resolution) — the + // cache is fresh, so install from it instead of re-enqueueing into the + // already-drained task. + let force_refresh_tarball = NEEDS_VERIFY + && resolution.tag.is_tarball_cache_keyed_by_url() + && (self.force_install + || self + .manager_mut() + .dependency_is_update_request(dependency_id)) + && { + let url = match resolution.tag { + resolution::Tag::RemoteTarball => { + resolution.remote_tarball().slice(string_buf!()) + } + _ => resolution.local_tarball().slice(string_buf!()), + }; + !self.manager_mut().tarball_task_enqueued_this_run(url) + }; + + let needs_install = force_refresh_tarball + || self.force_install || self.skip_verify_installed_version_number || !NEEDS_VERIFY || remove_patch @@ -1505,6 +1531,7 @@ impl<'a> PackageInstaller<'a> { self.manager_mut(), package_id, resolution.tag, + force_refresh_tarball, ) { if cfg!(debug_assertions) { diff --git a/src/install/PackageManager/PackageManagerEnqueue.rs b/src/install/PackageManager/PackageManagerEnqueue.rs index 6ed604d71ba9..2f4b11e2940b 100644 --- a/src/install/PackageManager/PackageManagerEnqueue.rs +++ b/src/install/PackageManager/PackageManagerEnqueue.rs @@ -252,7 +252,17 @@ pub fn enqueue_tarball_for_reading( return; } - let integrity = this.lockfile.packages.items_meta()[package_id as usize].integrity; + // Under `--force`, or when the dependency was explicitly named on the + // command line, the tarball at this path may have changed, so drop the + // lockfile-pinned integrity: `ExtractTarball::run` recomputes it from the + // fresh bytes instead of rejecting them. See `enqueue_tarball_for_download`. + let integrity = if resolution.tag.is_tarball_cache_keyed_by_url() + && (this.options.enable.force_install() || this.dependency_is_update_request(dependency_id)) + { + Integrity::default() + } else { + this.lockfile.packages.items_meta()[package_id as usize].integrity + }; let task = enqueue_local_tarball( this, diff --git a/src/install/PackageManager/PackageManagerLifecycle.rs b/src/install/PackageManager/PackageManagerLifecycle.rs index 776bdb49c691..ed2971b68877 100644 --- a/src/install/PackageManager/PackageManagerLifecycle.rs +++ b/src/install/PackageManager/PackageManagerLifecycle.rs @@ -22,10 +22,12 @@ use crate::lifecycle_script_runner::{ }; use crate::lockfile_real::package::scripts::List as ScriptsList; use crate::package_manager_real::Command; +use crate::package_manager_task as PmTask; use crate::resolution_real::Tag as ResolutionTag; use bun_install::lockfile::{self, Lockfile, Package}; use bun_install::{ - PackageID, PackageManager, PreinstallState, TruncatedPackageNameHash, invalid_package_id, + DependencyID, PackageID, PackageManager, PreinstallState, TruncatedPackageNameHash, + invalid_package_id, }; #[derive(Default)] @@ -533,6 +535,39 @@ impl PackageManager { set } + + /// Whether `dependency_id` was explicitly named on the command line + /// (`bun add ` / `bun install ` / `bun update `). + /// URL/path arguments produce unnamed requests that match on the + /// dependency's version literal. + pub fn dependency_is_update_request(&self, dependency_id: DependencyID) -> bool { + if self.update_requests.is_empty() { + return false; + } + // `dependency_id` can be `invalid_dependency_id` (e.g. a root entry). + let Some(dep) = self + .lockfile + .buffers + .dependencies + .get(dependency_id as usize) + else { + return false; + }; + let string_buf = self.lockfile.buffers.string_bytes.as_slice(); + self.update_requests + .iter() + .any(|request| request.matches(dep, string_buf)) + } + + /// Whether a fetch/read task for this tarball URL/path was already + /// enqueued during this run (e.g. the resolve phase re-downloaded it + /// because `bun update` invalidated the resolution). The extract success + /// path drains the task's callback list but leaves the key in + /// `task_queue`, so the key's presence means the bytes were already + /// refreshed; re-enqueueing would push a callback nothing ever drains. + pub fn tarball_task_enqueued_this_run(&self, url: &[u8]) -> bool { + self.task_queue.contains(&PmTask::Id::for_tarball(url)) + } } fn add_dependencies_to_set( diff --git a/src/install/PackageManager/runTasks.rs b/src/install/PackageManager/runTasks.rs index aa74b85ee9ef..5a7381cbe72a 100644 --- a/src/install/PackageManager/runTasks.rs +++ b/src/install/PackageManager/runTasks.rs @@ -9,6 +9,7 @@ use bun_http::{self as http, AsyncHTTP}; use bun_threading::thread_pool::Batch as ThreadPoolBatch; use crate::extract_tarball; +use crate::integrity::Integrity; use crate::network_task::Callback as NetworkTaskCallback; use crate::npm; use crate::patch_install::{Callback as PatchTaskCallback, PatchTask}; @@ -1143,6 +1144,35 @@ pub fn run_tasks( bun_core::analytics::Features::extracted_packages_inc(); if C::HAS_ON_EXTRACT { + // When a re-fetch of a URL/local tarball was requested + // (`--force`, or the dependency was explicitly named on the + // command line), the bytes may have changed, so + // `ExtractTarball::run` recomputed the integrity from the fresh + // bytes (the stored pin was dropped before download). Persist it + // over the stale lockfile hash; otherwise a later cache-cleared + // install would reject the new content against the old pin. This + // is the install-phase path (hoisted + isolated), where + // `package_id` is already the final mapping and neither installer + // otherwise rewrites an already-resolved package's hash. The + // resolve phase below goes through + // `process_extracted_tarball_package`, which records the + // integrity on the final package itself. + if package_id != INVALID_PACKAGE_ID + && resolution.tag.is_tarball_cache_keyed_by_url() + && (manager.options.enable.force_install() + || manager.dependency_is_update_request(dependency_id)) + { + let new_integrity = task.data_extract().integrity; + if new_integrity.tag.is_supported() { + manager.lockfile.packages.items_meta_mut()[package_id as usize] + .integrity = new_integrity; + manager + .options + .enable + .set(Enable::FORCE_SAVE_LOCKFILE, true); + } + } + if C::IS_PACKAGE_INSTALLER { C::as_package_installer(extract_ctx).fix_cached_lockfile_package_slices(); C::on_extract_package_installer( @@ -1804,6 +1834,21 @@ pub fn generate_network_task_for_tarball<'a>( // so the task's drop never closes them. let cache_dir = directories::get_cache_directory(this); let temp_dir = directories::get_temporary_directory(this).handle.fd(); + // A URL/local tarball is re-fetched under `--force` or when its dependency + // was explicitly named on the command line, because its bytes may have + // changed at the same cache key. Verifying fresh bytes against the + // lockfile-pinned hash would reject the new content, so drop the stored + // integrity here: `ExtractTarball::run` recomputes it from the bytes and it + // is persisted back to the lockfile after extraction. Content-addressed + // resolutions (npm/git/github) keep their pinned hash. + let force_refresh_tarball = package.resolution.tag.is_tarball_cache_keyed_by_url() + && (this.options.enable.force_install() + || this.dependency_is_update_request(dependency_id)); + let integrity = if force_refresh_tarball { + Integrity::default() + } else { + package.meta.integrity + }; // Backref address only — stored, not dereffed in this function. The tag is // immediately popped by the next `this` use; that's fine for a stored // back-pointer. @@ -1845,7 +1890,7 @@ pub fn generate_network_task_for_tarball<'a>( dependency_id, skip_verify: false, in_trusted_dependencies: this.lockfile.in_trusted_dependencies(pkg_name), - integrity: package.meta.integrity, + integrity, url: strings::StringOrTinyString::init_append_if_needed( url, &mut crate::network_task::filename_store_appender(), diff --git a/src/install/isolated_install.rs b/src/install/isolated_install.rs index 2dac7d988071..ec85401521d6 100644 --- a/src/install/isolated_install.rs +++ b/src/install/isolated_install.rs @@ -2194,6 +2194,10 @@ pub(crate) fn install_isolated_packages( let uses_global_store = installer.entry_uses_global_store(entry_id); let needs_install = installer.manager().options.enable.force_install() + // A URL/local tarball named on the command line re-fetches + // its bytes; see `force_refresh_tarball` below. + || (pkg_res_tag.is_tarball_cache_keyed_by_url() + && installer.manager().dependency_is_update_request(dep_id)) // A freshly-created `node_modules/.bun` only implies the // *project-local* entries are missing; global virtual- // store entries persist across `rm -rf node_modules` and @@ -2342,41 +2346,75 @@ pub(crate) fn install_isolated_packages( installer.manager_mut().get_cache_directory_and_abs_path(); let _ = &cache_dir_path; // dropped at scope exit - let missing_from_cache = match installer.manager().get_preinstall_state(pkg_id) - { - install::PreinstallState::Done => false, - _ => 'missing_from_cache: { - if matches!(patch_info, installer::PatchInfo::None) { - let exists = match pkg_res_tag { - ResolutionTag::Npm => { - // Reshaped for borrowck — capture length - // instead of `save()` so the path stays unborrowed. - let cache_dir_path_save = pkg_cache_dir_subpath.len(); - pkg_cache_dir_subpath.append(b"package.json").assume_ok(); - let exists = sys::exists_at( + let preinstall_state = installer.manager().get_preinstall_state(pkg_id); + // `--force`, or explicitly naming the dependency on the command + // line, must re-fetch a URL/local tarball: its cache folder is + // keyed by URL/path hash, so changed content hides behind the + // same key. Treat it as missing so a download/read task is + // enqueued. Gated on the state not already being `Done`: once this + // run has fetched+extracted the tarball the cache is fresh, so it + // must install from cache rather than re-enqueue. Mirrors + // `PackageInstall::package_missing_from_cache`. + let force_refresh_tarball = preinstall_state != install::PreinstallState::Done + && pkg_res_tag.is_tarball_cache_keyed_by_url() + && (installer.manager().options.enable.force_install() + || installer.manager().dependency_is_update_request(dep_id)) + && { + // Skip when this run already fetched the tarball (a task + // for it is already in the queue, e.g. from the resolve + // phase after `bun update` invalidated the resolution) -- + // the cache is fresh; install from it rather than + // re-enqueue into the already-drained task. + let url = if pkg_res_tag == ResolutionTag::RemoteTarball { + pkg_res.remote_tarball().slice(string_buf) + } else { + pkg_res.local_tarball().slice(string_buf) + }; + !installer.manager().tarball_task_enqueued_this_run(url) + }; + let missing_from_cache = if force_refresh_tarball { + true + } else { + match preinstall_state { + install::PreinstallState::Done => false, + _ => 'missing_from_cache: { + if matches!(patch_info, installer::PatchInfo::None) { + let exists = match pkg_res_tag { + ResolutionTag::Npm => { + // Reshaped for borrowck — capture length + // instead of `save()` so the path stays unborrowed. + let cache_dir_path_save = pkg_cache_dir_subpath.len(); + pkg_cache_dir_subpath + .append(b"package.json") + .assume_ok(); + let exists = sys::exists_at( + cache_dir, + pkg_cache_dir_subpath.slice_z(), + ); + pkg_cache_dir_subpath.set_length(cache_dir_path_save); + exists + } + _ => sys::directory_exists_at( cache_dir, pkg_cache_dir_subpath.slice_z(), + ) + .unwrap_or(false), + }; + if exists { + installer.manager_mut().set_preinstall_state( + pkg_id, + install::PreinstallState::Done, ); - pkg_cache_dir_subpath.set_length(cache_dir_path_save); - exists } - _ => sys::directory_exists_at( - cache_dir, - pkg_cache_dir_subpath.slice_z(), - ) - .unwrap_or(false), - }; - if exists { - installer.manager_mut().set_preinstall_state( - pkg_id, - install::PreinstallState::Done, - ); + break 'missing_from_cache !exists; } - break 'missing_from_cache !exists; - } - // TODO: why does this look like it will never work? - break 'missing_from_cache true; + // Patched packages: the non-`None` patch_info + // branch unconditionally reports missing (ported + // as-is from isolated_install.zig; its + // effectiveness is an open question upstream). + break 'missing_from_cache true; + } } }; diff --git a/src/install/resolution.rs b/src/install/resolution.rs index 467573105257..4faf7f329e1a 100644 --- a/src/install/resolution.rs +++ b/src/install/resolution.rs @@ -966,6 +966,15 @@ impl Tag { self == Tag::Git || self == Tag::Github } + /// Tarballs whose cache folder is keyed by their URL/path hash rather than + /// by content (`cached_tarball_folder_name`). The bytes behind a URL or a + /// local path can change while the key stays the same, so `--force` must be + /// able to re-fetch and re-extract them. Npm/git/github resolutions are + /// content-addressed by version/commit and never need this. + pub fn is_tarball_cache_keyed_by_url(self) -> bool { + self == Tag::RemoteTarball || self == Tag::LocalTarball + } + pub fn can_enqueue_install_task(self) -> bool { self == Tag::Npm || self == Tag::LocalTarball diff --git a/test/cli/install/bun-install-tarball-integrity.test.ts b/test/cli/install/bun-install-tarball-integrity.test.ts index 98bc78c8e5a3..773351a926f1 100644 --- a/test/cli/install/bun-install-tarball-integrity.test.ts +++ b/test/cli/install/bun-install-tarball-integrity.test.ts @@ -17,6 +17,31 @@ import { setDefaultTimeout(1000 * 60 * 5); +// Minimal ustar tarball builders shared by the hand-rolled-tarball tests below. +function octal(n: number, width: number) { + return n.toString(8).padStart(width - 1, "0") + "\0"; +} +function tarHeader(name: string, size: number) { + const buf = Buffer.alloc(512, 0); + buf.write(name, 0, 100, "utf8"); + buf.write(octal(0o644, 8), 100); + buf.write(octal(0, 8), 108); + buf.write(octal(0, 8), 116); + buf.write(octal(size, 12), 124); + buf.write(octal(0, 12), 136); + buf.fill(" ", 148, 156); + buf.write("0", 156); + buf.write("ustar\0", 257); + buf.write("00", 263); + let sum = 0; + for (let i = 0; i < 512; i++) sum += buf[i]; + buf.write(octal(sum, 8), 148); + return buf; +} +function pad512(len: number) { + return Buffer.alloc((512 - (len % 512)) % 512, 0); +} + beforeAll(() => { dummyBeforeAll(); }); @@ -506,29 +531,6 @@ describe.concurrent.each(["hoisted", "isolated"] as const)("tarball integrity mi // callback is the void `onPackageDownloadError = {}` — i.e. the branch the // fix in runTasks.zig now cleans up. it("should fail (not hang) when tarball bytes don't match manifest SHA-512", { timeout: 60_000 }, async () => { - function octal(n: number, width: number) { - return n.toString(8).padStart(width - 1, "0") + "\0"; - } - function tarHeader(name: string, size: number) { - const buf = Buffer.alloc(512, 0); - buf.write(name, 0, 100, "utf8"); - buf.write(octal(0o644, 8), 100); - buf.write(octal(0, 8), 108); - buf.write(octal(0, 8), 116); - buf.write(octal(size, 12), 124); - buf.write(octal(0, 12), 136); - buf.fill(" ", 148, 156); - buf.write("0", 156); - buf.write("ustar\0", 257); - buf.write("00", 263); - let sum = 0; - for (let i = 0; i < 512; i++) sum += buf[i]; - buf.write(octal(sum, 8), 148); - return buf; - } - function pad512(len: number) { - return Buffer.alloc((512 - (len % 512)) % 512, 0); - } function buildTarball(body: Buffer) { const tar = Buffer.concat([ tarHeader("package/package.json", body.length), @@ -845,3 +847,333 @@ describe.concurrent.each(["hoisted", "isolated"] as const)("tarball download fai }); }); }); + +describe.concurrent.each(["hoisted", "isolated"] as const)("tarball --force refresh (%s)", linker => { + // https://github.com/oven-sh/bun/issues/31864 — URL/local tarballs are cached + // under a folder named from the URL/path hash, not the content. When the bytes + // behind the same URL changed, `bun install --force` copied the stale + // extraction into node_modules and never re-downloaded, so the code never + // updated. A forced re-download would also have tripped the lockfile-pinned + // integrity, hence the reporter having to clear the cache + lockfile by hand. + // One-package tarball whose package.json and index.js carry `marker`, so the + // installed content can be asserted byte-for-byte. + function buildTarball(marker: string) { + const files: Array<[string, Buffer]> = [ + ["package/package.json", Buffer.from(JSON.stringify({ name: "my-url-pkg", version: "1.0.0" }) + "\n")], + ["package/index.js", Buffer.from(`module.exports = ${JSON.stringify(marker)};\n`)], + ]; + const parts: Buffer[] = []; + for (const [name, body] of files) { + parts.push(tarHeader(name, body.length), body, pad512(body.length)); + } + parts.push(Buffer.alloc(1024, 0)); + const tgz = gzipSync(Buffer.concat(parts)); + return { tgz, integrity: "sha512-" + createHash("sha512").update(tgz).digest("base64") }; + } + + it("re-downloads the changed tarball instead of reusing the stale cache", async () => { + const v1 = buildTarball("VERSION_ONE"); + const v2 = buildTarball("VERSION_TWO"); + expect(v1.integrity).not.toBe(v2.integrity); + + // Same URL serves v1 until `serveV2` flips, then v2. Track every tarball + // request so we can prove `--force` actually hit the network again. + let serveV2 = false; + const tarballRequests: string[] = []; + await using server = Bun.serve({ + port: 0, + hostname: "127.0.0.1", + fetch(req) { + const url = new URL(req.url); + if (url.pathname.endsWith("/my-url-pkg.tgz")) { + tarballRequests.push(serveV2 ? "v2" : "v1"); + const { tgz } = serveV2 ? v2 : v1; + return new Response(tgz, { headers: { "content-length": String(tgz.length) } }); + } + return new Response("Not found", { status: 404 }); + }, + }); + const tarballUrl = `http://127.0.0.1:${server.port}/my-url-pkg.tgz`; + + using dir = tempDir("issue-31864-" + linker, { + "package.json": JSON.stringify({ + name: "app", + version: "1.0.0", + dependencies: { "my-url-pkg": tarballUrl }, + }), + "bunfig.toml": `[install]\nlinker = "${linker}"\n`, + }); + + const installedIndex = join(String(dir), "node_modules", "my-url-pkg", "index.js"); + const cacheDir = join(String(dir), ".cache"); + const spawnOpts = { + cwd: String(dir), + env: { ...env, BUN_INSTALL_CACHE_DIR: cacheDir }, + stdout: "pipe" as const, + stderr: "pipe" as const, + }; + + // First install: serves v1 and populates the URL-hash cache folder. + { + await using proc = spawn({ cmd: [bunExe(), "install"], ...spawnOpts }); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + expect(stderr).not.toContain("error:"); + expect(stdout + stderr).not.toContain("Integrity check failed"); + expect(exitCode).toBe(0); + } + expect(await file(installedIndex).text()).toBe('module.exports = "VERSION_ONE";\n'); + expect(tarballRequests).toEqual(["v1"]); + + // Swap the bytes served at the same URL, then force a reinstall. Before the + // fix, `--force` copied the stale extraction and never re-requested the + // tarball, so node_modules stayed on VERSION_ONE. + serveV2 = true; + tarballRequests.length = 0; + { + await using proc = spawn({ cmd: [bunExe(), "install", "--force"], ...spawnOpts }); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + expect(stderr).not.toContain("error:"); + // A naive re-download without recomputing integrity would reject v2 + // against v1's lockfile-pinned hash. + expect(stdout + stderr).not.toContain("Integrity check failed"); + expect(exitCode).toBe(0); + } + + expect(tarballRequests).toEqual(["v2"]); + expect(await file(installedIndex).text()).toBe('module.exports = "VERSION_TWO";\n'); + + // The lockfile integrity should now match v2, so a later cache-cleared + // install of the current bytes does not fail the integrity check. + const lockContent = await file(join(String(dir), "bun.lock")).text(); + expect(lockContent).toContain(v2.integrity); + expect(lockContent).not.toContain(v1.integrity); + }); + + it("re-reads a changed local tarball at the same path", async () => { + const v1 = buildTarball("VERSION_ONE"); + const v2 = buildTarball("VERSION_TWO"); + expect(v1.integrity).not.toBe(v2.integrity); + + // Local tarballs are cached by the same path-hash key as URLs, so the same + // bug applies: overwriting the file at the same path must still refresh + // under `--force`. + using dir = tempDir("issue-31864-local-" + linker, { + "package.json": JSON.stringify({ + name: "app", + version: "1.0.0", + dependencies: { "my-url-pkg": "./pkg.tgz" }, + }), + "bunfig.toml": `[install]\nlinker = "${linker}"\n`, + }); + const tgzPath = join(String(dir), "pkg.tgz"); + await writeFile(tgzPath, v1.tgz); + + const installedIndex = join(String(dir), "node_modules", "my-url-pkg", "index.js"); + const spawnOpts = { + cwd: String(dir), + env: { ...env, BUN_INSTALL_CACHE_DIR: join(String(dir), ".cache") }, + stdout: "pipe" as const, + stderr: "pipe" as const, + }; + + { + await using proc = spawn({ cmd: [bunExe(), "install"], ...spawnOpts }); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + expect(stderr).not.toContain("error:"); + expect(stdout + stderr).not.toContain("Integrity check failed"); + expect(exitCode).toBe(0); + } + expect(await file(installedIndex).text()).toBe('module.exports = "VERSION_ONE";\n'); + + // Overwrite the tarball at the same path, then force a reinstall. + await writeFile(tgzPath, v2.tgz); + { + await using proc = spawn({ cmd: [bunExe(), "install", "--force"], ...spawnOpts }); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + expect(stderr).not.toContain("error:"); + expect(stdout + stderr).not.toContain("Integrity check failed"); + expect(exitCode).toBe(0); + } + + expect(await file(installedIndex).text()).toBe('module.exports = "VERSION_TWO";\n'); + + const lockContent = await file(join(String(dir), "bun.lock")).text(); + expect(lockContent).toContain(v2.integrity); + expect(lockContent).not.toContain(v1.integrity); + }); + + it("installs the tarball on a first --force install with no lockfile", async () => { + // `--force` as the very first install (no bun.lock): the resolve phase + // downloads+extracts the tarball and marks it done, so the install phase must + // copy the fresh cache into node_modules. Without a `Done` guard on the + // force-refresh cache-miss path, the install phase re-enqueued into the + // already-drained task and silently skipped installing the package — the run + // reported success with an empty node_modules. + const v1 = buildTarball("VERSION_ONE"); + await using server = Bun.serve({ + port: 0, + hostname: "127.0.0.1", + fetch(req) { + if (new URL(req.url).pathname.endsWith("/my-url-pkg.tgz")) { + return new Response(v1.tgz, { headers: { "content-length": String(v1.tgz.length) } }); + } + return new Response("Not found", { status: 404 }); + }, + }); + + using dir = tempDir("issue-31864-fresh-" + linker, { + "package.json": JSON.stringify({ + name: "app", + version: "1.0.0", + dependencies: { "my-url-pkg": `http://127.0.0.1:${server.port}/my-url-pkg.tgz` }, + }), + "bunfig.toml": `[install]\nlinker = "${linker}"\n`, + }); + + await using proc = spawn({ + cmd: [bunExe(), "install", "--force"], + cwd: String(dir), + env: { ...env, BUN_INSTALL_CACHE_DIR: join(String(dir), ".cache") }, + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + expect(stderr).not.toContain("error:"); + expect(exitCode).toBe(0); + expect(await file(join(String(dir), "node_modules", "my-url-pkg", "index.js")).text()).toBe( + 'module.exports = "VERSION_ONE";\n', + ); + }); + + it("re-adding the URL on the command line refreshes it without --force", async () => { + // Bun 1.3 refreshed a URL tarball whenever it was explicitly named on the + // command line (`bun i `), with no --force. Naming the dep makes it an + // update request, which re-fetches the bytes at the same URL. + const v1 = buildTarball("VERSION_ONE"); + const v2 = buildTarball("VERSION_TWO"); + + let serveV2 = false; + const tarballRequests: string[] = []; + await using server = Bun.serve({ + port: 0, + hostname: "127.0.0.1", + fetch(req) { + if (new URL(req.url).pathname.endsWith("/my-url-pkg.tgz")) { + tarballRequests.push(serveV2 ? "v2" : "v1"); + const { tgz } = serveV2 ? v2 : v1; + return new Response(tgz, { headers: { "content-length": String(tgz.length) } }); + } + return new Response("Not found", { status: 404 }); + }, + }); + const tarballUrl = `http://127.0.0.1:${server.port}/my-url-pkg.tgz`; + + using dir = tempDir("issue-31864-readd-" + linker, { + "package.json": JSON.stringify({ name: "app", version: "1.0.0" }), + "bunfig.toml": `[install]\nlinker = "${linker}"\n`, + }); + const installedIndex = join(String(dir), "node_modules", "my-url-pkg", "index.js"); + const spawnOpts = { + cwd: String(dir), + env: { ...env, BUN_INSTALL_CACHE_DIR: join(String(dir), ".cache") }, + stdout: "pipe" as const, + stderr: "pipe" as const, + }; + + // First add: installs v1, writes package.json + bun.lock. + { + await using proc = spawn({ cmd: [bunExe(), "install", tarballUrl], ...spawnOpts }); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + expect(stderr).not.toContain("error:"); + expect(stdout + stderr).not.toContain("Integrity check failed"); + expect(exitCode).toBe(0); + } + expect(await file(installedIndex).text()).toBe('module.exports = "VERSION_ONE";\n'); + expect(tarballRequests).toEqual(["v1"]); + + // Swap the bytes at the same URL, then re-add the same URL (no --force). + serveV2 = true; + tarballRequests.length = 0; + { + await using proc = spawn({ cmd: [bunExe(), "install", tarballUrl], ...spawnOpts }); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + expect(stderr).not.toContain("error:"); + expect(stdout + stderr).not.toContain("Integrity check failed"); + expect(exitCode).toBe(0); + } + + expect(tarballRequests).toEqual(["v2"]); + expect(await file(installedIndex).text()).toBe('module.exports = "VERSION_TWO";\n'); + + // package.json must not grow a second, URL-keyed entry (#30499), and the + // lockfile integrity must now pin v2. + const pkg = JSON.parse(await file(join(String(dir), "package.json")).text()); + expect(Object.keys(pkg.dependencies)).toEqual(["my-url-pkg"]); + const lockContent = await file(join(String(dir), "bun.lock")).text(); + expect(lockContent).toContain(v2.integrity); + expect(lockContent).not.toContain(v1.integrity); + }); + + it("bun update refreshes a changed URL tarball into node_modules", async () => { + // `bun update` invalidates the resolution, so the resolve phase re-fetches + // the tarball; the install phase must then copy the fresh extraction into + // node_modules rather than re-enqueueing into the already-drained task. + const v1 = buildTarball("VERSION_ONE"); + const v2 = buildTarball("VERSION_TWO"); + + let serveV2 = false; + const tarballRequests: string[] = []; + await using server = Bun.serve({ + port: 0, + hostname: "127.0.0.1", + fetch(req) { + if (new URL(req.url).pathname.endsWith("/my-url-pkg.tgz")) { + tarballRequests.push(serveV2 ? "v2" : "v1"); + const { tgz } = serveV2 ? v2 : v1; + return new Response(tgz, { headers: { "content-length": String(tgz.length) } }); + } + return new Response("Not found", { status: 404 }); + }, + }); + const tarballUrl = `http://127.0.0.1:${server.port}/my-url-pkg.tgz`; + + using dir = tempDir("issue-31864-update-" + linker, { + "package.json": JSON.stringify({ + name: "app", + version: "1.0.0", + dependencies: { "my-url-pkg": tarballUrl }, + }), + "bunfig.toml": `[install]\nlinker = "${linker}"\n`, + }); + const installedIndex = join(String(dir), "node_modules", "my-url-pkg", "index.js"); + const spawnOpts = { + cwd: String(dir), + env: { ...env, BUN_INSTALL_CACHE_DIR: join(String(dir), ".cache") }, + stdout: "pipe" as const, + stderr: "pipe" as const, + }; + + { + await using proc = spawn({ cmd: [bunExe(), "install"], ...spawnOpts }); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + expect(stderr).not.toContain("error:"); + expect(stdout + stderr).not.toContain("Integrity check failed"); + expect(exitCode).toBe(0); + } + expect(await file(installedIndex).text()).toBe('module.exports = "VERSION_ONE";\n'); + expect(tarballRequests).toEqual(["v1"]); + + serveV2 = true; + tarballRequests.length = 0; + { + await using proc = spawn({ cmd: [bunExe(), "update", "my-url-pkg"], ...spawnOpts }); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + expect(stderr).not.toContain("error:"); + expect(stdout + stderr).not.toContain("Integrity check failed"); + expect(exitCode).toBe(0); + } + + expect(tarballRequests).toEqual(["v2"]); + expect(await file(installedIndex).text()).toBe('module.exports = "VERSION_TWO";\n'); + }); +});