Make concurrent bun test and bun install processes safe on shared files

src/bun_core/env_var.rs

@@ -105,6 +105,11 @@ new!(pub BUN_INSTALL_STREAMING_MIN_SIZE: unsigned, "BUN_INSTALL_STREAMING_MIN_SI
 // thread schedules a drain; collapses the per-chunk thread-pool futex wake
 // into roughly one per `threshold` bytes.
 new!(pub BUN_INSTALL_STREAMING_DRAIN_THRESHOLD: unsigned, "BUN_INSTALL_STREAMING_DRAIN_THRESHOLD", { default: 256 * 1024 });
+// Set in the environment of the lifecycle scripts that a package manager process runs while it
+// holds the lock of the project at this path (`lock_project` in bun_install). A bun process that
+// such a script starts to edit the same project runs under its parent's lock instead of waiting
+// for it forever.
+new!(pub BUN_INTERNAL_INSTALL_LOCK_DIR: string, "BUN_INTERNAL_INSTALL_LOCK_DIR", {});
 new!(pub BUN_NEEDS_PROC_SELF_WORKAROUND: boolean, "BUN_NEEDS_PROC_SELF_WORKAROUND", { default: false });
 new!(pub BUN_OPTIONS: string, "BUN_OPTIONS", {});
 new!(pub BUN_POSTGRES_SOCKET_MONITOR: string, "BUN_POSTGRES_SOCKET_MONITOR", {});

src/install/PackageManager.rs

@@ -339,7 +339,9 @@ pub struct PackageManager {
     /// Only set in `bun pm`
     pub root_package_json_name_at_time_of_init: Box<[u8]>,
 
-    pub root_package_json_file: bun_sys::File,
+    /// The lock file of the project this process edits, held (see `lock_project`) until the
+    /// process exits. `None` while no lock is held.
+    pub(crate) project_lock: Option<bun_sys::File>,
 
     /// The package id corresponding to the workspace the install is happening in. Could be root, or
     /// could be any of the workspaces.
@@ -541,6 +543,24 @@ impl Subcommand {
     pub(crate) fn should_chdir_to_root(self) -> bool {
         !matches!(self, Self::Link)
     }
+
+    /// Subcommands that rewrite package.json, the lockfile or node_modules on every run, and so
+    /// hold the project lock (`lock_project`) from `init` on. `Pm` and `Audit` only write for
+    /// some of their arguments; those commands take the lock themselves.
+    pub(crate) fn always_edits_project(self) -> bool {
+        matches!(
+            self,
+            Self::Install
+                | Self::Update
+                | Self::Add
+                | Self::Remove
+                | Self::Link
+                | Self::Patch
+                | Self::PatchCommit
+                | Self::Dedupe
+                | Self::Prune
+        )
+    }
 }
 
 /// The resolved outcome of `--filter` for one install: the importer ids whose dependencies get installed.
@@ -1864,7 +1884,13 @@ pub fn init(
             root_buf[..p.len()].copy_from_slice(p);
             p.len()
         } else {
-            bun_sys::get_fd_path(root_package_json_file.handle, root_buf)?.len()
+            // The cwd is the project root now. Resolved by name rather than through the fd:
+            // package.json is replaced by rename (`write_file_atomically`), and once another
+            // process has done that, the fd's path names a deleted file.
+            match bun_sys::realpath(bun_core::zstr!("package.json"), root_buf) {
+                Ok(path) => path.len(),
+                Err(_) => bun_sys::get_fd_path(root_package_json_file.handle, root_buf)?.len(),
+            }
         };
         root_buf[plen] = 0;
         ROOT_PACKAGE_JSON_PATH.write(ZStr::from_raw(root_buf.as_ptr(), plen));
@@ -2061,7 +2087,7 @@ pub fn init(
         // zero-bit pattern is UB; allocate the real (empty) lockfile here directly.
         // `Lockfile::default()` ≡ `Lockfile::init_empty()`.
         wr!(lockfile, Box::new(Lockfile::default()));
-        wr!(root_package_json_file, root_package_json_file);
+        wr!(project_lock, None);
         // .progress
         wr!(event_loop, AnyEventLoop::init());
         wr!(
@@ -2248,6 +2274,13 @@ pub fn init(
                 manager.options.hoist_pattern = Some(p);
             }
         }
+
+        if subcommand.always_edits_project() && !manager.options.dry_run {
+            manager.lock_project();
+            // The workspace package.json files read while looking for the root above were read
+            // before the lock. Edits start from what is on disk now that this process holds it.
+            manager.workspace_package_json_cache.map.clear();
+        }
     }
 
     // Singleton fully initialized; main thread, no workers yet. Wrapped once as
@@ -2501,13 +2534,7 @@ fn init_with_runtime_once(
         // `Lockfile` holds `HashMap`/`Vec`/`NonNull` (zero-bit pattern is
         // UB), so allocate the real empty lockfile here directly instead of a zeroed placeholder.
         wr!(lockfile, Box::new(Lockfile::default()));
-        // `.root_package_json_file` is never read in the runtime
-        // path. Use the explicit invalid-fd sentinel rather than `mem::zeroed()` —
-        // on posix `Fd(0)` is stdin, not the invalid marker.
-        wr!(
-            root_package_json_file,
-            bun_sys::File::from_fd(Fd::invalid())
-        );
+        wr!(project_lock, None);
         // erased *mut () set by tier-6; `js_current()` resolves the per-thread JS
         // event loop via `bun_io::__bun_get_vm_ctx` (link-time, definer in bun_runtime).
         wr!(event_loop, AnyEventLoop::js_current());

src/install/PackageManager/PackageManagerDirectories.rs

@@ -65,6 +65,104 @@ impl PackageManager {
     pub(crate) fn get_temporary_directory(&mut self) -> &'static TemporaryDirectory {
         get_temporary_directory(self)
     }
+
+    #[inline]
+    pub fn lock_project(&mut self) {
+        lock_project(self)
+    }
+}
+
+// ───────────────────────────── project lock ───────────────────────────────────
+
+/// Makes the bun processes that edit one project run one at a time. Without it, `bun add` and
+/// `bun remove` started together each read package.json and bun.lock, and the one that writes
+/// last silently drops the other one's edit; two installs also undo each other's work while they
+/// place the same packages in node_modules. The lock is a file in the install cache directory,
+/// named after the project root, locked with `sys::flock` and held until
+/// this process exits. Lifecycle scripts get `BUN_INTERNAL_INSTALL_LOCK_DIR` so that a bun they
+/// run in the same project does not wait for this process.
+///
+/// Best effort: when the lock file cannot be created or locked, the command runs without it,
+/// as every command did before the lock existed. Calling this again is a no-op.
+pub fn lock_project(this: &mut PackageManager) {
+    if this.project_lock.is_some() {
+        return;
+    }
+    let project_dir =
+        bun_core::strings::without_trailing_slash(FileSystem::instance().top_level_dir());
+    if env_var::BUN_INTERNAL_INSTALL_LOCK_DIR
+        .get()
+        .is_some_and(|held| bun_core::strings::without_trailing_slash(held) == project_dir)
+    {
+        return;
+    }
+
+    let Some(lock_file) = open_project_lock_file(this, project_dir) else {
+        bun_core::scoped_log!(project_lock, "no lock for {}", bstr::BStr::new(project_dir));
+        return;
+    };
+    let locked = match sys::flock(lock_file.handle, sys::FileLockMode::Exclusive, true) {
+        Ok(true) => true,
+        Ok(false) => {
+            if !this.options.log_level.is_silent() {
+                bun_core::pretty_errorln!(
+                    "<d>Waiting for another bun process to finish in {}<r>",
+                    bstr::BStr::new(project_dir)
+                );
+                Output::flush();
+            }
+            matches!(
+                sys::flock(lock_file.handle, sys::FileLockMode::Exclusive, false),
+                Ok(true)
+            )
+        }
+        Err(_) => false,
+    };
+    if !locked {
+        bun_core::scoped_log!(
+            project_lock,
+            "could not lock {}",
+            bstr::BStr::new(project_dir)
+        );
+        return;
+    }
+
+    bun_core::handle_oom(
+        this.env_mut()
+            .map
+            .put(env_var::BUN_INTERNAL_INSTALL_LOCK_DIR.key(), project_dir),
+    );
+    this.project_lock = Some(lock_file);
+}
+
+bun_core::declare_scope!(project_lock, hidden);
+
+/// `<cache dir>/.locks/<hash of the project root>`. Opened read-only: `flock` does not need write
+/// access, so a lock file created by another user of a shared cache directory works too.
+fn open_project_lock_file(this: &mut PackageManager, project_dir: &[u8]) -> Option<File> {
+    let mut locks_dir_path = fetch_cache_directory_path(this.env_mut(), Some(&this.options)).path;
+    if locks_dir_path.last() != Some(&SEP) {
+        locks_dir_path.push(SEP);
+    }
+    locks_dir_path.extend_from_slice(b".locks");
+    let locks_dir = Dir::cwd()
+        .make_open_path(&locks_dir_path, Default::default())
+        .ok()?;
+
+    let mut lock_file_name = [0u8; b"0123456789abcdef.lock".len()];
+    write!(
+        &mut lock_file_name[..],
+        "{:016x}.lock",
+        bun_wyhash::hash(project_dir)
+    )
+    .expect("the buffer is sized for this format");
+    File::openat(
+        locks_dir.fd(),
+        &lock_file_name,
+        sys::O::RDONLY | sys::O::CREAT | sys::O::CLOEXEC,
+        0o644,
+    )
+    .ok()
 }
 
 // ───────────────────────────── cache directory ────────────────────────────────
@@ -1052,26 +1150,35 @@ pub fn compute_cache_dir_and_subpath<'a>(
 // ─────────────────────────── package.json / lockfile ──────────────────────────
 
 pub(crate) fn attempt_to_create_package_json_and_open() -> Result<File, Error> {
-    let package_json_file = match Dir::cwd().create_file_z(
-        z_static(b"package.json\0"),
-        sys::CreateFlags {
-            read: true,
-            ..Default::default()
-        },
+    let open_flags = sys::O::RDWR | sys::O::CLOEXEC;
+    let opened = match File::openat(
+        Fd::cwd(),
+        b"package.json",
+        open_flags | sys::O::CREAT | sys::O::EXCL,
+        0o666,
     ) {
-        Ok(f) => f,
+        Ok(package_json_file) => {
+            package_json_file.pwrite_all(b"{\"dependencies\": {}}", 0)?;
+            return Ok(package_json_file);
+        }
+        // Another bun process created it after this one found none. That process writes the
+        // initial contents; until it does, the empty file parses as `{}`.
+        Err(err) if err.get_errno() == sys::E::EEXIST => {
+            File::openat(Fd::cwd(), b"package.json", open_flags, 0)
+        }
+        Err(err) => Err(err),
+    };
+
+    match opened {
+        Ok(package_json_file) => Ok(package_json_file),
         Err(err) => {
             bun_core::pretty_errorln!(
                 "<r><red>error:<r> {} create package.json",
                 bun_fmt::s(err.name())
             );
             Global::crash();
         }
-    };
-
-    package_json_file.pwrite_all(b"{\"dependencies\": {}}", 0)?;
-
-    Ok(package_json_file)
+    }
 }
 
 pub fn attempt_to_create_package_json() -> Result<(), Error> {

src/install/PackageManager/add_remove_with_filter.rs

@@ -11,7 +11,7 @@ use bun_install::dependency;
 use bun_install::{Lockfile, PackageID, PackageNameHash};
 use bun_paths::path_buffer_pool;
 use bun_paths::resolve_path::{self, Platform, join_abs_string_buf, platform};
-use bun_sys::{Fd, File};
+use bun_sys::File;
 
 use super::add_catalog;
 use super::install_with_manager::install_with_manager;
@@ -295,7 +295,7 @@ pub(crate) fn write_target(manager: &mut PackageManager, target: &WorkspaceTarge
     let entry = fetch_entry(manager, target);
     let mut zbuf = path_buffer_pool::get();
     let path = resolve_path::z(&target.package_json_path, &mut zbuf);
-    match File::write_file(Fd::cwd(), path, &entry.source.contents) {
+    match File::write_file_atomically(path, &entry.source.contents, 0o644) {
         Ok(()) => true,
         Err(err) => {
             Output::err_generic(

src/install/PackageManager/updatePackageJSONAndInstall.rs

@@ -12,7 +12,7 @@ use bun_core::{Global, Output};
 use bun_core::{ZStr, strings};
 use bun_js_printer as js_printer;
 use bun_paths::{self, PathBuffer};
-use bun_sys::{self, Fd, File};
+use bun_sys::{self, File};
 
 use super::add_catalog;
 use super::add_remove_with_filter::WorkspaceTarget;
@@ -698,14 +698,7 @@ fn update_package_json_and_install_with_manager_with_updates(
 
         // Now that we've run the install step
         // We can save our in-memory package.json to disk
-        let workspace_package_json_file =
-            File::openat(Fd::cwd(), path, bun_sys::O::RDWR, 0).map_err(Error::from)?;
-
-        workspace_package_json_file
-            .pwrite_all(source, 0)
-            .map_err(Error::from)?;
-        let _ = bun_sys::ftruncate(workspace_package_json_file.handle, source.len() as i64);
-        let _ = workspace_package_json_file.close(); // close error is non-actionable
+        File::write_file_atomically(path, source, 0o644).map_err(Error::from)?;
 
         if subcommand == Subcommand::Remove {
             if !any_changes {

src/install/bin.rs

@@ -1268,7 +1268,7 @@ impl<'a> Linker<'a> {
 
         debug_assert!(strings::has_prefix(rel_target.as_bytes(), b".."));
 
-        match sys::symlink_running_executable(rel_target, abs_dest) {
+        match Self::symlink_or_keep(rel_target, abs_dest) {
             sys::Result::Err(err) => {
                 if err.get_errno() != sys::Errno::EEXIST && err.get_errno() != sys::Errno::ENOENT {
                     self.err = Some(err.into());
@@ -1291,7 +1291,7 @@ impl<'a> Linker<'a> {
                     let _ = sys::Dir::cwd().make_path(self.node_modules_path.slice());
                     self.node_modules_path.set_length(node_modules_path_save);
 
-                    match sys::symlink_running_executable(rel_target, abs_dest) {
+                    match Self::symlink_or_keep(rel_target, abs_dest) {
                         sys::Result::Err(real_error) => {
                             // It was just created, no need to delete destination and symlink again
                             self.err = Some(real_error.into());
@@ -1316,12 +1316,38 @@ impl<'a> Linker<'a> {
 
         // delete and try again
         let _ = sys::delete_tree_absolute(abs_dest.as_bytes());
-        if let Err(err) = sys::symlink_running_executable(rel_target, abs_dest) {
+        if let Err(err) = Self::symlink_or_keep(rel_target, abs_dest) {
             self.err = Some(err.into());
         }
         Self::chmod_on_ok(self.err, abs_target);
     }
 
+    /// A destination that already links to `rel_target` counts as linked: a
+    /// repeat install must not unlink and recreate it, and concurrent installs
+    /// into one directory (`bunx` runs one `bun add` per invocation in a shared
+    /// install dir) race to create the same link.
+    #[cfg(not(windows))]
+    fn symlink_or_keep(rel_target: &ZStr, abs_dest: &ZStr) -> sys::Result<()> {
+        match sys::symlink_running_executable(rel_target, abs_dest) {
+            Err(err)
+                if err.get_errno() == sys::Errno::EEXIST
+                    && Self::already_links_to(abs_dest, rel_target) =>
+            {
+                Ok(())
+            }
+            result => result,
+        }
+    }
+
+    #[cfg(not(windows))]
+    fn already_links_to(abs_dest: &ZStr, rel_target: &ZStr) -> bool {
+        let mut existing_target_buf = path::path_buffer_pool::get();
+        match sys::readlink(abs_dest, &mut existing_target_buf) {
+            Ok(len) => strings::eql(&existing_target_buf[..len], rel_target.as_bytes()),
+            Err(_) => false,
+        }
+    }
+
     #[cfg(not(windows))]
     fn chmod_on_ok(err: Option<Error>, abs_target: &ZStr) {
         // hoisted from `defer` block in create_symlink

src/install/migration.rs

@@ -100,7 +100,7 @@ pub fn detect_and_load_other_lockfile<'a>(
         let Ok(data) = File::read_from(dir, b"pnpm-lock.yaml") else {
             break 'pnpm;
         };
-        let migrate_result = match pnpm::migrate_pnpm_lockfile(this, manager, log, &data, dir) {
+        let migrate_result = match pnpm::migrate_pnpm_lockfile(this, manager, log, &data) {
             Ok(r) => r,
             Err(MigratePnpmLockfileError::PnpmLockfileTooOld) => {
                 report_unsupported_lockfile_version(