Skip to content
Open
Show file tree
Hide file tree
Changes from 6 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions src/install/PackageManager/patchPackage.rs
Original file line number Diff line number Diff line change
Expand Up @@ -371,6 +371,7 @@ pub fn do_patch_commit(
random_tempdir.as_bytes(),
sys::RenameOptions {
move_fallback: true,
..Default::default()
},
)
.is_err()
Expand Down Expand Up @@ -426,6 +427,7 @@ pub fn do_patch_commit(
patch_tag_tmpname.as_bytes(),
sys::RenameOptions {
move_fallback: true,
..Default::default()
},
) {
bun_core::warn!(
Expand Down Expand Up @@ -458,7 +460,7 @@ pub fn do_patch_commit(
random_tempdir.as_bytes(),
new_folder_handle.fd,
b"node_modules",
sys::RenameOptions { move_fallback: true },
sys::RenameOptions { move_fallback: true, ..Default::default() },
) {
bun_core::warn!("failed renaming nested node_modules folder, this may cause issues: {}", e);
}
Expand All @@ -470,7 +472,7 @@ pub fn do_patch_commit(
patch_tag_tmpname.as_bytes(),
new_folder_handle.fd,
patch_tag,
sys::RenameOptions { move_fallback: true },
sys::RenameOptions { move_fallback: true, ..Default::default() },
) {
bun_core::warn!("failed renaming the bun patch tag, this may cause issues: {}", e);
}
Expand Down Expand Up @@ -621,6 +623,7 @@ pub fn do_patch_commit(
path_in_patches_dir,
sys::RenameOptions {
move_fallback: true,
..Default::default()
},
) {
Output::err(e, "failed renaming patch file to patches dir", ());
Expand Down
80 changes: 47 additions & 33 deletions src/install/extract_tarball.rs
Original file line number Diff line number Diff line change
Expand Up @@ -258,6 +258,11 @@ impl ExtractTarball {
let mut resolved: &'static [u8] = b"";
let tmpname =
FileSystem::tmpname(tmpname_suffix, &mut tmpname_buf.0, bun_core::fast_random())?;
// Delete the temp dir if extraction fails before it's renamed into the
// cache; defused on success.
let tmpdir_cleanup = scopeguard::guard((), |()| {
let _ = Dir::borrow(&self.temp_dir).delete_tree(tmpname.as_bytes());
});
{
let extract_destination = match bun_sys::make_path::make_open_path(
tmpdir,
Expand Down Expand Up @@ -468,7 +473,11 @@ impl ExtractTarball {
}
}

self.move_to_cache_directory(log, tmpname, name, basename, resolved)
let result = self.move_to_cache_directory(log, tmpname, name, basename, resolved);
if result.is_ok() {
scopeguard::ScopeGuard::into_inner(tmpdir_cleanup);
}
result
}

/// Rename the freshly-extracted temp directory into the cache, read
Expand Down Expand Up @@ -544,6 +553,27 @@ impl ExtractTarball {
}
let cache_dir = Dir::borrow(&self.cache_dir);

// An existing npm cache entry without package.json is invalid (the
// same check `package_missing_from_cache` uses), so no concurrent
// install reads from it. Delete it so the fresh copy below replaces
// it instead of being kept as an equivalent existing destination.
if self.resolution.tag == ResolutionTag::Npm {
let mut folder_name_z_buf = PathBuffer::uninit();
folder_name_z_buf[..folder_name.len()].copy_from_slice(folder_name);
folder_name_z_buf[folder_name.len()] = 0;
let folder_name_z = ZStr::from_buf(&folder_name_z_buf, folder_name.len());
if sys::directory_exists_at(cache_dir.fd(), folder_name_z).unwrap_or(false) {
let mut json_buf = PathBuffer::uninit();
let json_z = path::resolve_path::join_z_buf::<path::platform::Auto>(
&mut json_buf.0,
&[folder_name, b"package.json"],
);
if !sys::exists_at(cache_dir.fd(), json_z) {
let _ = cache_dir.delete_tree(folder_name);
}
}
}

Comment thread
robobun marked this conversation as resolved.
// e.g. @next
// if it's a namespace package, we need to make sure the @name folder exists
let create_subdir = basename.len() != name.len() && !self.resolution.tag.is_git();
Expand Down Expand Up @@ -609,39 +639,22 @@ impl ExtractTarball {
| sys::Errno::PERM
| sys::Errno::BUSY
| sys::Errno::EXIST => {
// before we attempt to delete the destination, let's close the source dir.
let _ = sys::close(dir_to_move);

// We tried to move the folder over
// but it didn't work!
// so instead of just simply deleting the folder
// we rename it back into the temp dir
// and then delete that temp dir
// The goal is to make it more difficult for an application to reach this folder
let mut tempdest_buf = PathBuffer::uninit();
tempdest_buf[0..tmpname.len()]
.copy_from_slice(tmpname.as_bytes());
tempdest_buf[tmpname.len()..][0..4]
.copy_from_slice(&[b't', b'm', b'p', 0]);
let tempdest =
ZStr::from_buf(&tempdest_buf, tmpname.len() + 3);
let mut folder_name_z_buf = PathBuffer::uninit();
folder_name_z_buf[0..folder_name.len()]
.copy_from_slice(folder_name);
folder_name_z_buf[folder_name.len()] = 0;
let folder_name_z =
ZStr::from_buf(&folder_name_z_buf, folder_name.len());
match sys::renameat(
// A concurrent install sharing the cache published
// this entry first. Keep it (it may already have
// readers) and drop our equivalent copy instead of
// renaming it out from under them.
if sys::directory_exists_at_w(
Fd::from_std_dir(cache_dir),
folder_name_z,
Fd::from_std_dir(tmpdir),
tempdest,
) {
bun_sys::Result::Err(_) => {}
bun_sys::Result::Ok(_) => {
let _ = tmpdir.delete_tree(tempdest.as_bytes());
}
path_to_use,
)
.unwrap_or(false)
{
let _ = tmpdir.delete_tree(tmpname.as_bytes());
break;
}

retries += 1;
// 10ms, 20ms, 40ms, 80ms — long enough
// for a concurrent close to land,
Expand Down Expand Up @@ -683,9 +696,9 @@ impl ExtractTarball {
//
// By:
// 1. Rename from temporary directory to cache directory and fail if it already exists
// 2a. If the rename fails, swap the cache directory with the temporary directory version
// 2b. Delete the temporary directory version ONLY if we're not using a provided temporary directory
// 3. If rename still fails, fallback to racily deleting the cache directory version and then renaming the temporary directory version again.
// 2. If the rename fails because the destination exists, a concurrent install
// published an equivalent copy first: keep it and delete the temporary
// directory version (`keep_existing_destination`).
//

if create_subdir {
Expand All @@ -701,6 +714,7 @@ impl ExtractTarball {
folder_name,
sys::RenameatConcurrentlyOptions {
move_fallback: true,
keep_existing_destination: true,
},
) {
log.add_error_fmt(
Expand Down
8 changes: 7 additions & 1 deletion src/install/patch_install.rs
Original file line number Diff line number Diff line change
Expand Up @@ -452,6 +452,12 @@ impl PatchTask {

let system_tmpdir = self.tempdir;

// Delete the temp dir on failure; after a successful rename into the
// cache this is a no-op.
scopeguard::defer! {
let _ = sys::Dir::borrow(&system_tmpdir).delete_tree(tempdir_name.as_bytes());
}

let pkg_name = patch.pkgname;

let dummy_node_modules = crate::package_installer::NodeModulesFolder {
Expand Down Expand Up @@ -606,7 +612,7 @@ impl PatchTask {
cache_dir_subpath_z,
sys::RenameOptions {
move_fallback: true,
..Default::default()
keep_existing_destination: true,
},
) {
log.add_error_fmt_opts(
Expand Down
93 changes: 91 additions & 2 deletions src/sys/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9225,7 +9225,13 @@ pub fn exists(path: &[u8]) -> bool {
/// retries; on EXDEV falls back to the slow open+copy path. Only opens the
/// source inside the EXDEV branch.
pub fn move_file_z(from_dir: Fd, filename: &ZStr, to_dir: Fd, destination: &ZStr) -> Maybe<()> {
match renameat_concurrently_without_fallback(from_dir, filename, to_dir, destination) {
match renameat_concurrently_without_fallback(
from_dir,
filename,
to_dir,
destination,
Default::default(),
) {
Ok(()) => Ok(()),
// allow over-writing an empty directory
Err(e) if e.get_errno() == E::EISDIR => {
Expand Down Expand Up @@ -9302,6 +9308,12 @@ pub fn renameat_z(from_dir: impl AsFd, from: &ZStr, to_dir: impl AsFd, to: &ZStr
#[derive(Default, Clone, Copy)]
pub struct RenameatConcurrentlyOptions {
pub move_fallback: bool,
/// If the destination already exists (a concurrent process published an
/// equivalent tree first, e.g. a shared package cache entry), keep it and
/// delete the source instead of atomically replacing it. Replacing would
/// yank entries out from under processes still reading the existing tree,
/// and swapping it into the source leaks the temp directory (#33977).
pub keep_existing_destination: bool,
}
/// Alias: `bun_install` call sites spell this `RenameOptions`.
pub type RenameOptions = RenameatConcurrentlyOptions;
Expand All @@ -9328,7 +9340,7 @@ pub fn renameat_concurrently(
to: &ZStr,
opts: RenameatConcurrentlyOptions,
) -> Maybe<()> {
match renameat_concurrently_without_fallback(from_dir_fd, from, to_dir_fd, to) {
match renameat_concurrently_without_fallback(from_dir_fd, from, to_dir_fd, to, opts) {
Ok(()) => Ok(()),
Err(e) => {
if opts.move_fallback && e.get_errno() == E::EXDEV {
Expand All @@ -9348,7 +9360,15 @@ pub fn renameat_concurrently_without_fallback(
from: &ZStr,
to_dir_fd: Fd,
to: &ZStr,
opts: RenameatConcurrentlyOptions,
) -> Maybe<()> {
let delete_source = || {
if from_dir_fd.is_valid() {
let _ = Dir::borrow(&from_dir_fd).delete_tree(from.as_bytes());
} else {
let _ = delete_tree_absolute(from.as_bytes());
}
};
'attempt: {
{
// Happy path: the folder doesn't exist in the cache dir, so we can
Expand All @@ -9373,6 +9393,14 @@ pub fn renameat_concurrently_without_fallback(
Ok(()) => break 'attempt,
};

if opts.keep_existing_destination && matches!(err.get_errno(), E::EEXIST | E::ENOTEMPTY)
{
// Another process won the race; its tree is equivalent to
// ours and may already have readers, so drop our copy.
delete_source();
break 'attempt;
}
Comment thread
robobun marked this conversation as resolved.

// Windows doesn't have any equivalent of renameat with swap
#[cfg(not(windows))]
{
Expand Down Expand Up @@ -9401,6 +9429,21 @@ pub fn renameat_concurrently_without_fallback(
}

// sad path: let's try to delete the folder and then rename it
if opts.keep_existing_destination {
// The errno didn't tell us whether the destination exists (e.g.
// EOPNOTSUPP when the filesystem lacks RENAME_NOREPLACE); check
// before deleting a tree another process may be reading. Windows
// `exists_at` is file-only, so ask for the directory explicitly.
let dir_fd = if to_dir_fd.is_valid() {
to_dir_fd
} else {
Fd::cwd()
};
if directory_exists_at(dir_fd, to).unwrap_or(false) {
delete_source();
break 'attempt;
}
}
Comment thread
robobun marked this conversation as resolved.
if to_dir_fd.is_valid() {
let _ = Dir::borrow(&to_dir_fd).delete_tree(to.as_bytes());
} else {
Expand Down Expand Up @@ -9742,6 +9785,7 @@ mod owned_handle_tests {
b"sub",
RenameatConcurrentlyOptions {
move_fallback: true,
..Default::default()
},
)
.expect("rename");
Expand All @@ -9757,6 +9801,51 @@ mod owned_handle_tests {
let _ = close(root);
let _ = Dir::open(&tmp).map(|d| d.delete_tree(b"."));
}

/// With `keep_existing_destination`, losing the publish race keeps the
/// destination untouched and deletes the source instead of swapping the
/// two (which stranded the swapped-out tree in the temp dir, #33977).
#[test]
fn renameat_concurrently_keep_existing_destination() {
let _g = crate::file::tests::FD_TEST_LOCK.lock();
let mut tmp = std::env::temp_dir().as_os_str().as_encoded_bytes().to_vec();
tmp.extend_from_slice(b"/bun_sys_renameat_keep_test");
let _ = open_dir_at(Fd::cwd(), &tmp).map(close);
let _ = mkdir_recursive_at(Fd::cwd(), &tmp);
let root = open_dir_at(Fd::cwd(), &tmp).expect("open root");
let _ = mkdir_recursive_at(root, b"from/sub");
let _ = mkdir_recursive_at(root, b"to/sub");
let to_dir = open_dir_at(root, b"to").expect("open to");
File::write_file(root, ZStr::from_static(b"to/sub/winner\0"), b"").expect("marker");

renameat_concurrently_a(
root,
b"from/sub",
to_dir,
b"sub",
RenameatConcurrentlyOptions {
move_fallback: true,
keep_existing_destination: true,
},
)
.expect("rename");

// The existing destination survives with its contents...
assert!(
exists_at(to_dir, ZStr::from_static(b"sub/winner\0")),
"existing destination was replaced"
);
// ...and the source was cleaned up rather than left (or swapped)
// behind. Windows `exists_at` is file-only, so check the directory.
assert!(
!directory_exists_at(root, ZStr::from_static(b"from/sub\0")).unwrap_or(false),
"source left behind"
);
Comment thread
robobun marked this conversation as resolved.

let _ = close(to_dir);
let _ = close(root);
let _ = Dir::open(&tmp).map(|d| d.delete_tree(b"."));
}
}

#[cfg(all(test, windows))]
Expand Down
Loading
Loading