From 02d0c07e763d9886b106827f02b1520c35051d86 Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Fri, 14 Aug 2026 17:57:23 +0000 Subject: [PATCH] Blob: append onto the first part's store instead of copying it new Blob([blob, ...parts]) copied the leading Blob's bytes into a fresh store, so the accumulation idiom b = new Blob([b, chunk]) re-copied the whole prefix on every step and cost O(total^2). Blobs built this way now share one AppendBuffer: an allocation with spare capacity that every store produced by successive appends points into. Each store is still an ordinary immutable Bytes viewing a prefix of the buffer; an append claims the tail past the longest published prefix with a CAS, writes only the new parts there and publishes a new store. When the buffer is full the next append allocates a new one with 50% headroom, so the bytes copied stay linear in the final size. The buffer travels in Bytes.allocator like LinuxMemFdAllocator does and is released when the last store built on it is dropped. Because several stores can now view one allocation, the zero-copy transfer of a store's bytes into a writable ArrayBuffer additionally requires that no other store is built on the same buffer, and to_internal_blob takes the allocation over only when the buffer has a single store (copying otherwise, as it does for memfd-backed stores). --- bench/snippets/blob-append.mjs | 22 ++ src/jsc/webcore_types.rs | 24 +- src/runtime/webcore/Blob.rs | 34 ++- src/runtime/webcore/blob/AppendBuffer.rs | 285 +++++++++++++++++++++++ src/runtime/webcore/blob/Store.rs | 9 +- test/js/web/fetch/blob.test.ts | 259 +++++++++++++++++++- 6 files changed, 625 insertions(+), 8 deletions(-) create mode 100644 bench/snippets/blob-append.mjs create mode 100644 src/runtime/webcore/blob/AppendBuffer.rs diff --git a/bench/snippets/blob-append.mjs b/bench/snippets/blob-append.mjs new file mode 100644 index 000000000000..f022bf19c8dd --- /dev/null +++ b/bench/snippets/blob-append.mjs @@ -0,0 +1,22 @@ +import { bench, run } from "../runner.mjs"; + +// Accumulating into a Blob by re-wrapping it. Each case builds the whole chain, +// so the per-iteration cost should grow linearly with the number of chunks. +const chunk64KiB = new Uint8Array(64 * 1024); +const chunk64B = new Uint8Array(64); + +function accumulate(chunk, count) { + let blob = new Blob([chunk]); + for (let i = 1; i < count; i++) blob = new Blob([blob, chunk]); + return blob; +} + +bench("b = new Blob([b, 64 KiB chunk]) x 64", () => accumulate(chunk64KiB, 64)); +bench("b = new Blob([b, 64 KiB chunk]) x 256", () => accumulate(chunk64KiB, 256)); +bench("b = new Blob([b, 64 B chunk]) x 1024", () => accumulate(chunk64B, 1024)); +bench("b = new Blob([b, 64 B chunk]) x 4096", () => accumulate(chunk64B, 4096)); + +const oneMiB = new Blob([new Uint8Array(1024 * 1024)]); +bench("new Blob([1 MiB blob, 64 B chunk]) once", () => new Blob([oneMiB, chunk64B])); + +await run(); diff --git a/src/jsc/webcore_types.rs b/src/jsc/webcore_types.rs index b3a6a52daae4..f74a93f8e77e 100644 --- a/src/jsc/webcore_types.rs +++ b/src/jsc/webcore_types.rs @@ -606,6 +606,14 @@ pub mod store { /// rather than `Vec` so the memfd-backed path /// (`LinuxMemFdAllocator::create` → `mmap`'d region freed via `munmap`) /// can carry its allocator vtable with the buffer. + /// + /// `ptr[..len]` is immutable for as long as this `Bytes` exists. The + /// allocation behind it is not necessarily this value's alone: the stores + /// that `bun_runtime`'s `AppendBuffer` builds are several `Bytes` viewing + /// prefixes of one allocation (their `allocator` keeps it alive), so only + /// `allocator.free` may be assumed about the memory past what `slice()` + /// returns, and writing through `ptr` additionally needs the allocation to + /// be exclusively this value's (see `as_array_list_leak`). pub struct Bytes { pub ptr: Option>, pub len: SizeType, @@ -616,11 +624,12 @@ pub mod store { pub stored_name: Box<[u8]>, } - // SAFETY: `Bytes` is morally `Vec`-with-custom-free. The raw - // `NonNull` is uniquely owned (`ptr` is the sole alias) and + // SAFETY: `Bytes` is morally `Vec`-with-custom-free. It never writes + // through `ptr` itself, other `Bytes` sharing the allocation (see the type + // doc) only read it too, the allocator's `free` is thread-safe, and // `StdAllocator` is `Send + Sync`. unsafe impl Send for Bytes {} - // SAFETY: `&Bytes` only reads the uniquely-owned slice via `slice()`; no + // SAFETY: `&Bytes` only reads the immutable `ptr[..len]` via `slice()`; no // interior mutability, so sharing references across threads is sound. unsafe impl Sync for Bytes {} @@ -753,9 +762,16 @@ pub mod store { self.as_array_list_leak() } + /// Writing through the result (or handing it out writable, as the + /// `Lifetime::Transfer` ArrayBuffer path does) is only sound when no + /// other `Bytes` shares the allocation: a freshly created store, or a + /// store that `AppendBuffer::shares_allocation` clears. Everything + /// else only reads through it. pub fn as_array_list_leak(&mut self) -> &mut [u8] { match self.ptr { - // SAFETY: `ptr[..len]` is live and uniquely owned by `*self`. + // SAFETY: `ptr[..len]` is live for as long as `*self` is, and + // `&mut self` rules out other users of this value; see the doc + // comment for what writers must additionally ensure. Some(p) => unsafe { core::slice::from_raw_parts_mut(p.as_ptr(), self.len as usize) }, diff --git a/src/runtime/webcore/Blob.rs b/src/runtime/webcore/Blob.rs index fc94eefd027d..224fc7225f73 100644 --- a/src/runtime/webcore/Blob.rs +++ b/src/runtime/webcore/Blob.rs @@ -52,6 +52,9 @@ use crate::node::types::{PathLikeExt as _, PathOrFdExt as _}; use store::{BytesExt as _, FileExt as _, S3Ext as _, StoreExt as _}; pub use store::{Store, StoreRef}; +#[path = "blob/AppendBuffer.rs"] +pub(crate) mod append_buffer; +use append_buffer::AppendBuffer; #[path = "blob/copy_file.rs"] pub mod copy_file; #[cfg(not(windows))] @@ -3035,7 +3038,12 @@ impl BlobExt for Blob { } } Lifetime::Transfer => { - if self.store().is_some_and(|s| !s.has_one_ref()) { + // The bytes become a writable ArrayBuffer, so besides being the + // store's only holder nothing else may be able to read them. + if self + .store() + .is_some_and(|s| !s.has_one_ref() || AppendBuffer::shares_allocation(s)) + { // SAFETY: same `buf` contract as the caller; the `Clone` arm only reads it. let copied = unsafe { self.to_array_buffer_view_with_bytes::<{ Lifetime::Clone }, TYPED_ARRAY_VIEW>( @@ -3344,6 +3352,11 @@ impl BlobExt for Blob { let mut stack: Vec = Vec::new(); let mut joiner = bun_core::string_joiner::StringJoiner::default(); let mut could_have_non_ascii = false; + // Store of a Blob part that opens the result (`new Blob([blob, ...])`). + // Its bytes stay out of `joiner` and are appended onto instead of + // copied; see `AppendBuffer`. Held as a ref so user JS run by a later + // part cannot release it. + let mut append_prefix: Option = None; loop { match current.js_type_loose() { @@ -3436,6 +3449,12 @@ impl BlobExt for Blob { if let Some(blob) = item.as_class_ref::() { could_have_non_ascii = could_have_non_ascii || blob.charset.get() != strings::AsciiStatus::AllAscii; + if append_prefix.is_none() && joiner.len == 0 { + if let Some(store) = AppendBuffer::prefix_store(blob) { + append_prefix = Some(store.clone()); + continue; + } + } // A later part may run user JS that drops the // last ref to this Blob's Store before `done()`. if parts_can_run_js { @@ -3510,6 +3529,19 @@ impl BlobExt for Blob { }; } + if let Some(prefix) = append_prefix { + // As below, only a positive ASCII answer is recorded. + let is_all_ascii = (!could_have_non_ascii).then_some(true); + // `joiner` still holds the remaining parts (borrowed ones are kept + // alive by `_keep`/`arg` exactly as for `done()`); `concat` reads + // them in place and dropping the joiner frees the owned ones. + let store = AppendBuffer::concat(&prefix, &joiner, is_all_ascii); + let blob = Blob::init_with_store(store, global); + blob.charset + .set(strings::AsciiStatus::from_bool(is_all_ascii)); + return Ok(blob); + } + let joined: Vec = joiner.done().expect("oom").into_vec(); if !could_have_non_ascii { diff --git a/src/runtime/webcore/blob/AppendBuffer.rs b/src/runtime/webcore/blob/AppendBuffer.rs new file mode 100644 index 000000000000..c58347e0293b --- /dev/null +++ b/src/runtime/webcore/blob/AppendBuffer.rs @@ -0,0 +1,285 @@ +//! Backing storage shared by the Blobs produced by `b = new Blob([b, chunk])`. +//! +//! `new Blob(parts)` copies every part into a fresh store, so that idiom +//! re-copies the whole prefix on every step and costs O(total²). When the +//! first part is a Blob viewing a whole in-memory store, the result is built on +//! an [`AppendBuffer`] instead: one allocation, carrying spare capacity, shared +//! by every store that successive appends produce. Each of those stores is an +//! ordinary immutable `Bytes` viewing the prefix `[0, len)` of the buffer. +//! Appending onto the store that views the longest published prefix claims +//! `[len, len + n)`, fills it and publishes a new store of length `len + n`; +//! nothing an existing store can see is ever written again, so readers of the +//! older stores (including ones on other threads) are unaffected. Once the +//! buffer is full the next append allocates a bigger one with headroom, which +//! keeps the total number of bytes copied linear in the final size. +//! +//! The buffer rides along in `Bytes::allocator` the same way +//! `LinuxMemFdAllocator` does: every store built on it owns one reference, +//! released by the vtable's `free` when the store's `Bytes` is dropped. + +use core::ffi::c_void; +use core::mem::ManuallyDrop; +use core::ptr::NonNull; +use core::sync::atomic::{AtomicUsize, Ordering}; + +use bun_alloc::{Alignment, AllocatorVTable, StdAllocator}; +use bun_core::UnwrapOrOom as _; +use bun_core::string_joiner::StringJoiner; + +use super::store::{Bytes, Data, Store, StoreRef}; +use super::{Blob, SizeType}; + +#[derive(bun_ptr::ThreadSafeRefCounted)] +pub(crate) struct AppendBuffer { + ref_count: bun_ptr::ThreadSafeRefCount, + /// `capacity` bytes from the global allocator, never reallocated: stores + /// point straight into it. + ptr: NonNull, + capacity: usize, + /// Length of the longest prefix published as a store. `[0, committed)` is + /// initialized and immutable; an append moves it forward with a CAS, so two + /// appends onto the same prefix cannot both claim the tail. + committed: AtomicUsize, +} + +impl Drop for AppendBuffer { + fn drop(&mut self) { + // SAFETY: `ptr`/`capacity` came out of the `Vec` in `create` (or were + // reset to an empty Vec's by `take_unique_storage`), and the refcount + // reaching zero means no store points into the allocation. + drop(unsafe { Vec::from_raw_parts(self.ptr.as_ptr(), 0, self.capacity) }); + } +} + +/// Releases the reference the dropped store's `Bytes` held. `buf` is that +/// store's prefix view, not an allocation of its own; the memory goes away +/// with the buffer's last store. +unsafe fn free(buffer: *mut c_void, _buf: &mut [u8], _: Alignment, _: usize) { + // SAFETY: `buffer` is the pointer `allocator()` stored in this `Bytes`, + // and that `Bytes` owned one reference (see `store`). + unsafe { bun_ptr::ThreadSafeRefCount::::deref(buffer.cast::()) }; +} + +/// Its address identifies buffer-backed `Bytes`, like the memfd vtable does. +static VTABLE: &AllocatorVTable = &AllocatorVTable::free_only(free); + +impl AppendBuffer { + /// The store whose bytes open the result when `blob` is the first part of + /// `new Blob(parts)`, if appending onto it is possible: in memory, + /// non-empty, and viewed in full (so the result's first bytes are exactly + /// the store's bytes, and for a buffer-backed store the append can extend + /// it in place). + pub(crate) fn prefix_store(blob: &Blob) -> Option<&StoreRef> { + let store = blob.store()?; + let Data::Bytes(bytes) = &store.data else { + return None; + }; + (blob.offset.get() == 0 && bytes.len() > 0 && blob.size.get() == bytes.len()) + .then_some(store) + } + + /// Whether other stores can view the same memory as this store's bytes, in + /// which case the bytes must not be handed out writable even when the + /// store itself has a single reference. + pub(crate) fn shares_allocation(store: &Store) -> bool { + let Data::Bytes(bytes) = &store.data else { + return false; + }; + let Some(buffer) = Self::from_allocator(bytes.allocator()) else { + return false; + }; + // SAFETY: `bytes` holds a reference on the buffer, so it is live. + !unsafe { &(*buffer).ref_count }.has_one_ref() + } + + /// For `Bytes::to_internal_blob`: when `bytes` is the only store built on + /// its buffer, hands the allocation over as a `Vec` of the bytes the store + /// viewed instead of copying them, and leaves `bytes` empty. `None` when + /// `bytes` is not buffer-backed or the buffer is shared. + pub(crate) fn take_unique_storage(bytes: &mut Bytes) -> Option> { + let buffer = Self::from_allocator(bytes.allocator())?; + // SAFETY: `bytes` holds a reference on the buffer, so it is live. + if !unsafe { &(*buffer).ref_count }.has_one_ref() { + return None; + } + let ptr = bytes.ptr.take()?; + let len = core::mem::take(&mut bytes.len) as usize; + bytes.cap = 0; + bytes.allocator = bun_alloc::basic::C_ALLOCATOR; + // SAFETY: `bytes` owns the only reference, so nothing else can reach the + // buffer or its allocation. `ptr` is the allocation `create` made, + // `capacity` is what the `Vec` there reported, and `[0, len)` is + // initialized. The header is left describing an empty `Vec`, so the + // deref below (the reference `bytes` owned, which its `free` will no + // longer release now that `bytes.ptr` is `None`) frees only the header. + unsafe { + let capacity = core::mem::replace(&mut (*buffer).capacity, 0); + (*buffer).ptr = NonNull::dangling(); + bun_ptr::ThreadSafeRefCount::::deref(buffer); + Some(Vec::from_raw_parts(ptr.as_ptr(), len, capacity)) + } + } + + /// The store holding `prefix`'s bytes followed by the joiner's contents. + /// `prefix` must come from [`Self::prefix_store`]. + pub(crate) fn concat( + prefix: &StoreRef, + suffix: &StringJoiner<'_>, + is_all_ascii: Option, + ) -> StoreRef { + let Data::Bytes(prefix_bytes) = &prefix.data else { + unreachable!("AppendBuffer::concat prefix is not an in-memory store") + }; + let mut grow = false; + if let Some(buffer) = Self::from_allocator(prefix_bytes.allocator()) { + // SAFETY: `prefix_bytes` holds a reference on the buffer. + if let Some(store) = unsafe { Self::append(buffer, prefix_bytes, suffix, is_all_ascii) } + { + return store; + } + // The buffer is full, or another append already claimed its tail. + // This is at least the second append onto this data, so leave room + // for the next one; the first append (onto a plain store) stays an + // exact-size copy so a one-off `new Blob([blob, x])` costs what it + // did before. + grow = true; + } + Self::create(prefix_bytes.slice(), suffix, grow, is_all_ascii) + } + + fn from_allocator(allocator: StdAllocator) -> Option<*mut AppendBuffer> { + core::ptr::eq(allocator.vtable, VTABLE).then(|| allocator.ptr.cast::()) + } + + /// In-place append: succeeds only when `prefix` views exactly the + /// committed bytes and the suffix fits. + /// + /// # Safety + /// `prefix` must be a `Bytes` built on `*this` by [`Self::store`], so the + /// buffer is live for the duration of the call. + unsafe fn append( + this: *mut AppendBuffer, + prefix: &Bytes, + suffix: &StringJoiner<'_>, + is_all_ascii: Option, + ) -> Option { + // SAFETY: caller contract. The header's fields are only ever written + // by `take_unique_storage`, which needs the buffer's single remaining + // store exclusively, while this call holds a store of the buffer + // shared; so nothing writes the header during this shared borrow. + let buffer = unsafe { &*this }; + debug_assert_eq!(prefix.slice().as_ptr(), buffer.ptr.as_ptr()); + let old_len = prefix.len() as usize; + let new_len = old_len + suffix.len; + if new_len > buffer.capacity { + return None; + } + buffer + .committed + .compare_exchange(old_len, new_len, Ordering::AcqRel, Ordering::Relaxed) + .ok()?; + // SAFETY: the CAS made this call the only writer of `[old_len, + // new_len)`, which lies inside the allocation and which no store views + // until the one created below is published. + unsafe { write_suffix(buffer.ptr.as_ptr().add(old_len), suffix) }; + + // SAFETY: `this` is live (caller contract); the new store gets its + // own reference. + unsafe { bun_ptr::ThreadSafeRefCount::::ref_(this) }; + // SAFETY: `[0, new_len)` is initialized and the reference taken above + // belongs to the new store. + Some(unsafe { Self::store(this, new_len, is_all_ascii) }) + } + + /// A fresh buffer holding `prefix` followed by the joiner's contents, with + /// 50% headroom when `grow` is set. + fn create( + prefix: &[u8], + suffix: &StringJoiner<'_>, + grow: bool, + is_all_ascii: Option, + ) -> StoreRef { + let len = prefix.len() + suffix.len; + let wanted = if grow { + len.saturating_add(len / 2) + } else { + len + }; + let mut storage: Vec = Vec::new(); + storage + .try_reserve_exact(wanted) + .or_else(|_| storage.try_reserve_exact(len)) + .unwrap_or_oom(); + let mut storage = ManuallyDrop::new(storage); + let capacity = storage.capacity(); + // `len <= capacity`, and nothing else points into `storage` yet. + let ptr = storage.as_mut_ptr(); + // SAFETY: the ranges `[0, prefix.len())` and `[prefix.len(), len)` are + // inside the reserved capacity, and `prefix` lives in some other + // allocation (a store's bytes), so the copies cannot overlap. + unsafe { + core::ptr::copy_nonoverlapping(prefix.as_ptr(), ptr, prefix.len()); + write_suffix(ptr.add(prefix.len()), suffix); + } + let buffer = bun_core::heap::into_raw(Box::new(AppendBuffer { + ref_count: bun_ptr::ThreadSafeRefCount::init(), + // SAFETY: `Vec::as_mut_ptr` is non-null even for a zero-length + // buffer, and `len > 0` here anyway (`prefix` is non-empty). + ptr: unsafe { NonNull::new_unchecked(ptr) }, + capacity, + committed: AtomicUsize::new(len), + })); + // SAFETY: `[0, len)` was just written, and the reference `init()` + // created belongs to the first store. + unsafe { Self::store(buffer, len, is_all_ascii) } + } + + /// A store viewing `[0, len)` of the buffer. + /// + /// # Safety + /// `this` must be live, `[0, len)` must be initialized, and the caller + /// must hand over one reference on the buffer, which the store's `Bytes` + /// releases through [`free`]. + unsafe fn store(this: *mut AppendBuffer, len: usize, is_all_ascii: Option) -> StoreRef { + // SAFETY: `this` is live (caller contract); this only copies the field. + let ptr = unsafe { (*this).ptr }.as_ptr(); + let len = len as SizeType; + // SAFETY: `ptr[..len]` is initialized (caller contract) and stays valid + // until `free` releases the reference the caller handed over. `cap == + // len` so `allocated_slice()` covers only what this store can read. + let bytes = unsafe { + Bytes::from_raw_parts( + ptr, + len, + len, + StdAllocator { + ptr: this.cast::(), + vtable: VTABLE, + }, + ) + }; + StoreRef::from(Store::new(Store { + data: Data::Bytes(bytes), + mime_type: bun_http_types::MimeType::NONE, + ref_count: bun_ptr::ThreadSafeRefCount::init(), + is_all_ascii, + })) + } +} + +/// Writes the joiner's nodes back to back at `dst`. +/// +/// # Safety +/// `dst` must be valid for writing `suffix.len` bytes that no node of the +/// joiner reads from (the nodes are either joiner-owned or borrowed from +/// already published bytes; the destination is never published yet). +unsafe fn write_suffix(dst: *mut u8, suffix: &StringJoiner<'_>) { + let mut written = 0usize; + for node in suffix.node_slices() { + // SAFETY: `written + node.len() <= suffix.len` because `suffix.len` is + // the sum of the node lengths; non-overlap is the caller's contract. + unsafe { core::ptr::copy_nonoverlapping(node.as_ptr(), dst.add(written), node.len()) }; + written += node.len(); + } + debug_assert_eq!(written, suffix.len); +} diff --git a/src/runtime/webcore/blob/Store.rs b/src/runtime/webcore/blob/Store.rs index f34c88640bcb..2a12fad4d414 100644 --- a/src/runtime/webcore/blob/Store.rs +++ b/src/runtime/webcore/blob/Store.rs @@ -22,6 +22,8 @@ use bun_core::{ZigString, strings}; use bun_http_types::MimeType::MimeType; use bun_url::URL; +use super::append_buffer::AppendBuffer; + #[cfg(unix)] use super::SizeType; @@ -496,10 +498,13 @@ impl BytesExt for Bytes { fn to_internal_blob(&mut self) -> super::Internal { // `Internal.bytes` is `Vec` (global allocator), so - // round-trip only when the storage *is* the global allocator; otherwise - // copy + free through the original allocator (e.g. memfd → munmap). + // round-trip only when the storage *is* the global allocator (directly, + // or through an `AppendBuffer` nothing else shares); otherwise copy + + // free through the original allocator (e.g. memfd → munmap). let bytes = if self.ptr.is_none() { Vec::new() + } else if let Some(storage) = AppendBuffer::take_unique_storage(self) { + storage } else if core::ptr::eq( std::ptr::from_ref(self.allocator.vtable), std::ptr::from_ref(bun_alloc::basic::C_ALLOCATOR.vtable), diff --git a/test/js/web/fetch/blob.test.ts b/test/js/web/fetch/blob.test.ts index fc9b4500bcfc..e747f3db4a28 100644 --- a/test/js/web/fetch/blob.test.ts +++ b/test/js/web/fetch/blob.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from "bun:test"; -import { bunEnv, bunExe, isASAN, tempDir } from "harness"; +import { bunEnv, bunExe, gcTick, isASAN, tempDir } from "harness"; import type { BlobOptions } from "node:buffer"; import type { BinaryLike } from "node:crypto"; import path from "node:path"; @@ -735,3 +735,260 @@ describe("Blob from ArrayBuffer-like values", () => { expect(await blob.text()).toBe("abcdefgh"); }); }); + +describe("new Blob([blob, ...]) appends onto the first part's store", () => { + // When the first part is a Blob viewing a whole in-memory store, the + // constructor does not copy it into a fresh buffer: the result shares the + // prefix's allocation (which keeps spare capacity once it has been appended + // onto more than once) and only the new parts are written, past the bytes + // every existing Blob can see. These tests pin both halves of that: the cost + // of `b = new Blob([b, chunk])` stays linear, and no Blob ever observes bytes + // appended by a later construction. + + const byteChunk = (len: number, fill: number) => new Uint8Array(len).fill(fill); + const concat = (...parts: Uint8Array[]) => new Uint8Array(Buffer.concat(parts)); + + test.concurrent("accumulating chunks costs linear time, not one prefix copy per step", async () => { + // Unfixed, step i copies the i chunks accumulated so far: for these sizes + // that is 16 GiB of memcpy, about 140x the baseline below in a release + // build (4.1 s vs 30 ms) and far worse under ASAN. Fixed, each step copies + // one chunk plus the occasional buffer regrowth, so it costs a small + // multiple (about 4x in a debug build, mostly GC work from the growing + // reported sizes) of building the Blob once from all the chunks plus + // constructing n single-chunk Blobs. That baseline is measured in the same + // process so the bound calibrates itself to the build type and machine. + await using proc = Bun.spawn({ + cmd: [ + bunExe(), + "-e", + ` + const cpuMs = () => { const u = process.cpuUsage(); return (u.user + u.system) / 1000; }; + const n = 1024; + const chunks = Array.from({ length: 8 }, (_, i) => new Uint8Array(32 * 1024).fill(i + 1)); + const parts = Array.from({ length: n }, (_, i) => chunks[i % chunks.length]); + + let t0 = cpuMs(); + const oneShot = new Blob(parts); + for (const part of parts) new Blob([part]); + const baselineMs = cpuMs() - t0; + + t0 = cpuMs(); + let accumulated = new Blob([]); + for (const part of parts) accumulated = new Blob([accumulated, part]); + const accumulateMs = cpuMs() - t0; + + const same = Buffer.compare(await oneShot.bytes(), await accumulated.bytes()) === 0; + console.log(JSON.stringify({ size: accumulated.size, same, baselineMs, accumulateMs })); + `, + ], + env: bunEnv, + stdout: "pipe", + stderr: "pipe", + timeout: 30_000, + killSignal: "SIGKILL", + }); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + expect(stderr).toBe(""); + expect(exitCode).toBe(0); + + const { size, same, baselineMs, accumulateMs } = JSON.parse(stdout); + expect({ size, same }).toEqual({ size: 1024 * 32 * 1024, same: true }); + expect(accumulateMs).toBeLessThan(20 * baselineMs + 100); + }); + + test("every intermediate Blob keeps its own size and bytes as the chain grows", async () => { + // Chunk sizes vary so the chain goes through both in-place appends and + // buffer regrowths many times over. + const chain: Blob[] = []; + const chunks: Uint8Array[] = []; + let blob = new Blob([]); + for (let i = 0; i < 48; i++) { + const chunk = byteChunk(1 + ((i * 37) % 100), i); + chunks.push(chunk); + blob = new Blob([blob, chunk]); + chain.push(blob); + } + + const sizes = await Promise.all(chain.map(async b => [b.size, (await b.bytes()).length])); + let expected = new Uint8Array(0); + for (let i = 0; i < chain.length; i++) { + expected = concat(expected, chunks[i]); + expect(sizes[i]).toEqual([expected.length, expected.length]); + expect(await chain[i].bytes()).toEqual(expected); + } + }); + + test("appending twice onto the same Blob gives independent results", async () => { + // The second append cannot reuse the tail the first one already claimed. + let base = new Blob([byteChunk(64, 1)]); + base = new Blob([base, byteChunk(64, 2)]); + base = new Blob([base, byteChunk(64, 3)]); + const baseBytes = concat(byteChunk(64, 1), byteChunk(64, 2), byteChunk(64, 3)); + + const left = new Blob([base, byteChunk(8, 0x11)]); + const right = new Blob([base, byteChunk(8, 0x22)]); + const left2 = new Blob([left, byteChunk(8, 0x33)]); + const right2 = new Blob([right, byteChunk(8, 0x44)]); + + expect(await base.bytes()).toEqual(baseBytes); + expect(await left.bytes()).toEqual(concat(baseBytes, byteChunk(8, 0x11))); + expect(await right.bytes()).toEqual(concat(baseBytes, byteChunk(8, 0x22))); + expect(await left2.bytes()).toEqual(concat(baseBytes, byteChunk(8, 0x11), byteChunk(8, 0x33))); + expect(await right2.bytes()).toEqual(concat(baseBytes, byteChunk(8, 0x22), byteChunk(8, 0x44))); + }); + + test("a Blob can be appended onto itself", async () => { + let blob = new Blob(["ab"]); + for (let i = 0; i < 6; i++) blob = new Blob([blob, blob]); + expect(blob.size).toBe(2 * 64); + expect(await blob.text()).toBe(Buffer.alloc(2 * 64, "ab").toString()); + }); + + test("mixed parts after the Blob prefix, and empty parts around it", async () => { + const prefix = new Blob(["abc"]); + expect(await new Blob([prefix, "d", new Uint8Array([0x65]), new Blob(["f"]), "", new Blob()]).text()).toBe( + "abcdef", + ); + expect(await new Blob(["", new Blob(), prefix, "d"]).text()).toBe("abcd"); + expect(await new Blob([prefix.slice(1), "d"]).text()).toBe("bcd"); + expect(await new Blob([prefix, { toString: () => "!" }]).text()).toBe("abc!"); + expect(await prefix.text()).toBe("abc"); + + const copy = new Blob([prefix, ""]); + expect(copy).not.toBe(prefix); + expect(await copy.text()).toBe("abc"); + }); + + test("new File([file, ...], name) does not rename the source", async () => { + const source = new File(["abc"], "source.txt"); + const derived = new File([source, ""], "derived.txt"); + const longer = new File([source, "def"], "longer.txt"); + expect([source.name, derived.name, longer.name]).toEqual(["source.txt", "derived.txt", "longer.txt"]); + expect([await source.text(), await derived.text(), await longer.text()]).toEqual(["abc", "abc", "abcdef"]); + }); + + test("text() decodes each Blob in the chain by its own bytes", async () => { + const ascii = new Blob(["ascii"]); + const ascii2 = new Blob([ascii, "-more"]); + expect(await ascii2.text()).toBe("ascii-more"); + const utf8 = new Blob([ascii2, " héllo ✓"]); + const asciiAgain = new Blob([utf8, " end"]); + expect(await ascii.text()).toBe("ascii"); + expect(await utf8.text()).toBe("ascii-more héllo ✓"); + expect(await asciiAgain.text()).toBe("ascii-more héllo ✓ end"); + expect(await ascii2.text()).toBe("ascii-more"); + + const utf8First = new Blob(["ü"]); + const appended = new Blob([utf8First, "x"]); + const appendedAgain = new Blob([appended, "y"]); + expect([await utf8First.text(), await appended.text(), await appendedAgain.text()]).toEqual(["ü", "üx", "üxy"]); + }); + + test("slice(), stream(), structuredClone() and Bun.write() see only their Blob's bytes", async () => { + let blob = new Blob([byteChunk(10, 1)]); + blob = new Blob([blob, byteChunk(10, 2)]); + const shorter = new Blob([blob, byteChunk(10, 3)]); + const longer = new Blob([shorter, byteChunk(10, 4)]); + const shorterBytes = concat(byteChunk(10, 1), byteChunk(10, 2), byteChunk(10, 3)); + const longerBytes = concat(shorterBytes, byteChunk(10, 4)); + + expect(await shorter.slice(25).bytes()).toEqual(shorterBytes.subarray(25)); + expect(await longer.slice(25).bytes()).toEqual(longerBytes.subarray(25)); + expect(await new Response(shorter.stream()).bytes()).toEqual(shorterBytes); + expect(await new Response(longer.stream()).bytes()).toEqual(longerBytes); + expect(await structuredClone(shorter).bytes()).toEqual(shorterBytes); + expect(await structuredClone(longer).bytes()).toEqual(longerBytes); + + using dir = tempDir("blob-append-write", {}); + const shorterPath = path.join(String(dir), "shorter.bin"); + const longerPath = path.join(String(dir), "longer.bin"); + expect(await Promise.all([Bun.write(shorterPath, shorter), Bun.write(longerPath, longer)])).toEqual([30, 40]); + expect(await Bun.file(shorterPath).bytes()).toEqual(shorterBytes); + expect(await Bun.file(longerPath).bytes()).toEqual(longerBytes); + }); + + test("a body's transferred ArrayBuffer is not backed by memory a longer Blob shares", async () => { + // Response.arrayBuffer() hands out the body's bytes without copying when + // the body holds the only reference to the store. Once the `middle` Blob + // object has been collected (gcTick() reliably does that here; should it + // ever not, the copying path is taken and the test still passes) that is + // the case for its store, but `longer` was appended in place onto the same + // allocation, so writing into the returned buffer must not be able to + // change it. Without the sharing check in the transfer path, `longer`'s + // first 192 bytes read back as 0xff. + function build() { + let middle = new Blob([byteChunk(64, 1)]); + middle = new Blob([middle, byteChunk(64, 2)]); + middle = new Blob([middle, byteChunk(64, 3)]); + return { response: new Response(middle), longer: new Blob([middle, byteChunk(16, 4)]) }; + } + const { response, longer } = build(); + await gcTick(); + + new Uint8Array(await response.arrayBuffer()).fill(0xff); + expect(await longer.bytes()).toEqual( + concat(byteChunk(64, 1), byteChunk(64, 2), byteChunk(64, 3), byteChunk(16, 4)), + ); + }); + + test("consuming a stream that holds the last reference to an appended Blob's store", async () => { + // With the Blob objects collected, the stream's source holds the only + // reference to the store, and the native consumers take the store's buffer + // over instead of copying it. For a buffer nothing else shares that hands + // the allocation itself over; `shared` below still has a sibling on its + // buffer, so it is copied. Either way the bytes must come out intact. + function build() { + let unique = new Blob([byteChunk(32, 1)]); + unique = new Blob([unique, byteChunk(32, 2)]); + unique = new Blob([unique, byteChunk(32, 3)]); + let shared = new Blob([byteChunk(32, 4)]); + shared = new Blob([shared, byteChunk(32, 5)]); + shared = new Blob([shared, byteChunk(32, 6)]); + const sibling = new Blob([shared, byteChunk(8, 7)]); + return { unique: unique.stream(), shared: shared.stream(), sibling }; + } + const { unique, shared, sibling } = build(); + await gcTick(); + + expect(await new Response(unique).bytes()).toEqual(concat(byteChunk(32, 1), byteChunk(32, 2), byteChunk(32, 3))); + expect(await Bun.readableStreamToBytes(shared)).toEqual( + concat(byteChunk(32, 4), byteChunk(32, 5), byteChunk(32, 6)), + ); + expect(await sibling.bytes()).toEqual( + concat(byteChunk(32, 4), byteChunk(32, 5), byteChunk(32, 6), byteChunk(8, 7)), + ); + }); + + test("a worker holding the same Blob through a blob: URL can append onto it too", async () => { + let shared = new Blob(["shared"]); + shared = new Blob([shared, "-data"]); + shared = new Blob([shared, "-twice"]); + const url = URL.createObjectURL(shared); + const worker = new Worker( + URL.createObjectURL( + new Blob([ + ` + self.onmessage = async ({ data: url }) => { + const fromMain = await (await fetch(url)).blob(); + const appended = new Blob([fromMain, "+worker"]); + postMessage([await fromMain.text(), await appended.text()]); + }; + `, + ]), + ), + ); + try { + const { promise, resolve, reject } = Promise.withResolvers(); + worker.onmessage = event => resolve(event.data); + worker.onerror = reject; + worker.postMessage(url); + const appendedOnMain = new Blob([shared, "+main"]); + expect(await promise).toEqual(["shared-data-twice", "shared-data-twice+worker"]); + expect(await appendedOnMain.text()).toBe("shared-data-twice+main"); + expect(await shared.text()).toBe("shared-data-twice"); + } finally { + worker.terminate(); + URL.revokeObjectURL(url); + } + }); +});