Skip to content

Blob: read file-backed parts in multi-part new Blob([...]) #33600

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
robobun wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Blob: read file-backed parts in multi-part new Blob([...]) #33600

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
142 changes: 127 additions & 15 deletions src/runtime/webcore/Blob.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3513,19 +3513,12 @@ impl BlobExt for Blob {
if let Some(blob) = item.as_class_ref::<Blob>() {
could_have_non_ascii = could_have_non_ascii
|| blob.charset.get() != strings::AsciiStatus::AllAscii;
// A later part may run user JS that drops the
// last ref to this Blob's Store before `done()`.
if parts_can_run_js {
joiner.push_cloned(blob.shared_view());
} else {
// SAFETY: the prescan above proved no
// remaining part can run user JS, so this
// Blob (rooted via `_keep`/`arg`) keeps its
// Store alive until `joiner.done()` below.
joiner.push(unsafe {
bun_ptr::detach_lifetime(blob.shared_view())
});
}
push_blob_part_bytes(
blob,
&mut joiner,
global,
!parts_can_run_js,
)?;
continue;
} else {
let sliced = item.to_slice_clone(global)?;
Expand All @@ -3552,8 +3545,8 @@ impl BlobExt for Blob {
|| blob.charset.get() != strings::AsciiStatus::AllAscii;
// This arm only handles entries deferred onto the walk
// stack; other pending entries may still run user JS and
// free this Blob's Store before `done()`, so always copy.
joiner.push_cloned(blob.shared_view());
// free this Blob's Store before `done()`, so never borrow.
push_blob_part_bytes(blob, &mut joiner, global, false)?;
} else {
let sliced = current.to_slice_clone(global)?;
could_have_non_ascii = could_have_non_ascii || sliced.is_allocated();
Expand Down Expand Up @@ -3790,6 +3783,125 @@ impl BlobExt for Blob {
}
}

/// Push one Blob part's bytes into the multi-part joiner; reads file-backed stores synchronously.
fn push_blob_part_bytes(
blob: &Blob,
joiner: &mut bun_core::string_joiner::StringJoiner,
global: &JSGlobalObject,
borrow_bytes: bool,
) -> JsResult<()> {
let Some(store) = blob.store.get() else {
return Ok(());
};
match &store.data {
store::Data::Bytes(_) => {
if borrow_bytes {
// SAFETY: caller's prescan proved no remaining part runs user
// JS, so this Blob (rooted via the constructor's `_keep`/`arg`)
// keeps its Store alive until `joiner.done()`.
joiner.push(unsafe { bun_ptr::detach_lifetime(blob.shared_view()) });
} else {
joiner.push_cloned(blob.shared_view());
}
}
store::Data::File(file) => {
let size = blob.size.get();
let offset = blob.offset.get();
match &file.pathlike {
PathOrFileDescriptor::Path(_) => {
let mut node_fs = crate::node::fs::NodeFS::default();
let mut rf_args = crate::node::fs::args::ReadFile::default();
rf_args.encoding = crate::node::types::Encoding::Buffer;
rf_args.path = file.pathlike.clone();
rf_args.offset = offset;
rf_args.max_size = if size == MAX_SIZE { None } else { Some(size) };
match node_fs.read_file(&rf_args, crate::node::fs::Flavor::Sync) {
Err(err) => return Err(global.throw_value(err.to_js(global))),
Ok(mut result) => {
joiner.push_cloned(result.slice());
if let crate::node::types::StringOrBuffer::Buffer(buf) = &mut result {
buf.destroy();
}
}
}
}
PathOrFileDescriptor::Fd(fd) => {
// pread (not read_file): the fd's cursor must not decide what a repeated part reads.
let stat = match bun_sys::fstat(*fd) {
bun_sys::Result::Ok(s) => s,
bun_sys::Result::Err(err) => {
return Err(global.throw_value(err.with_fd(*fd).to_js(global)));
}
};
let file_len = stat.st_size.max(0) as SizeType;
// st_size==0 may be a virtual file (procfs): grow-until-EOF instead of capping at 0.
let avail = if file_len > 0 {
file_len.saturating_sub(offset) as usize
} else {
usize::MAX
};
let cap = if size != MAX_SIZE {
(size as usize).min(avail)
} else {
avail
};
if cap == 0 {
return Ok(());
}
Comment thread
robobun marked this conversation as resolved.
let enomem = || {
global.throw_value(
bun_sys::Error::from_code(bun_sys::E::ENOMEM, bun_sys::Tag::read)
.with_fd(*fd)
.to_js(global),
)
};
let initial = if file_len > 0 { cap } else { 8192.min(cap) };
let mut buf: Vec<u8> = Vec::new();
if buf.try_reserve_exact(initial.min(8 << 30)).is_err() {
return Err(enomem());
}
buf.resize(initial.min(8 << 30), 0);
let mut total = 0usize;
loop {
if total == buf.len() {
if total == cap {
break;
}
let new_len = buf.len().saturating_mul(2).min(cap);
if buf.try_reserve(new_len - buf.len()).is_err() {
return Err(enomem());
}
buf.resize(new_len, 0);
}
let n = match bun_sys::pread(
*fd,
&mut buf[total..],
(offset as i64).saturating_add(total as i64),
) {
bun_sys::Result::Ok(n) => n,
bun_sys::Result::Err(err) => {
return Err(global.throw_value(err.with_fd(*fd).to_js(global)));
}
};
if n == 0 {
break;
}
total += n;
}
buf.truncate(total);
joiner.push_owned(buf.into_boxed_slice());
}
}
}
store::Data::S3(_) => {
return Err(global.throw_invalid_arguments(format_args!(
"Blob parts backed by S3 cannot be read synchronously; await .bytes() or .arrayBuffer() first"
)));
Comment thread
robobun marked this conversation as resolved.
}
}
Ok(())
}

// ──────────────────────────────────────────────────────────────────────────
// Basic accessors
// ──────────────────────────────────────────────────────────────────────────
Expand Down
76 changes: 75 additions & 1 deletion test/js/web/fetch/blob.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
import { describe, expect, test } from "bun:test";
import { bunEnv, bunExe, isASAN, tempDir } from "harness";
import { bunEnv, bunExe, isASAN, isWindows, tempDir } from "harness";
import type { BlobOptions } from "node:buffer";
import type { BinaryLike } from "node:crypto";
import fs from "node:fs";
import path from "node:path";

test("blob: imports have sourcemapped stacktraces", async () => {
Expand Down Expand Up @@ -688,3 +689,76 @@ describe("file-backed slice bounds are respected when streaming and serving", ()
expect(s.size).toBe(5);
});
});

describe("new Blob([...]) with a file-backed Blob part", () => {
async function check(blob: Blob, expected: string) {
const text = await blob.text();
expect({ size: blob.size, text }).toEqual({ size: expected.length, text: expected });
}

test("contributes the file's bytes alongside other parts", async () => {
using dir = tempDir("blob-file-part", { "f.bin": "ABCDEFGHIJ" });
const p = path.join(String(dir), "f.bin");
const file = Bun.file(p);

// single-part fast path (already worked; kept as baseline)
await check(new Blob([file]), "ABCDEFGHIJ");
// file + string (after)
await check(new Blob([file, "-tail"]), "ABCDEFGHIJ-tail");
// string + file (before)
await check(new Blob(["head-", file]), "head-ABCDEFGHIJ");
// in-memory Blob + sliced file
await check(new Blob([new Blob(["M"]), file.slice(2, 6)]), "MCDEF");
// File constructor
await check(new File([file, "!"], "n"), "ABCDEFGHIJ!");
// file + typed array + file (same file twice)
await check(new Blob([file, new Uint8Array([0x2d]), file]), "ABCDEFGHIJ-ABCDEFGHIJ");
// empty string sibling (must not take the single-part clone path)
await check(new Blob(["", file]), "ABCDEFGHIJ");
// structuredClone of a Bun.file part
await check(new Blob(["<", structuredClone(file), ">"]), "<ABCDEFGHIJ>");
// Response.blob() over a Bun.file still has a file store
await check(new Blob(["<", await new Response(file).blob(), ">"]), "<ABCDEFGHIJ>");
});

test("fd-backed BunFile part", async () => {
using dir = tempDir("blob-fd-part", { "f.bin": "0123456789", "empty.bin": "" });
const fd = fs.openSync(path.join(String(dir), "f.bin"), "r");
const emptyFd = fs.openSync(path.join(String(dir), "empty.bin"), "r");
try {
const f = Bun.file(fd);
await check(new Blob(["<", f, ">"]), "<0123456789>");
// same fd used twice: each read must start at the Blob's offset, not
// wherever the previous part left the cursor
await check(new Blob([f, "-", f]), "0123456789-0123456789");
await check(new Blob([f.slice(2, 6), "|", f.slice(7, 9)]), "2345|78");
// slice end past EOF must clamp to the fd's actual size, not allocate to the declared end
await check(new Blob(["<", f.slice(0, 1e12), ">"]), "<0123456789>");
// an empty fd with a huge slice (st_size==0 so avail is unbounded) must not over-allocate
await check(new Blob(["<", Bun.file(emptyFd).slice(0, 1e12), ">"]), "<>");
if (!isWindows) {
// On POSIX pread(2) never touches the fd cursor, so the constructor
// must not mutate the caller's position. Windows ReadFile with an
// OVERLAPPED offset on a synchronous handle advances the pointer (a
// libuv/Node quirk), so this guarantee does not hold there.
const buf = Buffer.alloc(20);
const n = fs.readSync(fd, buf, 0, 20, null);
expect(buf.subarray(0, n).toString()).toBe("0123456789");
}
} finally {
fs.closeSync(fd);
fs.closeSync(emptyFd);
}
});

test("throws when the file cannot be read", () => {
using dir = tempDir("blob-file-part-missing", {});
const p = path.join(String(dir), "does-not-exist");
expect(() => new Blob(["x", Bun.file(p)])).toThrow(expect.objectContaining({ code: "ENOENT" }));
// still throws after `.size` was accessed (which resolves to 0 for a
// nonexistent path)
const observed = Bun.file(p);
void observed.size;
expect(() => new Blob(["x", observed])).toThrow(expect.objectContaining({ code: "ENOENT" }));
});
});
Loading