From 12551fef43cf890ec157213c9669621acdf890e8 Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Sun, 9 Aug 2026 11:28:28 +0000 Subject: [PATCH 1/6] Widen UTF-16/Latin-1 encode-into byte counts from u32 to usize A UTF-16 source can encode to exactly 2^32 UTF-8 bytes, and JSC allows 2^32-byte ArrayBuffers, so EncodeIntoResult's u32 counts wrapped to 0 even though every byte was written: Buffer.write returned 0 and TextEncoder.encodeInto returned {written: 0} for a full 4 GiB destination. EncodeIntoResult.read/written are now usize end to end, the TextEncoder__encodeInto8/16 FFI returns the struct by value instead of packing two u32 halves into a u64, and V8String's WriteUtf8V2 drops the chunking workaround it carried for those packed 32-bit counts. --- src/bun_core/fmt.rs | 10 ++-- src/bun_core/lib.rs | 30 ++++++------ src/bun_core/string/immutable/unicode.rs | 13 +++-- src/http_jsc/websocket_client.rs | 18 +++---- src/install/PackageInstall.rs | 2 +- src/jsc/bindings/headers-handwritten.h | 10 ++++ src/jsc/bindings/v8/V8String.cpp | 47 +++++-------------- src/jsc/bindings/webcore/JSTextEncoder.cpp | 9 ++-- src/paths/string_paths.rs | 2 +- src/runtime/node/node_process.rs | 4 +- src/runtime/webcore/TextDecoder.rs | 2 +- src/runtime/webcore/TextEncoder.rs | 34 +++++--------- .../webcore/TextEncoderStreamEncoder.rs | 4 +- src/runtime/webcore/encoding.rs | 12 ++--- src/watcher/WindowsWatcher.rs | 2 +- test/js/node/buffer.test.js | 46 ++++++++++++++++++ 16 files changed, 134 insertions(+), 111 deletions(-) diff --git a/src/bun_core/fmt.rs b/src/bun_core/fmt.rs index 6a0a57b27e95..c3bf503f592f 100644 --- a/src/bun_core/fmt.rs +++ b/src/bun_core/fmt.rs @@ -540,8 +540,8 @@ pub fn format_utf16_type(slice_: &[u16], writer: &mut impl fmt::Write) -> fmt::R if result.read == 0 || result.written == 0 { break; } - write_bytes(writer, &chunk[..result.written as usize])?; - slice = &slice[result.read as usize..]; + write_bytes(writer, &chunk[..result.written])?; + slice = &slice[result.read..]; } Ok(()) } @@ -562,7 +562,7 @@ pub(crate) fn format_utf16_type_with_path_options( break; } - let to_write = &chunk[..result.written as usize]; + let to_write = &chunk[..result.written]; if !opts.escape_backslashes && opts.path_sep == PathSep::Any { write_bytes(writer, to_write)?; } else { @@ -1095,8 +1095,8 @@ pub fn format_latin1(slice_: &[u8], writer: &mut impl fmt::Write) -> fmt::Result if result.read == 0 || result.written == 0 { break; } - write_bytes(writer, &chunk[..result.written as usize])?; - slice = &slice[result.read as usize..]; + write_bytes(writer, &chunk[..result.written])?; + slice = &slice[result.read..]; } if !slice.is_empty() { diff --git a/src/bun_core/lib.rs b/src/bun_core/lib.rs index 7a6664d9e134..bc3705cc7933 100644 --- a/src/bun_core/lib.rs +++ b/src/bun_core/lib.rs @@ -1682,10 +1682,18 @@ pub(crate) mod strings_impl { } /// Result of an encode-into-fixed-buffer operation. Port of `EncodeIntoResult`. + /// + /// `read`/`written` are `usize`: a UTF-16 source can encode to exactly + /// 2^32 UTF-8 bytes (JSC allows 2^32-byte ArrayBuffers), which a `u32` + /// count would silently wrap to 0. + /// + /// `repr(C)`: returned by value from `TextEncoder__encodeInto8/16` + /// (mirrored as `TextEncoderEncodeIntoResult` in `headers-handwritten.h`). + #[repr(C)] #[derive(Clone, Copy, Default, Debug)] pub struct EncodeIntoResult { - pub read: u32, - pub written: u32, + pub read: usize, + pub written: usize, } /// Port of `elementLengthUTF16IntoUTF8`: the exact UTF-8 byte length of a @@ -1741,8 +1749,8 @@ pub(crate) mod strings_impl { }; if r.status == simdutf::Status::SUCCESS { return EncodeIntoResult { - read: utf16.len() as u32, - written: r.count as u32, + read: utf16.len(), + written: r.count, }; } } @@ -1760,10 +1768,7 @@ pub(crate) mod strings_impl { written += n; read += adv as usize; } - EncodeIntoResult { - read: read as u32, - written: written as u32, - } + EncodeIntoResult { read, written } } /// Port of `copyLatin1IntoUTF8` — encode Latin-1 into a fixed-size UTF-8 buffer. @@ -1827,8 +1832,8 @@ pub(crate) mod strings_impl { debug_assert!(latin1_[read] >= 0x80); if STOP { return EncodeIntoResult { - written: u32::MAX, - read: u32::MAX, + written: usize::MAX, + read: usize::MAX, }; } if buf_.len() - written < 2 { @@ -1840,10 +1845,7 @@ pub(crate) mod strings_impl { read += 1; } - EncodeIntoResult { - written: written as u32, - read: read as u32, - } + EncodeIntoResult { written, read } } /// Null-terminated variant of `to_utf8_from_latin1`. Returns `ZBox` so diff --git a/src/bun_core/string/immutable/unicode.rs b/src/bun_core/string/immutable/unicode.rs index 6901ec9fceb8..8c86daa4982d 100644 --- a/src/bun_core/string/immutable/unicode.rs +++ b/src/bun_core/string/immutable/unicode.rs @@ -448,8 +448,8 @@ pub fn copy_cp1252_into_utf16(buf_: &mut [u16], latin1_: &[u8]) -> EncodeIntoRes } EncodeIntoResult { - read: (buf_total - buf.len()) as u32, - written: (latin1_total - latin1.len()) as u32, + read: buf_total - buf.len(), + written: latin1_total - latin1.len(), } } @@ -459,7 +459,6 @@ pub fn copy_latin1_into_utf16(buf_: &mut [u16], latin1_: &[u8]) -> EncodeIntoRes for (out, &inp) in buf_[..len].iter_mut().zip(latin1_[..len].iter()) { *out = u16::from(inp); } - let len = len as u32; EncodeIntoResult { read: len, written: len, @@ -1039,8 +1038,8 @@ fn copy_utf16_into_utf8_with_buffer_impl WebSocket { let content_byte_len: usize = strings::element_length_utf16_into_utf8(utf16); let mut buf = vec![0u8; content_byte_len]; let encode_result = strings::copy_utf16_into_utf8(&mut buf, utf16); - buf.truncate(encode_result.written as usize); + buf.truncate(encode_result.written); utf8_storage = buf; &utf8_storage } @@ -976,7 +976,7 @@ impl WebSocket { } else { let mut buf = vec![0u8; content_byte_len]; let encode_result = strings::copy_latin1_into_utf8(&mut buf, latin1); - buf.truncate(encode_result.written as usize); + buf.truncate(encode_result.written); utf8_storage = buf; &utf8_storage } @@ -1829,7 +1829,7 @@ fn encode_close_reason(reason: &ZigString, buf: &mut [u8; MAX_CONTROL_PAYLOAD]) } else { // Latin-1 → UTF-8: raw Latin-1 bytes would fail `send_close_with_body`'s UTF-8 check. let result = strings::copy_latin1_into_utf8(cursor.get_mut(), reason.slice()); - if (result.read as usize) < reason.slice().len() { + if result.read < reason.slice().len() { return None; } cursor.set_position(result.written as u64); @@ -2341,20 +2341,20 @@ impl Copy<'_> { match self { Copy::Utf16(utf16) => { let encoded = strings::copy_utf16_into_utf8_impl::(parts.payload, utf16); - debug_assert_eq!(encoded.written as usize, content_byte_len); - debug_assert_eq!(encoded.read as usize, utf16.len()); + debug_assert_eq!(encoded.written, content_byte_len); + debug_assert_eq!(encoded.read, utf16.len()); header - .write_header(&mut parts.header, encoded.written as usize) + .write_header(&mut parts.header, encoded.written) .expect("unreachable"); Mask::fill_in_place(global_this, parts.mask, parts.payload); } Copy::Latin1(latin1) => { let encoded = strings::copy_latin1_into_utf8(parts.payload, latin1); - debug_assert_eq!(encoded.written as usize, content_byte_len); + debug_assert_eq!(encoded.written, content_byte_len); // latin1 can contain non-ascii - debug_assert_eq!(encoded.read as usize, latin1.len()); + debug_assert_eq!(encoded.read, latin1.len()); header - .write_header(&mut parts.header, encoded.written as usize) + .write_header(&mut parts.header, encoded.written) .expect("unreachable"); Mask::fill_in_place(global_this, parts.mask, parts.payload); } diff --git a/src/install/PackageInstall.rs b/src/install/PackageInstall.rs index 00ab60a992c3..4f0eea7ef778 100644 --- a/src/install/PackageInstall.rs +++ b/src/install/PackageInstall.rs @@ -2200,7 +2200,7 @@ impl<'a> PackageInstall<'a> { } let res = strings::copy_utf16_into_utf8(&mut dest_buf[..], &wbuf[..i]); - let mut offset: usize = res.written as usize; + let mut offset: usize = res.written; if dest_buf[offset - 1] != bun_paths::SEP_WINDOWS { dest_buf[offset] = bun_paths::SEP_WINDOWS; offset += 1; diff --git a/src/jsc/bindings/headers-handwritten.h b/src/jsc/bindings/headers-handwritten.h index 67e94e8c3deb..6b38b6089734 100644 --- a/src/jsc/bindings/headers-handwritten.h +++ b/src/jsc/bindings/headers-handwritten.h @@ -396,6 +396,16 @@ extern "C" void ZigString__freeGlobal(const unsigned char* ptr, size_t len); extern "C" size_t Bun__encoding__writeLatin1(const unsigned char* ptr, size_t len, unsigned char* to, size_t other_len, Encoding encoding); extern "C" size_t Bun__encoding__writeUTF16(const char16_t* ptr, size_t len, unsigned char* to, size_t other_len, Encoding encoding); +// Mirrors `EncodeIntoResult` in bun_core (repr(C)). The counts are size_t +// because `written` can be exactly 2^32 (a full 2^32-byte Uint8Array +// destination), which would wrap a 32-bit count to 0. +typedef struct TextEncoderEncodeIntoResult { + size_t read; + size_t written; +} TextEncoderEncodeIntoResult; +extern "C" TextEncoderEncodeIntoResult TextEncoder__encodeInto8(const unsigned char* stringPtr, size_t stringLen, void* ptr, size_t len); +extern "C" TextEncoderEncodeIntoResult TextEncoder__encodeInto16(const char16_t* stringPtr, size_t stringLen, void* ptr, size_t len); + extern "C" size_t Bun__encoding__byteLengthLatin1AsUTF8(const unsigned char* ptr, size_t len); extern "C" size_t Bun__encoding__byteLengthUTF16AsUTF8(const char16_t* ptr, size_t len); diff --git a/src/jsc/bindings/v8/V8String.cpp b/src/jsc/bindings/v8/V8String.cpp index 7a455559f7b4..c0284491f206 100644 --- a/src/jsc/bindings/v8/V8String.cpp +++ b/src/jsc/bindings/v8/V8String.cpp @@ -2,6 +2,7 @@ #include "V8HandleScope.h" #include "wtf/SIMDUTF.h" #include "v8_compatibility_assertions.h" +#include "headers-handwritten.h" ASSERT_V8_TYPE_LAYOUT_MATCHES(v8::String) @@ -149,9 +150,6 @@ bool String::IsExternalOneByte() const return !impl->isNull() && impl->impl()->isExternal() && impl->is8Bit(); } -extern "C" size_t TextEncoder__encodeInto8(const Latin1Character* stringPtr, size_t stringLen, void* ptr, size_t len); -extern "C" size_t TextEncoder__encodeInto16(const char16_t* stringPtr, size_t stringLen, void* ptr, size_t len); - int String::WriteUtf8(Isolate* isolate, char* buffer, int length, int* nchars_ref, int options) const { RELEASE_ASSERT(options == 0); @@ -160,10 +158,11 @@ int String::WriteUtf8(Isolate* isolate, char* buffer, int length, int* nchars_re size_t unsigned_length = length < 0 ? static_cast(std::numeric_limits::max()) : static_cast(length); - uint64_t result = string.is8Bit() ? TextEncoder__encodeInto8(string.span8().data(), string.span8().size(), buffer, unsigned_length) - : TextEncoder__encodeInto16(string.span16().data(), string.span16().size(), buffer, unsigned_length); - uint32_t read = static_cast(result); - uint32_t written = static_cast(result >> 32); + TextEncoderEncodeIntoResult result = string.is8Bit() ? TextEncoder__encodeInto8(string.span8().data(), string.span8().size(), buffer, unsigned_length) + : TextEncoder__encodeInto16(string.span16().data(), string.span16().size(), buffer, unsigned_length); + // unsigned_length <= INT_MAX, so both counts fit in 32 bits here. + uint32_t read = static_cast(result.read); + uint32_t written = static_cast(result.written); if (written < length && read == string.length()) { buffer[written] = 0; @@ -239,37 +238,15 @@ size_t String::WriteUtf8V2(Isolate* isolate, char* buffer, size_t capacity, int // uses when kReplaceInvalidUtf8 is not set, so the result size matches either // way). if (str->is8Bit()) { - // Latin-1 expands at most 2x: 2 * (2^31 - 1) < 2^32, so the packed - // 32-bit counts cannot wrap. const auto span = str->span8(); - uint64_t result = TextEncoder__encodeInto8(span.data(), span.size(), buffer, writableCapacity); - read = static_cast(result); - written = static_cast(result >> 32); + TextEncoderEncodeIntoResult result = TextEncoder__encodeInto8(span.data(), span.size(), buffer, writableCapacity); + read = result.read; + written = result.written; } else { - // UTF-16 expands up to 3x, which can exceed the 32-bit counts - // TextEncoder__encodeInto packs its result into (3 * (2^31 - 1) > - // 2^32). Encode in chunks small enough that each chunk's counts - // fit, accumulating in size_t. const auto span = str->span16(); - const size_t total = span.size(); - constexpr size_t maxChunk = static_cast(1) << 30; // <= 3 GiB UTF-8 per chunk - while (read < total) { - size_t chunkLength = std::min(maxChunk, total - read); - // Never split a surrogate pair across chunks: the encoder - // would see two unpaired halves and write U+FFFD twice. - if (read + chunkLength < total && U16_IS_LEAD(span[read + chunkLength - 1])) { - chunkLength--; - } - uint64_t result = TextEncoder__encodeInto16(span.data() + read, chunkLength, buffer + written, writableCapacity - written); - const uint32_t chunkRead = static_cast(result); - const uint32_t chunkWritten = static_cast(result >> 32); - read += chunkRead; - written += chunkWritten; - if (chunkRead < chunkLength) { - // Ran out of output capacity. - break; - } - } + TextEncoderEncodeIntoResult result = TextEncoder__encodeInto16(span.data(), span.size(), buffer, writableCapacity); + read = result.read; + written = result.written; } } diff --git a/src/jsc/bindings/webcore/JSTextEncoder.cpp b/src/jsc/bindings/webcore/JSTextEncoder.cpp index a82a257a86c9..ce7fb5f8b57d 100644 --- a/src/jsc/bindings/webcore/JSTextEncoder.cpp +++ b/src/jsc/bindings/webcore/JSTextEncoder.cpp @@ -53,14 +53,13 @@ #include "JSDOMOperation.h" #include "JSDOMWrapperCache.h" #include "BunClientData.h" +#include "headers-handwritten.h" namespace WebCore { using namespace JSC; extern "C" JSC::EncodedJSValue TextEncoder__encode8(JSC::JSGlobalObject* global, const Latin1Character* stringPtr, size_t stringLen); extern "C" JSC::EncodedJSValue TextEncoder__encode16(JSC::JSGlobalObject* global, const char16_t* stringPtr, size_t stringLen); -extern "C" size_t TextEncoder__encodeInto8(const Latin1Character* stringPtr, size_t stringLen, void* ptr, size_t len); -extern "C" size_t TextEncoder__encodeInto16(const char16_t* stringPtr, size_t stringLen, void* ptr, size_t len); extern "C" JSC::EncodedJSValue TextEncoder__encodeRopeString(JSC::JSGlobalObject* lexicalGlobalObject, JSC::JSString* str); template<> TextEncoder::EncodeIntoResult convertDictionary(JSGlobalObject& lexicalGlobalObject, JSValue value) @@ -345,7 +344,7 @@ static inline JSC::EncodedJSValue jsTextEncoderPrototypeFunction_encodeIntoBody( return {}; } - size_t res = 0; + TextEncoderEncodeIntoResult res = {}; if (!source->is8Bit()) { const auto span = source->span16(); res = TextEncoder__encodeInto16(span.data(), span.size(), destination->vector(), destination->byteLength()); @@ -356,8 +355,8 @@ static inline JSC::EncodedJSValue jsTextEncoderPrototypeFunction_encodeIntoBody( Bun::GlobalScope* globalScope = reinterpret_cast(lexicalGlobalObject); auto* result = JSC::constructEmptyObject(vm, globalScope->encodeIntoObjectStructure()); - result->putDirectOffset(vm, 0, JSC::jsNumber(static_cast(res))); - result->putDirectOffset(vm, 1, JSC::jsNumber(static_cast(res >> 32))); + result->putDirectOffset(vm, 0, JSC::jsNumber(res.read)); + result->putDirectOffset(vm, 1, JSC::jsNumber(res.written)); return JSValue::encode(result); } diff --git a/src/paths/string_paths.rs b/src/paths/string_paths.rs index 7197850ae526..93dafc44f408 100644 --- a/src/paths/string_paths.rs +++ b/src/paths/string_paths.rs @@ -69,7 +69,7 @@ pub fn from_w_path<'a>(buf: &'a mut [u8], utf16: &[u16]) -> &'a ZStr { let to_copy = strings::trim_prefix_comptime::(utf16, &windows::LONG_PATH_PREFIX); let last = buf.len() - 1; let encode_into_result = strings::copy_utf16_into_utf8(&mut buf[..last], to_copy); - let written = encode_into_result.written as usize; + let written = encode_into_result.written; debug_assert!(written < buf.len()); buf[written] = 0; ZStr::from_buf(buf, written) diff --git a/src/runtime/node/node_process.rs b/src/runtime/node/node_process.rs index d021b241ea51..525f2b1404ea 100644 --- a/src/runtime/node/node_process.rs +++ b/src/runtime/node/node_process.rs @@ -586,7 +586,7 @@ mod _impl { let mut buf1: Vec = vec![0u16; k.utf16_byte_length() + 1]; let mut buf2: Vec = vec![0u16; v.utf16_byte_length() + 1]; let len1: usize = if k.is_8bit() { - strings::copy_latin1_into_utf16(&mut buf1, k.latin1()).written as usize + strings::copy_latin1_into_utf16(&mut buf1, k.latin1()).written } else { buf1[0..k.length()].copy_from_slice(k.utf16()); k.length() @@ -600,7 +600,7 @@ mod _impl { break 'str_ EMPTY_W.as_ptr(); } let len2: usize = if v.is_8bit() { - strings::copy_latin1_into_utf16(&mut buf2, v.latin1()).written as usize + strings::copy_latin1_into_utf16(&mut buf2, v.latin1()).written } else { buf2[0..v.length()].copy_from_slice(v.utf16()); v.length() diff --git a/src/runtime/webcore/TextDecoder.rs b/src/runtime/webcore/TextDecoder.rs index 869c63d5fa4f..de80aebe1913 100644 --- a/src/runtime/webcore/TextDecoder.rs +++ b/src/runtime/webcore/TextDecoder.rs @@ -309,7 +309,7 @@ impl TextDecoder { Ok(unsafe { jsc::zig_string::to_external_u16( bun_core::heap::into_raw(bytes).cast::(), - out.written as usize, + out.written, global_this, ) }) diff --git a/src/runtime/webcore/TextEncoder.rs b/src/runtime/webcore/TextEncoder.rs index a8c14102ab30..e09d982e4c57 100644 --- a/src/runtime/webcore/TextEncoder.rs +++ b/src/runtime/webcore/TextEncoder.rs @@ -44,8 +44,8 @@ unsafe extern "C" fn TextEncoder__encode8( }; debug_assert!(array_buffer.len == utf8_len); let result = strings::copy_latin1_into_utf8(array_buffer.byte_slice_mut(), slice); - debug_assert!(result.written as usize == utf8_len); - debug_assert!(result.read as usize == slice.len()); + debug_assert!(result.written == utf8_len); + debug_assert!(result.read == slice.len()); uint8array } @@ -69,8 +69,8 @@ fn encode16_impl(global_this: &JSGlobalObject, slice: &[u16]) -> JSValue { if result.read == 0 || result.written == 0 { return replacement_char_uint8_array(global_this); } - let written = result.written as usize; - debug_assert!(result.read as usize == slice.len()); + let written = result.written; + debug_assert!(result.read == slice.len()); let Ok(uint8array) = create_uninitialized_uint8_array(global_this, written) else { return JSValue::ZERO; }; @@ -99,7 +99,7 @@ fn encode16_impl(global_this: &JSGlobalObject, slice: &[u16]) -> JSValue { debug_assert!(array_buffer.len == need); let result = strings::copy_utf16_into_utf8_with_utf8_len(array_buffer.byte_slice_mut(), slice, need); - if result.written as usize == need && result.read as usize == slice.len() { + if result.written == need && result.read == slice.len() { return uint8array; } @@ -177,11 +177,11 @@ impl<'a> RopeStringEncoder<'a> { &mut this.buf[this.tail..], src, ); - if result.read == u32::MAX && result.written == u32::MAX { + if result.read == usize::MAX && result.written == usize::MAX { it.stop = 1; this.any_non_ascii = true; } else { - this.tail += result.written as usize; + this.tail += result.written; } } @@ -199,7 +199,7 @@ impl<'a> RopeStringEncoder<'a> { &mut this.buf[offset as usize..], src, ); - if result.read == u32::MAX && result.written == u32::MAX { + if result.read == usize::MAX && result.written == usize::MAX { it.stop = 1; this.any_non_ascii = true; } @@ -267,17 +267,12 @@ unsafe extern "C" fn TextEncoder__encodeInto16( input_len: usize, buf_ptr: *mut u8, buf_len: usize, -) -> u64 { +) -> strings::EncodeIntoResult { // SAFETY: caller guarantees buf_ptr[0..buf_len] is a valid mutable buffer let output = unsafe { core::slice::from_raw_parts_mut(buf_ptr, buf_len) }; // SAFETY: caller guarantees input_ptr[0..input_len] is valid UTF-16 data let input = unsafe { core::slice::from_raw_parts(input_ptr, input_len) }; - let result: strings::EncodeIntoResult = strings::copy_utf16_into_utf8(output, input); - // Pack `read` at byte offset 0 and `written` at offset 4 via native-endian bytes — no `unsafe`. - let mut b = [0u8; 8]; - b[..4].copy_from_slice(&result.read.to_ne_bytes()); - b[4..].copy_from_slice(&result.written.to_ne_bytes()); - u64::from_ne_bytes(b) + strings::copy_utf16_into_utf8(output, input) } /// # Safety @@ -289,15 +284,10 @@ unsafe extern "C" fn TextEncoder__encodeInto8( input_len: usize, buf_ptr: *mut u8, buf_len: usize, -) -> u64 { +) -> strings::EncodeIntoResult { // SAFETY: caller guarantees buf_ptr[0..buf_len] is a valid mutable buffer let output = unsafe { core::slice::from_raw_parts_mut(buf_ptr, buf_len) }; // SAFETY: caller guarantees input_ptr[0..input_len] is valid Latin-1 data let input = unsafe { core::slice::from_raw_parts(input_ptr, input_len) }; - let result: strings::EncodeIntoResult = strings::copy_latin1_into_utf8(output, input); - // Pack `read` at byte offset 0 and `written` at offset 4 via native-endian bytes — no `unsafe`. - let mut b = [0u8; 8]; - b[..4].copy_from_slice(&result.read.to_ne_bytes()); - b[4..].copy_from_slice(&result.written.to_ne_bytes()); - u64::from_ne_bytes(b) + strings::copy_latin1_into_utf8(output, input) } diff --git a/src/runtime/webcore/TextEncoderStreamEncoder.rs b/src/runtime/webcore/TextEncoderStreamEncoder.rs index b19cc103dc23..de1588fd24c3 100644 --- a/src/runtime/webcore/TextEncoderStreamEncoder.rs +++ b/src/runtime/webcore/TextEncoderStreamEncoder.rs @@ -69,10 +69,10 @@ impl TextEncoderStreamEncoder { let result = unsafe { bun_core::vec::fill_spare(buffer, 0, |spare| { let r = strings::copy_latin1_into_utf8(spare, remain); - (r.written as usize, r) + (r.written, r) }) }; - remain = &remain[result.read as usize..]; + remain = &remain[result.read..]; if result.written == 0 && result.read == 0 { buffer.reserve(2); diff --git a/src/runtime/webcore/encoding.rs b/src/runtime/webcore/encoding.rs index d7c2fb127210..b0832bebc702 100644 --- a/src/runtime/webcore/encoding.rs +++ b/src/runtime/webcore/encoding.rs @@ -528,13 +528,13 @@ pub(crate) unsafe fn write_u8 { let r = strings::copy_latin1_into_utf8(to_slice, input_slice); - let mut written = r.written as usize; + let mut written = r.written; // `copy_latin1_into_utf8` stops at whole code points. Under // byte-level truncation, a Latin-1 char >= 0x80 whose 2-byte // sequence straddles the end still gets its lead byte. - if ALLOW_PARTIAL_WRITE && written < to_len && (r.read as usize) < len { - debug_assert!(input_slice[r.read as usize] >= 0x80); - to_slice[written] = 0xC0 | (input_slice[r.read as usize] >> 6); + if ALLOW_PARTIAL_WRITE && written < to_len && r.read < len { + debug_assert!(input_slice[r.read] >= 0x80); + to_slice[written] = 0xC0 | (input_slice[r.read] >> 6); written += 1; } Ok(written) @@ -550,7 +550,7 @@ pub(crate) unsafe fn write_u8()) { let output: &mut [u16] = bytemuck::cast_slice_mut(&mut to_slice[..out_units * 2]); - strings::copy_latin1_into_utf16(output, buf).written as usize * 2 + strings::copy_latin1_into_utf16(output, buf).written * 2 } else { // Rust `&mut [u16]` requires natural alignment, so inline the // (trivial) widen loop for the misaligned-dest case @@ -643,7 +643,7 @@ pub(crate) unsafe fn write_u16(to_slice, input_slice) - .written as usize, + .written, ) } Encoding::Latin1 | Encoding::Ascii | Encoding::Buffer => { diff --git a/src/watcher/WindowsWatcher.rs b/src/watcher/WindowsWatcher.rs index d78a82c7e9dd..5fe286b7f045 100644 --- a/src/watcher/WindowsWatcher.rs +++ b/src/watcher/WindowsWatcher.rs @@ -425,7 +425,7 @@ pub(crate) fn watch_loop_cycle(this: &mut Watcher) -> bun_sys::Result<()> { let filename: &[u16] = event.filename.slice(); let convert_res = strings::copy_utf16_into_utf8(&mut this.platform.buf[base_idx..], filename); - let eventpath_len = base_idx + convert_res.written as usize; + let eventpath_len = base_idx + convert_res.written; bun_core::scoped_log!( watcher, diff --git a/test/js/node/buffer.test.js b/test/js/node/buffer.test.js index c33e319a2c2c..5bb57a2c9f54 100644 --- a/test/js/node/buffer.test.js +++ b/test/js/node/buffer.test.js @@ -4695,3 +4695,49 @@ it.skipIf(os.totalmem() < 10 * 1024 ** 3)( expect(exitCode).toBe(0); }, ); + +// The reported byte count itself can be exactly 2**32: a UTF-16 source whose +// UTF-8 encoding fills a MAX_LENGTH buffer. Buffer.write/TextEncoder.encodeInto +// used to round-trip that count through uint32, reporting 0 even though every +// byte was written. Needs ~10 GiB RSS (4 GiB destination + UTF-16 source). +it.skipIf(os.totalmem() < 16 * 1024 ** 3)( + "Buffer.write/TextEncoder.encodeInto report a byte count of exactly 2**32 without uint32 wrap", + async () => { + const script = ` + const N = 2 ** 32; + // 1431655765 three-byte chars (U+0800 -> E0 A0 80) + one ASCII char + // encode to exactly N UTF-8 bytes. + const str = "\\u0800".repeat((N - 1) / 3) + "a"; + const buf = Buffer.alloc(N); + const out = {}; + out.byteLength = Buffer.byteLength(str, "utf8"); + out.write_ret = buf.write(str, 0, N, "utf8"); + // subarray instead of indexing: 2**32 - 1 is not a valid array index. + out.tail = Array.from(buf.subarray(N - 3)); + const res = new TextEncoder().encodeInto(str, buf); + out.encode_read = res.read; + out.encode_written = res.written; + console.log(JSON.stringify(out)); + `; + await using proc = Bun.spawn({ + cmd: [bunExe(), "-e", script], + env: { ...bunEnv, BUN_GARBAGE_COLLECTOR_LEVEL: "0" }, + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + expect({ stdout: stdout.trim(), stderr }).toEqual({ + stdout: JSON.stringify({ + byteLength: 4294967296, + write_ret: 4294967296, + tail: [0xa0, 0x80, 0x61], + encode_read: 1431655766, + encode_written: 4294967296, + }), + stderr: "", + }); + expect(exitCode).toBe(0); + }, + // Two full 4.3 GB encode passes take ~1 min under a debug+ASAN build. + 5 * 60 * 1000, +); From 9255facf8697277ae7463a0dafdeb3ed96b76a15 Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Sun, 9 Aug 2026 11:42:21 +0000 Subject: [PATCH 2/6] Drop the last redundant as-usize cast on EncodeIntoResult.read --- src/bun_core/fmt.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bun_core/fmt.rs b/src/bun_core/fmt.rs index c3bf503f592f..3672dc13e420 100644 --- a/src/bun_core/fmt.rs +++ b/src/bun_core/fmt.rs @@ -584,7 +584,7 @@ pub(crate) fn format_utf16_type_with_path_options( } write_bytes(writer, ptr)?; } - slice = &slice[result.read as usize..]; + slice = &slice[result.read..]; } Ok(()) } From 4ed8c69b3cfddcf69313009ebb71d598d72f6d62 Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Sun, 9 Aug 2026 11:42:57 +0000 Subject: [PATCH 3/6] Tighten EncodeIntoResult doc comments --- src/bun_core/lib.rs | 11 ++++------- src/jsc/bindings/headers-handwritten.h | 5 ++--- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/src/bun_core/lib.rs b/src/bun_core/lib.rs index bc3705cc7933..283adc1090e0 100644 --- a/src/bun_core/lib.rs +++ b/src/bun_core/lib.rs @@ -1682,13 +1682,10 @@ pub(crate) mod strings_impl { } /// Result of an encode-into-fixed-buffer operation. Port of `EncodeIntoResult`. - /// - /// `read`/`written` are `usize`: a UTF-16 source can encode to exactly - /// 2^32 UTF-8 bytes (JSC allows 2^32-byte ArrayBuffers), which a `u32` - /// count would silently wrap to 0. - /// - /// `repr(C)`: returned by value from `TextEncoder__encodeInto8/16` - /// (mirrored as `TextEncoderEncodeIntoResult` in `headers-handwritten.h`). + /// Counts are `usize` because `written` can be exactly 2^32 (JSC's max + /// ArrayBuffer size), which a `u32` would wrap to 0. `repr(C)` to return by + /// value from `TextEncoder__encodeInto8/16` (`TextEncoderEncodeIntoResult` + /// in `headers-handwritten.h`). #[repr(C)] #[derive(Clone, Copy, Default, Debug)] pub struct EncodeIntoResult { diff --git a/src/jsc/bindings/headers-handwritten.h b/src/jsc/bindings/headers-handwritten.h index 6b38b6089734..092b259fa3e9 100644 --- a/src/jsc/bindings/headers-handwritten.h +++ b/src/jsc/bindings/headers-handwritten.h @@ -396,9 +396,8 @@ extern "C" void ZigString__freeGlobal(const unsigned char* ptr, size_t len); extern "C" size_t Bun__encoding__writeLatin1(const unsigned char* ptr, size_t len, unsigned char* to, size_t other_len, Encoding encoding); extern "C" size_t Bun__encoding__writeUTF16(const char16_t* ptr, size_t len, unsigned char* to, size_t other_len, Encoding encoding); -// Mirrors `EncodeIntoResult` in bun_core (repr(C)). The counts are size_t -// because `written` can be exactly 2^32 (a full 2^32-byte Uint8Array -// destination), which would wrap a 32-bit count to 0. +// Mirrors `EncodeIntoResult` in bun_core (repr(C)). size_t counts: `written` +// can be exactly 2^32 (a full max-size Uint8Array), which would wrap a u32. typedef struct TextEncoderEncodeIntoResult { size_t read; size_t written; From f4df733b79ace549f93f6e6608d1d4dc8c35de6e Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Sun, 9 Aug 2026 12:30:56 +0000 Subject: [PATCH 4/6] Cover TextEncoder.encode in the 2^32 count test Pre-fix the wrapped count failed encode16's completeness check and the 2^32-byte result fell into the allocating fallback, whose ArrayBuffer length cast panics, so encode() aborted the process rather than miscounting. --- test/js/node/buffer.test.js | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/test/js/node/buffer.test.js b/test/js/node/buffer.test.js index 5bb57a2c9f54..e3c5e44eeaf7 100644 --- a/test/js/node/buffer.test.js +++ b/test/js/node/buffer.test.js @@ -4699,9 +4699,11 @@ it.skipIf(os.totalmem() < 10 * 1024 ** 3)( // The reported byte count itself can be exactly 2**32: a UTF-16 source whose // UTF-8 encoding fills a MAX_LENGTH buffer. Buffer.write/TextEncoder.encodeInto // used to round-trip that count through uint32, reporting 0 even though every -// byte was written. Needs ~10 GiB RSS (4 GiB destination + UTF-16 source). +// byte was written, and TextEncoder.encode aborted (the wrapped count failed +// its completeness check and the fallback's ArrayBuffer length cast panics). +// Needs ~10 GiB RSS per spawn (4 GiB destination + UTF-16 source). it.skipIf(os.totalmem() < 16 * 1024 ** 3)( - "Buffer.write/TextEncoder.encodeInto report a byte count of exactly 2**32 without uint32 wrap", + "Buffer.write/TextEncoder.encodeInto/TextEncoder.encode handle a byte count of exactly 2**32 without uint32 wrap", async () => { const script = ` const N = 2 ** 32; @@ -4737,7 +4739,29 @@ it.skipIf(os.totalmem() < 16 * 1024 ** 3)( stderr: "", }); expect(exitCode).toBe(0); + + // TextEncoder.encode in a second spawn (sequential, so the ~10 GiB + // envelopes don't overlap): the count wrap made this abort, not miscount. + const encodeScript = ` + const N = 2 ** 32; + const str = "\\u0800".repeat((N - 1) / 3) + "a"; + const arr = new TextEncoder().encode(str); + console.log(JSON.stringify({ byteLength: arr.byteLength, tail: Array.from(arr.subarray(N - 3)) })); + `; + await using proc2 = Bun.spawn({ + cmd: [bunExe(), "-e", encodeScript], + env: { ...bunEnv, BUN_GARBAGE_COLLECTOR_LEVEL: "0" }, + stdout: "pipe", + stderr: "pipe", + }); + const [stdout2, stderr2, exitCode2] = await Promise.all([proc2.stdout.text(), proc2.stderr.text(), proc2.exited]); + expect({ stdout: stdout2.trim(), stderr: stderr2 }).toEqual({ + stdout: JSON.stringify({ byteLength: 4294967296, tail: [0xa0, 0x80, 0x61] }), + stderr: "", + }); + expect(exitCode2).toBe(0); }, - // Two full 4.3 GB encode passes take ~1 min under a debug+ASAN build. - 5 * 60 * 1000, + // Each spawn does multiple full 4.3 GB encode passes; ~1 min per spawn + // under a debug+ASAN build. + 10 * 60 * 1000, ); From 3e38026ac63afd84b8d1c1aba42f35ce88191a4f Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Sun, 9 Aug 2026 13:07:53 +0000 Subject: [PATCH 5/6] ci: retrigger From 501e796104f192ea8f01d4f21b521825d9b5c91b Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Sun, 9 Aug 2026 13:13:01 +0000 Subject: [PATCH 6/6] Tighten the 2^32 count test comment --- test/js/node/buffer.test.js | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/test/js/node/buffer.test.js b/test/js/node/buffer.test.js index e3c5e44eeaf7..c3cb78ab288c 100644 --- a/test/js/node/buffer.test.js +++ b/test/js/node/buffer.test.js @@ -4696,12 +4696,9 @@ it.skipIf(os.totalmem() < 10 * 1024 ** 3)( }, ); -// The reported byte count itself can be exactly 2**32: a UTF-16 source whose -// UTF-8 encoding fills a MAX_LENGTH buffer. Buffer.write/TextEncoder.encodeInto -// used to round-trip that count through uint32, reporting 0 even though every -// byte was written, and TextEncoder.encode aborted (the wrapped count failed -// its completeness check and the fallback's ArrayBuffer length cast panics). -// Needs ~10 GiB RSS per spawn (4 GiB destination + UTF-16 source). +// A UTF-16 source can encode to exactly 2**32 UTF-8 bytes (a full MAX_LENGTH +// buffer). Pre-fix, Buffer.write/encodeInto reported the u32-wrapped count (0) +// and TextEncoder.encode aborted. Needs ~10 GiB RSS per spawn. it.skipIf(os.totalmem() < 16 * 1024 ** 3)( "Buffer.write/TextEncoder.encodeInto/TextEncoder.encode handle a byte count of exactly 2**32 without uint32 wrap", async () => {