\xF0\xF0\x9F\xF0\x9F\x98
"; ++ let mut out = Vec::new(); ++ rewrite_html_to_vec( ++ input, ++ UTF_8, ++ vec![], ++ vec![doc_text!(|c| { ++ if c.last_in_text_node() { ++ c.after(" last", ContentType::Text); ++ } ++ Ok(()) ++ })], ++ &mut out, ++ ); ++ assert_eq!(b"\xF0\xF0\x9F\xF0\x9F\x98 last
"[..], out[..]); ++ } ++ ++ #[test] ++ fn non_utf8_bytes_pass_through_when_unmodified() { ++ let input = b"a\x93\xE9\x94\xC0\xAF\xFF\xED\xA0\x80b
c\xE9d"; ++ let mut out = Vec::new(); ++ rewrite_html_to_vec( ++ input, ++ UTF_8, ++ vec![], ++ vec![doc_text!(|_| Ok(()))], ++ &mut out, ++ ); ++ assert_eq!(input[..], out[..]); ++ } ++ ++ #[test] ++ fn non_utf8_bytes_replaced_when_modified() { + let rewritten = rewrite_text_chunk(b"\xF0\xF0\x9F\xF0\x9F\x98
", UTF_8, |c| { +- if c.last_in_text_node() { +- c.after(" last", ContentType::Text); +- } ++ c.as_mut_str(); + }); +- assert_eq!("\u{fffd}\u{fffd}\u{fffd} last
", rewritten); ++ assert_eq!("\u{fffd}\u{fffd}\u{fffd}
", rewritten); + } + } + } +--- a/src/rewritable_units/text_decoder.rs ++++ b/src/rewritable_units/text_decoder.rs +@@ -11,8 +11,14 @@ + text_buffer: String, + } + +-pub(crate) type OutputHandlerCallback<'tmp> = +- dyn FnMut(&str, bool, &'static Encoding, SourceLocation) -> Result<(), RewritingError> + 'tmp; ++pub(crate) type OutputHandlerCallback<'tmp> = dyn FnMut( ++ &str, ++ &[u8], ++ bool, ++ &'static Encoding, ++ SourceLocation, ++ ) -> Result<(), RewritingError> ++ + 'tmp; + + impl TextDecoder { + #[inline] +@@ -63,7 +69,13 @@ + SourceLocation::from_start_len(next_source_location_bytes_start, utf8_text.len()); + next_source_location_bytes_start = source_location.bytes().end; + +- (output_handler)(utf8_text, really_last, encoding, source_location)?; ++ (output_handler)( ++ utf8_text, ++ utf8_text.as_bytes(), ++ really_last, ++ encoding, ++ source_location, ++ )?; + + if really_last { + debug_assert!(self.pending_text_streaming_decoder.is_none()); +@@ -89,7 +101,16 @@ + SourceLocation::from_start_len(next_source_location_bytes_start, read); + next_source_location_bytes_start = source_location.bytes().end; + +- if written > 0 || last_in_text_node { ++ // The raw input bytes that this decode step consumed. A non-mutating ++ // text handler re-emits these bytes verbatim so that registering a ++ // handler does not itself change the output. On the flush call ++ // (`last_in_text_node` with empty input) the decoder may still emit ++ // replacement characters for bytes it buffered from an earlier ++ // `feed_text`; those bytes already went out with that earlier ++ // chunk's `raw`, so the raw slice here is correctly empty. ++ let raw = raw_input.get(..read).unwrap_or_default(); ++ ++ if written > 0 || last_in_text_node || !raw.is_empty() { + // the last call to feed_text() may make multiple calls to output_handler, + // but only one call to output_handler can be *the* last one. + let really_last = last_in_text_node && finished_decoding; +@@ -97,6 +118,7 @@ + (output_handler)( + // this will always be in bounds, but unwrap_or_default optimizes better + buffer.get(..written).unwrap_or_default(), ++ raw, + really_last, + encoding, + source_location, +--- a/src/rewritable_units/mod.rs ++++ b/src/rewritable_units/mod.rs +@@ -151,4 +151,27 @@ + + output.into() + } ++ ++ pub(crate) fn rewrite_html_to_vec<'h>( ++ html: &[u8], ++ encoding: &'static Encoding, ++ element_content_handlers: Vec<(Cow<'_, Selector>, ElementContentHandlers<'h>)>, ++ document_content_handlers: Veca + cp1252 quotes/é + overlong + FF + WTF-8 lone surrogate + b
c é d + // prettier-ignore + const bytes = new Uint8Array([ + 0x3c, 0x70, 0x3e, 0x61, 0x93, 0xe9, 0x94, 0xc0, 0xaf, 0xff, 0xed, 0xa0, + 0x80, 0x62, 0x3c, 0x2f, 0x70, 0x3e, 0x63, 0xe9, 0x64, + ]); + const transform = async setup => { + const rewriter = new HTMLRewriter(); + setup(rewriter); + return new Uint8Array(await rewriter.transform(new Response(bytes)).arrayBuffer()); + }; + + it.each([ + ["no handlers", r => r], + ["element handler", r => r.on("p", { element() {} })], + ["comments handler", r => r.on("p", { comments() {} })], + ["text handler that reads .text", r => r.on("p", { text(t) { void t.text; } })], // prettier-ignore + ["onDocument text handler", r => r.onDocument({ text() {} })], + ["text handler that inserts around the chunk", r => r.on("p", { text(t) { t.before(""); t.after(""); } })], // prettier-ignore + ])("%s", async (_, setup) => { + expect(await transform(setup)).toEqual(bytes); + }); + + it("text outside the selector is untouched", async () => { + const out = await transform(r => r.on("p", { text(t) { t.replace("x"); } })); // prettier-ignore + // Thetext is replaced (including the empty last-in-node chunk), but + // the trailing `c \xe9 d` outside the selector passes through verbatim. + expect(out).toEqual(new Uint8Array([...Buffer.from("
xx
c"), 0xe9, 0x64])); + }); + }); + it("it supports selfClosing", async () => { const selfClosing = {}; await new HTMLRewriter() From f71ac4291933b6cfca32f55ca9fd30201a686142 Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Sun, 26 Jul 2026 11:39:42 +0000 Subject: [PATCH 2/5] carry decoder-buffered tail bytes across chunks so raw stays aligned with text The previous revision emitted an extra empty-text handler call for a text node that is only an incomplete UTF-8 prefix (e.g.\xE2
), and left raw/text misaligned by up to 3 bytes when encoding_rs held an incomplete sequence across a feed_text boundary. TextDecoder now owns the <=3 raw bytes that encoding_rs has buffered and attributes them to the next emitted chunk's raw, so the handler sees the same chunks it always did and each chunk's raw is exactly the bytes its text was decoded from. utf8_incomplete_tail_len computes the buffered length via std from_utf8's error_len()==None, which matches encoding_rs's WHATWG UTF-8 state machine bit-for-bit. Adds a 26-test matrix covering truncated 2/3/4-byte prefixes immediately before a close tag, a text node that is only a truncated lead byte, and the >1KB fast-path/slow-path split, plus a test that pins the handler's observed chunk sequence to today's behavior. --- .../lolhtml/text-chunk-raw-passthrough.patch | 356 ++++++++++-------- test/js/workerd/html-rewriter.test.js | 79 ++++ 2 files changed, 273 insertions(+), 162 deletions(-) diff --git a/patches/lolhtml/text-chunk-raw-passthrough.patch b/patches/lolhtml/text-chunk-raw-passthrough.patch index 8553b13b0da..1dfb307d9a6 100644 --- a/patches/lolhtml/text-chunk-raw-passthrough.patch +++ b/patches/lolhtml/text-chunk-raw-passthrough.patch @@ -1,27 +1,191 @@ +--- a/src/rewritable_units/text_decoder.rs ++++ b/src/rewritable_units/text_decoder.rs +@@ -1,6 +1,7 @@ + use crate::base::{Bytes, SharedEncoding, SourceLocation, Spanned}; + use crate::rewriter::RewritingError; + use encoding_rs::{CoderResult, Decoder, Encoding, UTF_8}; ++use std::borrow::Cow; + + const DEFAULT_BUFFER_LEN: usize = if cfg!(test) { 13 } else { 1024 }; + +@@ -8,11 +9,23 @@ + encoding: SharedEncoding, + pending_source_location_bytes_start: usize, + pending_text_streaming_decoder: Option\xF0\xF0\x9F\xF0\x9F\x98
"; -+ let mut out = Vec::new(); -+ rewrite_html_to_vec( -+ input, -+ UTF_8, -+ vec![], -+ vec![doc_text!(|c| { -+ if c.last_in_text_node() { -+ c.after(" last", ContentType::Text); -+ } -+ Ok(()) -+ })], -+ &mut out, -+ ); -+ assert_eq!(b"\xF0\xF0\x9F\xF0\x9F\x98 last
"[..], out[..]); -+ } -+ -+ #[test] -+ fn non_utf8_bytes_pass_through_when_unmodified() { -+ let input = b"a\x93\xE9\x94\xC0\xAF\xFF\xED\xA0\x80b
c\xE9d"; -+ let mut out = Vec::new(); -+ rewrite_html_to_vec( -+ input, -+ UTF_8, -+ vec![], -+ vec![doc_text!(|_| Ok(()))], -+ &mut out, -+ ); -+ assert_eq!(input[..], out[..]); -+ } -+ -+ #[test] -+ fn non_utf8_bytes_replaced_when_modified() { - let rewritten = rewrite_text_chunk(b"\xF0\xF0\x9F\xF0\x9F\x98
", UTF_8, |c| { -- if c.last_in_text_node() { -- c.after(" last", ContentType::Text); -- } -+ c.as_mut_str(); - }); -- assert_eq!("\u{fffd}\u{fffd}\u{fffd} last
", rewritten); -+ assert_eq!("\u{fffd}\u{fffd}\u{fffd}
", rewritten); - } - } - } ---- a/src/rewritable_units/text_decoder.rs -+++ b/src/rewritable_units/text_decoder.rs -@@ -11,8 +11,14 @@ - text_buffer: String, - } - --pub(crate) type OutputHandlerCallback<'tmp> = -- dyn FnMut(&str, bool, &'static Encoding, SourceLocation) -> Result<(), RewritingError> + 'tmp; -+pub(crate) type OutputHandlerCallback<'tmp> = dyn FnMut( -+ &str, -+ &[u8], -+ bool, -+ &'static Encoding, -+ SourceLocation, -+ ) -> Result<(), RewritingError> -+ + 'tmp; - - impl TextDecoder { - #[inline] -@@ -63,7 +69,13 @@ - SourceLocation::from_start_len(next_source_location_bytes_start, utf8_text.len()); - next_source_location_bytes_start = source_location.bytes().end; - -- (output_handler)(utf8_text, really_last, encoding, source_location)?; -+ (output_handler)( -+ utf8_text, -+ utf8_text.as_bytes(), -+ really_last, -+ encoding, -+ source_location, -+ )?; - - if really_last { - debug_assert!(self.pending_text_streaming_decoder.is_none()); -@@ -89,7 +101,16 @@ - SourceLocation::from_start_len(next_source_location_bytes_start, read); - next_source_location_bytes_start = source_location.bytes().end; - -- if written > 0 || last_in_text_node { -+ // The raw input bytes that this decode step consumed. A non-mutating -+ // text handler re-emits these bytes verbatim so that registering a -+ // handler does not itself change the output. On the flush call -+ // (`last_in_text_node` with empty input) the decoder may still emit -+ // replacement characters for bytes it buffered from an earlier -+ // `feed_text`; those bytes already went out with that earlier -+ // chunk's `raw`, so the raw slice here is correctly empty. -+ let raw = raw_input.get(..read).unwrap_or_default(); -+ -+ if written > 0 || last_in_text_node || !raw.is_empty() { - // the last call to feed_text() may make multiple calls to output_handler, - // but only one call to output_handler can be *the* last one. - let really_last = last_in_text_node && finished_decoding; -@@ -97,6 +118,7 @@ - (output_handler)( - // this will always be in bounds, but unwrap_or_default optimizes better - buffer.get(..written).unwrap_or_default(), -+ raw, - really_last, - encoding, - source_location, ---- a/src/rewritable_units/mod.rs -+++ b/src/rewritable_units/mod.rs -@@ -151,4 +151,27 @@ - - output.into() - } -+ -+ pub(crate) fn rewrite_html_to_vec<'h>( -+ html: &[u8], -+ encoding: &'static Encoding, -+ element_content_handlers: Vec<(Cow<'_, Selector>, ElementContentHandlers<'h>)>, -+ document_content_handlers: Vec"), Buffer.from(body), Buffer.from("
")]); + const rewrite = async (src, spec) => + Buffer.from(await new HTMLRewriter().on("*", spec).transform(new Response(src)).arrayBuffer()); + + // - lone continuation byte, multiple invalid bytes, invalid/valid UTF-8 mixed + // - truncated 2/3/4-byte sequence immediately before the closing tag (these + // are the bytes encoding_rs holds as decoder state and flushes on the + // lastInTextNode chunk) + // - text node that is ONLY a truncated sequence (handler call for the body + // is skipped entirely; only the flush chunk fires) + // - invalid byte past 1 KB (fast-path prefix then slow path for the rest) + // and before 1 KB (slow path loops past one decode-buffer fill) + // - ScriptData text-type and multiple sibling text nodes + const cases = [ + ["lone continuation byte", wrap([0xa9])], + ["multiple invalid bytes", wrap([0xa9, 0xff, 0x80, 0xc0])], + ["mixed valid and invalid UTF-8", Buffer.concat([Buffer.from("héllo"), Buffer.from([0xa9]), Buffer.from("wörld
")])], + ["truncated 3-byte lead before close tag", Buffer.concat([Buffer.from("aa"), Buffer.from([0xe2]), Buffer.from("
")])], + ["truncated 3-byte prefix before close tag", Buffer.concat([Buffer.from("aa"), Buffer.from([0xe2, 0x82]), Buffer.from("
")])], + ["truncated 4-byte prefix before close tag", wrap([0xf0, 0x9f, 0x98])], + ["text node that is only a truncated lead byte", wrap([0xe2])], + ["invalid byte after the 1 KB fast-path cutoff", Buffer.concat([Buffer.from(""), Buffer.alloc(2000, 0x61), Buffer.from([0xa9]), Buffer.from("z
")])], + ["invalid byte before the 1 KB fast-path cutoff", Buffer.concat([Buffer.from(""), Buffer.from([0xa9]), Buffer.alloc(3000, 0x62), Buffer.from("
")])], + ["script text", Buffer.concat([Buffer.from("")])], + ["multiple text nodes", Buffer.concat([Buffer.from(""), Buffer.from([0xa9]), Buffer.from(""), Buffer.from([0xff]), Buffer.from(""), Buffer.from([0xc0]), Buffer.from("
")])], + ]; + + describe.each(cases)("%s", (name, src) => { + it("no-op element text handler", async () => { + expect(await rewrite(src, { text() {} })).toEqual(src); + }); + it("no-op onDocument text handler", async () => { + const out = Buffer.from(await new HTMLRewriter().onDocument({ text() {} }).transform(new Response(src)).arrayBuffer()); + expect(out).toEqual(src); + }); + }); + + it("reading .text observes U+FFFD but the output is still the raw bytes", async () => { + const src = Buffer.concat([Buffer.from("x"), Buffer.from([0xa9]), Buffer.from("y
")]); + let seen = ""; + const out = await rewrite(src, { text: t => void (seen += t.text) }); + expect(seen).toBe("x\uFFFDy"); + expect(out).toEqual(src); + }); + + it("a truncated lead byte is reported as U+FFFD on the lastInTextNode chunk and still round-trips", async () => { + const src = wrap([0xe2]); + const chunks = []; + const out = await rewrite(src, { text: t => chunks.push({ text: t.text, last: t.lastInTextNode }) }); + expect(out).toEqual(src); + expect(chunks).toEqual([{ text: "\uFFFD", last: true }]); + }); + + it("before()/after() keep the chunk body as raw bytes", async () => { + const src = wrap([0xa9]); + const out = await rewrite(src, { + text(t) { + if (!t.lastInTextNode) t.before("[", { html: true }); + if (t.lastInTextNode) t.after("]", { html: true }); + }, + }); + expect(out).toEqual(Buffer.concat([Buffer.from("["), Buffer.from([0xa9]), Buffer.from("]
")])); + }); + + it("replace() and remove() still drop the raw bytes", async () => { + const src = wrap([0xa9]); + expect(await rewrite(src, { text: t => t.replace("X", { html: true }) })).toEqual(Buffer.from("XX
")); + expect(await rewrite(src, { text: t => t.remove() })).toEqual(Buffer.from("")); + }); +}); From fe41a414ec5050b956e472417cbae0cae4988247 Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Sun, 26 Jul 2026 11:41:50 +0000 Subject: [PATCH 3/5] [autofix.ci] apply automated fixes --- test/js/workerd/html-rewriter.test.js | 45 ++++++++++++++++++++++----- 1 file changed, 38 insertions(+), 7 deletions(-) diff --git a/test/js/workerd/html-rewriter.test.js b/test/js/workerd/html-rewriter.test.js index 3ef4c7f004c..b8013cdcacd 100644 --- a/test/js/workerd/html-rewriter.test.js +++ b/test/js/workerd/html-rewriter.test.js @@ -1318,15 +1318,41 @@ describe("text handler does not transcode unmodified non-UTF-8 bytes", () => { const cases = [ ["lone continuation byte", wrap([0xa9])], ["multiple invalid bytes", wrap([0xa9, 0xff, 0x80, 0xc0])], - ["mixed valid and invalid UTF-8", Buffer.concat([Buffer.from("héllo"), Buffer.from([0xa9]), Buffer.from("wörld
")])], - ["truncated 3-byte lead before close tag", Buffer.concat([Buffer.from("aa"), Buffer.from([0xe2]), Buffer.from("
")])], - ["truncated 3-byte prefix before close tag", Buffer.concat([Buffer.from("aa"), Buffer.from([0xe2, 0x82]), Buffer.from("
")])], + [ + "mixed valid and invalid UTF-8", + Buffer.concat([Buffer.from("héllo"), Buffer.from([0xa9]), Buffer.from("wörld
")]), + ], + [ + "truncated 3-byte lead before close tag", + Buffer.concat([Buffer.from("aa"), Buffer.from([0xe2]), Buffer.from("
")]), + ], + [ + "truncated 3-byte prefix before close tag", + Buffer.concat([Buffer.from("aa"), Buffer.from([0xe2, 0x82]), Buffer.from("
")]), + ], ["truncated 4-byte prefix before close tag", wrap([0xf0, 0x9f, 0x98])], ["text node that is only a truncated lead byte", wrap([0xe2])], - ["invalid byte after the 1 KB fast-path cutoff", Buffer.concat([Buffer.from(""), Buffer.alloc(2000, 0x61), Buffer.from([0xa9]), Buffer.from("z
")])], - ["invalid byte before the 1 KB fast-path cutoff", Buffer.concat([Buffer.from(""), Buffer.from([0xa9]), Buffer.alloc(3000, 0x62), Buffer.from("
")])], + [ + "invalid byte after the 1 KB fast-path cutoff", + Buffer.concat([Buffer.from(""), Buffer.alloc(2000, 0x61), Buffer.from([0xa9]), Buffer.from("z
")]), + ], + [ + "invalid byte before the 1 KB fast-path cutoff", + Buffer.concat([Buffer.from(""), Buffer.from([0xa9]), Buffer.alloc(3000, 0x62), Buffer.from("
")]), + ], ["script text", Buffer.concat([Buffer.from("")])], - ["multiple text nodes", Buffer.concat([Buffer.from(""), Buffer.from([0xa9]), Buffer.from(""), Buffer.from([0xff]), Buffer.from(""), Buffer.from([0xc0]), Buffer.from("
")])], + [ + "multiple text nodes", + Buffer.concat([ + Buffer.from(""), + Buffer.from([0xa9]), + Buffer.from(""), + Buffer.from([0xff]), + Buffer.from(""), + Buffer.from([0xc0]), + Buffer.from("
"), + ]), + ], ]; describe.each(cases)("%s", (name, src) => { @@ -1334,7 +1360,12 @@ describe("text handler does not transcode unmodified non-UTF-8 bytes", () => { expect(await rewrite(src, { text() {} })).toEqual(src); }); it("no-op onDocument text handler", async () => { - const out = Buffer.from(await new HTMLRewriter().onDocument({ text() {} }).transform(new Response(src)).arrayBuffer()); + const out = Buffer.from( + await new HTMLRewriter() + .onDocument({ text() {} }) + .transform(new Response(src)) + .arrayBuffer(), + ); expect(out).toEqual(src); }); }); From 23e2cc6d8d505bc75830b0c933624c9cc09bed55 Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Sun, 26 Jul 2026 12:02:17 +0000 Subject: [PATCH 4/5] add lol-html unit tests for raw/text alignment across write boundaries Locks in that a conditional replace on one of two text chunks straddling a UTF-8 code point produces valid UTF-8 either way, and that a write carrying only an incomplete lead byte never fires the handler with an empty non-last chunk. --- .../lolhtml/text-chunk-raw-passthrough.patch | 96 +++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/patches/lolhtml/text-chunk-raw-passthrough.patch b/patches/lolhtml/text-chunk-raw-passthrough.patch index 1dfb307d9a6..54f6582ed3f 100644 --- a/patches/lolhtml/text-chunk-raw-passthrough.patch +++ b/patches/lolhtml/text-chunk-raw-passthrough.patch @@ -230,6 +230,102 @@ TextType::PlainText, true, encoding, +@@ -444,6 +456,7 @@ + + mod serialization { + use super::*; ++ use crate::{ElementContentHandlers, HandlerResult, HtmlRewriter, Settings}; + + const HTML: &str = "Lorem ipsum dolor sit amet, cÔnsectetur adipiscing elit, sed do eiusmod tempor \ + incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud \ +@@ -568,5 +581,87 @@ + }); + assert_eq!("\u{fffd}\u{fffd}\u{fffd} last
", rewritten); + } ++ ++ fn rewrite_with_writes( ++ writes: &[&[u8]], ++ handler: impl FnMut(&mut TextChunk<'_>) -> HandlerResult + 'static, ++ ) -> Veca😀b
` with the write boundary inside 😀 (F0 9F 98 80), ++ // and a handler that conditionally replaces only one of the two ++ // resulting chunks. Each unmodified chunk's raw bytes must begin ++ // and end on a code-point boundary, so the output stays valid ++ // UTF-8 regardless of which neighbour was replaced. ++ let writes: &[&[u8]] = &[b"a\xF0\x9F", b"\x98\x80b
"]; ++ ++ assert_eq!( ++ rewrite_with_writes(writes, |t| { ++ if t.as_str().contains('\u{1F600}') { ++ t.replace(":)", ContentType::Text); ++ } ++ Ok(()) ++ }), ++ b"a:)
", ++ ); ++ ++ assert_eq!( ++ rewrite_with_writes(writes, |t| { ++ if t.as_str() == "a" { ++ t.replace("X", ContentType::Text); ++ } ++ Ok(()) ++ }), ++ "X\u{1F600}b
".as_bytes(), ++ ); ++ ++ assert_eq!( ++ rewrite_with_writes(writes, |_| Ok(())), ++ "a\u{1F600}b
".as_bytes(), ++ ); ++ } ++ ++ #[test] ++ fn write_containing_only_an_incomplete_lead_does_not_emit_an_empty_chunk() { ++ // `a€b
` with the € split as `E2 82` / `AC`. The first text ++ // write (lexeme "a") is followed by one whose decoded text would ++ // be empty (just the buffered `E2 82`): the handler must not be ++ // invoked for that empty chunk, and a later `before()` must not ++ // land between the lead bytes and the continuation. ++ let writes: &[&[u8]] = &[b"a", b"\xE2\x82", b"\xACb
"]; ++ let out = rewrite_with_writes(writes, |t| { ++ assert!(t.last_in_text_node() || !t.as_str().is_empty()); ++ t.before("|", ContentType::Text); ++ Ok(()) ++ }); ++ assert_eq!(std::str::from_utf8(&out).unwrap(), "|a|\u{20AC}b|
"); ++ } ++ ++ #[test] ++ fn non_utf8_bytes_pass_through_when_unmodified() { ++ let input: &[u8] = b"a\x93\xE9\x94\xC0\xAF\xFF\xED\xA0\x80b
c\xE9d"; ++ assert_eq!( ++ rewrite_with_writes(&[input], |_| Ok(())), ++ input, ++ ); ++ } + } + } --- a/src/transform_stream/dispatcher.rs +++ b/src/transform_stream/dispatcher.rs @@ -11,6 +11,7 @@ From 9c5e0c8995b5c2d2972b80f9d8a251902cc313c7 Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Sun, 26 Jul 2026 12:15:16 +0000 Subject: [PATCH 5/5] scope raw-byte passthrough to UTF-8; drop redundant test block For non-UTF-8 document encodings the decoder slow path now passes raw=None and falls back to re-encoding the decoded text, which is the pre-patch behaviour: the tail accounting is UTF-8-specific and Bun only ever builds the rewriter with UTF-8. The fast path's ASCII prefix is byte-identical in every ASCII-compatible encoding so it keeps its borrowed raw slice. The earlier 7-case observer block duplicated the larger matrix further down without proving the handler fired; dropped it and moved its one unique case (text outside the selector) into the matrix block. --- .../lolhtml/text-chunk-raw-passthrough.patch | 47 +++++++++++-------- test/js/workerd/html-rewriter.test.js | 44 +++++------------ 2 files changed, 40 insertions(+), 51 deletions(-) diff --git a/patches/lolhtml/text-chunk-raw-passthrough.patch b/patches/lolhtml/text-chunk-raw-passthrough.patch index 54f6582ed3f..57f808e1d2a 100644 --- a/patches/lolhtml/text-chunk-raw-passthrough.patch +++ b/patches/lolhtml/text-chunk-raw-passthrough.patch @@ -89,47 +89,56 @@ if really_last { debug_assert!(self.pending_text_streaming_decoder.is_none()); -@@ -89,6 +110,28 @@ +@@ -89,6 +110,37 @@ SourceLocation::from_start_len(next_source_location_bytes_start, read); next_source_location_bytes_start = source_location.bytes().end; -+ // Raw-byte accounting for this chunk: ++ // Raw-byte accounting for this chunk (UTF-8 only): + // raw(chunk) = pending_raw ++ raw_input[..read - tail] + // pending_raw' = raw_input[read - tail .. read] + // where `tail` is the number of bytes encoding_rs kept as internal + // state (an incomplete sequence at end-of-input). OutputFull and + // last=true never leave state, so `tail` is only nonzero on the -+ // final InputEmpty of a non-last call. -+ let consumed = raw_input.get(..read).unwrap_or_default(); -+ let tail = if finished_decoding && !last_in_text_node && encoding == UTF_8 { -+ utf8_incomplete_tail_len(consumed) ++ // final InputEmpty of a non-last call. For non-UTF-8 document ++ // encodings `raw` is `None` and the serializer re-encodes the ++ // decoded text, which is the pre-patch behaviour: the tail-length ++ // computation is UTF-8-specific and other multi-byte encodings are ++ // not worth the extra state. ++ let (raw, carry): (Optiona + cp1252 quotes/é + overlong + FF + WTF-8 lone surrogate + b
c é d - // prettier-ignore - const bytes = new Uint8Array([ - 0x3c, 0x70, 0x3e, 0x61, 0x93, 0xe9, 0x94, 0xc0, 0xaf, 0xff, 0xed, 0xa0, - 0x80, 0x62, 0x3c, 0x2f, 0x70, 0x3e, 0x63, 0xe9, 0x64, - ]); - const transform = async setup => { - const rewriter = new HTMLRewriter(); - setup(rewriter); - return new Uint8Array(await rewriter.transform(new Response(bytes)).arrayBuffer()); - }; - - it.each([ - ["no handlers", r => r], - ["element handler", r => r.on("p", { element() {} })], - ["comments handler", r => r.on("p", { comments() {} })], - ["text handler that reads .text", r => r.on("p", { text(t) { void t.text; } })], // prettier-ignore - ["onDocument text handler", r => r.onDocument({ text() {} })], - ["text handler that inserts around the chunk", r => r.on("p", { text(t) { t.before(""); t.after(""); } })], // prettier-ignore - ])("%s", async (_, setup) => { - expect(await transform(setup)).toEqual(bytes); - }); - - it("text outside the selector is untouched", async () => { - const out = await transform(r => r.on("p", { text(t) { t.replace("x"); } })); // prettier-ignore - // Thetext is replaced (including the empty last-in-node chunk), but - // the trailing `c \xe9 d` outside the selector passes through verbatim. - expect(out).toEqual(new Uint8Array([...Buffer.from("
xx
c"), 0xe9, 0x64])); - }); - }); - it("it supports selfClosing", async () => { const selfClosing = {}; await new HTMLRewriter() @@ -1402,4 +1370,16 @@ describe("text handler does not transcode unmodified non-UTF-8 bytes", () => { expect(await rewrite(src, { text: t => t.replace("X", { html: true }) })).toEqual(Buffer.from("XX
")); expect(await rewrite(src, { text: t => t.remove() })).toEqual(Buffer.from("")); }); + + it("text outside the selector passes through even when matched text is replaced", async () => { + //a\xa9b
c\xe9d + const src = Buffer.concat([wrap([0x61, 0xa9, 0x62]), Buffer.from([0x63, 0xe9, 0x64])]); + const out = Buffer.from( + await new HTMLRewriter() + .on("p", { text: t => t.replace("x", { html: true }) }) + .transform(new Response(src)) + .arrayBuffer(), + ); + expect(out).toEqual(Buffer.concat([Buffer.from("xx
c"), Buffer.from([0xe9, 0x64])])); + }); });