\u{fffd}\u{fffd}\u{fffd} last
", rewritten); + } ++ ++ fn rewrite_with_writes( ++ writes: &[&[u8]], ++ handler: impl FnMut(&mut TextChunk<'_>) -> HandlerResult + 'static, ++ ) -> Veca😀b
` with the write boundary inside 😀 (F0 9F 98 80), ++ // and a handler that conditionally replaces only one of the two ++ // resulting chunks. Each unmodified chunk's raw bytes must begin ++ // and end on a code-point boundary, so the output stays valid ++ // UTF-8 regardless of which neighbour was replaced. ++ let writes: &[&[u8]] = &[b"a\xF0\x9F", b"\x98\x80b
"]; ++ ++ assert_eq!( ++ rewrite_with_writes(writes, |t| { ++ if t.as_str().contains('\u{1F600}') { ++ t.replace(":)", ContentType::Text); ++ } ++ Ok(()) ++ }), ++ b"a:)
", ++ ); ++ ++ assert_eq!( ++ rewrite_with_writes(writes, |t| { ++ if t.as_str() == "a" { ++ t.replace("X", ContentType::Text); ++ } ++ Ok(()) ++ }), ++ "X\u{1F600}b
".as_bytes(), ++ ); ++ ++ assert_eq!( ++ rewrite_with_writes(writes, |_| Ok(())), ++ "a\u{1F600}b
".as_bytes(), ++ ); ++ } ++ ++ #[test] ++ fn write_containing_only_an_incomplete_lead_does_not_emit_an_empty_chunk() { ++ // `a€b
` with the € split as `E2 82` / `AC`. The first text ++ // write (lexeme "a") is followed by one whose decoded text would ++ // be empty (just the buffered `E2 82`): the handler must not be ++ // invoked for that empty chunk, and a later `before()` must not ++ // land between the lead bytes and the continuation. ++ let writes: &[&[u8]] = &[b"a", b"\xE2\x82", b"\xACb
"]; ++ let out = rewrite_with_writes(writes, |t| { ++ assert!(t.last_in_text_node() || !t.as_str().is_empty()); ++ t.before("|", ContentType::Text); ++ Ok(()) ++ }); ++ assert_eq!(std::str::from_utf8(&out).unwrap(), "|a|\u{20AC}b|
"); ++ } ++ ++ #[test] ++ fn non_utf8_bytes_pass_through_when_unmodified() { ++ let input: &[u8] = b"a\x93\xE9\x94\xC0\xAF\xFF\xED\xA0\x80b
c\xE9d"; ++ assert_eq!( ++ rewrite_with_writes(&[input], |_| Ok(())), ++ input, ++ ); ++ } + } + } +--- a/src/transform_stream/dispatcher.rs ++++ b/src/transform_stream/dispatcher.rs +@@ -11,6 +11,7 @@ + use crate::rewritable_units::{DocumentEnd, Serialize, ToToken, Token, TokenCaptureFlags}; + use crate::rewriter::RewritingError; + use encoding_rs::Encoding; ++use std::borrow::Cow; + + pub(crate) struct AuxStartTagInfo<'i> { + pub input: &'i Bytes<'i>, +@@ -149,6 +150,7 @@ + fn text_token_produced( + &mut self, + text: &str, ++ raw: Option"), Buffer.from(body), Buffer.from("
")]); + const rewrite = async (src, spec) => + Buffer.from(await new HTMLRewriter().on("*", spec).transform(new Response(src)).arrayBuffer()); + + // - lone continuation byte, multiple invalid bytes, invalid/valid UTF-8 mixed + // - truncated 2/3/4-byte sequence immediately before the closing tag (these + // are the bytes encoding_rs holds as decoder state and flushes on the + // lastInTextNode chunk) + // - text node that is ONLY a truncated sequence (handler call for the body + // is skipped entirely; only the flush chunk fires) + // - invalid byte past 1 KB (fast-path prefix then slow path for the rest) + // and before 1 KB (slow path loops past one decode-buffer fill) + // - ScriptData text-type and multiple sibling text nodes + const cases = [ + ["lone continuation byte", wrap([0xa9])], + ["multiple invalid bytes", wrap([0xa9, 0xff, 0x80, 0xc0])], + [ + "mixed valid and invalid UTF-8", + Buffer.concat([Buffer.from("héllo"), Buffer.from([0xa9]), Buffer.from("wörld
")]), + ], + [ + "truncated 3-byte lead before close tag", + Buffer.concat([Buffer.from("aa"), Buffer.from([0xe2]), Buffer.from("
")]), + ], + [ + "truncated 3-byte prefix before close tag", + Buffer.concat([Buffer.from("aa"), Buffer.from([0xe2, 0x82]), Buffer.from("
")]), + ], + ["truncated 4-byte prefix before close tag", wrap([0xf0, 0x9f, 0x98])], + ["text node that is only a truncated lead byte", wrap([0xe2])], + [ + "invalid byte after the 1 KB fast-path cutoff", + Buffer.concat([Buffer.from(""), Buffer.alloc(2000, 0x61), Buffer.from([0xa9]), Buffer.from("z
")]), + ], + [ + "invalid byte before the 1 KB fast-path cutoff", + Buffer.concat([Buffer.from(""), Buffer.from([0xa9]), Buffer.alloc(3000, 0x62), Buffer.from("
")]), + ], + ["script text", Buffer.concat([Buffer.from("")])], + [ + "multiple text nodes", + Buffer.concat([ + Buffer.from(""), + Buffer.from([0xa9]), + Buffer.from(""), + Buffer.from([0xff]), + Buffer.from(""), + Buffer.from([0xc0]), + Buffer.from("
"), + ]), + ], + ]; + + describe.each(cases)("%s", (name, src) => { + it("no-op element text handler", async () => { + expect(await rewrite(src, { text() {} })).toEqual(src); + }); + it("no-op onDocument text handler", async () => { + const out = Buffer.from( + await new HTMLRewriter() + .onDocument({ text() {} }) + .transform(new Response(src)) + .arrayBuffer(), + ); + expect(out).toEqual(src); + }); + }); + + it("reading .text observes U+FFFD but the output is still the raw bytes", async () => { + const src = Buffer.concat([Buffer.from("x"), Buffer.from([0xa9]), Buffer.from("y
")]); + let seen = ""; + const out = await rewrite(src, { text: t => void (seen += t.text) }); + expect(seen).toBe("x\uFFFDy"); + expect(out).toEqual(src); + }); + + it("a truncated lead byte is reported as U+FFFD on the lastInTextNode chunk and still round-trips", async () => { + const src = wrap([0xe2]); + const chunks = []; + const out = await rewrite(src, { text: t => chunks.push({ text: t.text, last: t.lastInTextNode }) }); + expect(out).toEqual(src); + expect(chunks).toEqual([{ text: "\uFFFD", last: true }]); + }); + + it("before()/after() keep the chunk body as raw bytes", async () => { + const src = wrap([0xa9]); + const out = await rewrite(src, { + text(t) { + if (!t.lastInTextNode) t.before("[", { html: true }); + if (t.lastInTextNode) t.after("]", { html: true }); + }, + }); + expect(out).toEqual(Buffer.concat([Buffer.from("["), Buffer.from([0xa9]), Buffer.from("]
")])); + }); + + it("replace() and remove() still drop the raw bytes", async () => { + const src = wrap([0xa9]); + expect(await rewrite(src, { text: t => t.replace("X", { html: true }) })).toEqual(Buffer.from("XX
")); + expect(await rewrite(src, { text: t => t.remove() })).toEqual(Buffer.from("")); + }); + + it("text outside the selector passes through even when matched text is replaced", async () => { + //a\xa9b
c\xe9d + const src = Buffer.concat([wrap([0x61, 0xa9, 0x62]), Buffer.from([0x63, 0xe9, 0x64])]); + const out = Buffer.from( + await new HTMLRewriter() + .on("p", { text: t => t.replace("x", { html: true }) }) + .transform(new Response(src)) + .arrayBuffer(), + ); + expect(out).toEqual(Buffer.concat([Buffer.from("xx
c"), Buffer.from([0xe9, 0x64])])); + }); +});