From 823b6ea26836c4c7f6af7724634dd0be395daa30 Mon Sep 17 00:00:00 2001 From: Soma Aishwarya Date: Sun, 30 Aug 2026 18:37:51 +0530 Subject: [PATCH 1/2] fix orphaned utf-16 high surrogate dropping the next code unit --- src/stream.cpp | 2 +- test/integration/encoding_test.cpp | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/src/stream.cpp b/src/stream.cpp index 1959afa30..8c53dd33a 100644 --- a/src/stream.cpp +++ b/src/stream.cpp @@ -395,7 +395,7 @@ void Stream::StreamInUtf16() const { // Deal with the next UTF-16 unit if (chLow < 0xD800 || chLow >= 0xE000) { // Easiest case: queue the codepoint and return - QueueUnicodeCodepoint(m_readahead, ch); + QueueUnicodeCodepoint(m_readahead, chLow); return; } // Start the loop over with the new high surrogate diff --git a/test/integration/encoding_test.cpp b/test/integration/encoding_test.cpp index 9bd658637..acb545422 100644 --- a/test/integration/encoding_test.cpp +++ b/test/integration/encoding_test.cpp @@ -178,5 +178,28 @@ TEST_F(EncodingTest, UTF32BE_BOM) { SetUpEncoding(&EncodeToUtf32BE, true); Run(); } + +// An orphaned UTF-16 high surrogate must decode to a single replacement +// character, and the code unit after it must still be decoded. Previously the +// high surrogate was re-emitted as an ill-formed UTF-8 sequence and the +// following character was dropped. +TEST(Utf16SurrogateTest, OrphanHighSurrogateKeepsFollowingChar) { + auto put = [](std::string& s, int unit) { + s += Byte(unit & 0xFF); + s += Byte((unit >> 8) & 0xFF); + }; + + std::string input; + put(input, 0xFEFF); // UTF-16LE BOM + put(input, 'x'); + put(input, ':'); + put(input, ' '); + put(input, 0xD800); // high surrogate with no trailing low surrogate + put(input, 'A'); + + std::stringstream stream(input); + Node node = Load(stream); + EXPECT_EQ(node["x"].as(), "\xEF\xBF\xBD" "A"); +} } } From 6f3b14fdd4eb29372a58aaf36421ee144a899804 Mon Sep 17 00:00:00 2001 From: Soma Aishwarya Date: Wed, 9 Sep 2026 03:11:03 +0530 Subject: [PATCH 2/2] Spell out the U+FFFD byte sequence in the surrogate test comment --- test/integration/encoding_test.cpp | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/test/integration/encoding_test.cpp b/test/integration/encoding_test.cpp index acb545422..525bebb36 100644 --- a/test/integration/encoding_test.cpp +++ b/test/integration/encoding_test.cpp @@ -179,10 +179,10 @@ TEST_F(EncodingTest, UTF32BE_BOM) { Run(); } -// An orphaned UTF-16 high surrogate must decode to a single replacement -// character, and the code unit after it must still be decoded. Previously the -// high surrogate was re-emitted as an ill-formed UTF-8 sequence and the -// following character was dropped. +// An orphaned UTF-16 high surrogate must decode to a single U+FFFD +// (REPLACEMENT CHARACTER), and the code unit after it must still be decoded. +// Previously the high surrogate was re-emitted as an ill-formed UTF-8 sequence +// and the following character was dropped. TEST(Utf16SurrogateTest, OrphanHighSurrogateKeepsFollowingChar) { auto put = [](std::string& s, int unit) { s += Byte(unit & 0xFF); @@ -199,6 +199,10 @@ TEST(Utf16SurrogateTest, OrphanHighSurrogateKeepsFollowingChar) { std::stringstream stream(input); Node node = Load(stream); + // "\xEF\xBF\xBD" is U+FFFD encoded as UTF-8; it stands in for the orphaned + // 0xD800. Before the fix the scalar was "\xEF\xBF\xBD\xED\xA0\x80": the + // replacement followed by 0xD800 itself encoded as (invalid) UTF-8, and 'A' + // was lost. EXPECT_EQ(node["x"].as(), "\xEF\xBF\xBD" "A"); } }