diff --git a/src/stream.cpp b/src/stream.cpp index 1959afa30..8c53dd33a 100644 --- a/src/stream.cpp +++ b/src/stream.cpp @@ -395,7 +395,7 @@ void Stream::StreamInUtf16() const { // Deal with the next UTF-16 unit if (chLow < 0xD800 || chLow >= 0xE000) { // Easiest case: queue the codepoint and return - QueueUnicodeCodepoint(m_readahead, ch); + QueueUnicodeCodepoint(m_readahead, chLow); return; } // Start the loop over with the new high surrogate diff --git a/test/integration/encoding_test.cpp b/test/integration/encoding_test.cpp index 9bd658637..525bebb36 100644 --- a/test/integration/encoding_test.cpp +++ b/test/integration/encoding_test.cpp @@ -178,5 +178,32 @@ TEST_F(EncodingTest, UTF32BE_BOM) { SetUpEncoding(&EncodeToUtf32BE, true); Run(); } + +// An orphaned UTF-16 high surrogate must decode to a single U+FFFD +// (REPLACEMENT CHARACTER), and the code unit after it must still be decoded. +// Previously the high surrogate was re-emitted as an ill-formed UTF-8 sequence +// and the following character was dropped. +TEST(Utf16SurrogateTest, OrphanHighSurrogateKeepsFollowingChar) { + auto put = [](std::string& s, int unit) { + s += Byte(unit & 0xFF); + s += Byte((unit >> 8) & 0xFF); + }; + + std::string input; + put(input, 0xFEFF); // UTF-16LE BOM + put(input, 'x'); + put(input, ':'); + put(input, ' '); + put(input, 0xD800); // high surrogate with no trailing low surrogate + put(input, 'A'); + + std::stringstream stream(input); + Node node = Load(stream); + // "\xEF\xBF\xBD" is U+FFFD encoded as UTF-8; it stands in for the orphaned + // 0xD800. Before the fix the scalar was "\xEF\xBF\xBD\xED\xA0\x80": the + // replacement followed by 0xD800 itself encoded as (invalid) UTF-8, and 'A' + // was lost. + EXPECT_EQ(node["x"].as(), "\xEF\xBF\xBD" "A"); +} } }