fix orphaned utf-16 high surrogate dropping the next code unit

src/stream.cpp

@@ -395,7 +395,7 @@ void Stream::StreamInUtf16() const {
         // Deal with the next UTF-16 unit
         if (chLow < 0xD800 || chLow >= 0xE000) {
           // Easiest case: queue the codepoint and return
-          QueueUnicodeCodepoint(m_readahead, ch);
+          QueueUnicodeCodepoint(m_readahead, chLow);
           return;
         }
         // Start the loop over with the new high surrogate

test/integration/encoding_test.cpp

@@ -178,5 +178,28 @@ TEST_F(EncodingTest, UTF32BE_BOM) {
   SetUpEncoding(&EncodeToUtf32BE, true);
   Run();
 }
+
+// An orphaned UTF-16 high surrogate must decode to a single replacement
+// character, and the code unit after it must still be decoded. Previously the
+// high surrogate was re-emitted as an ill-formed UTF-8 sequence and the
+// following character was dropped.
+TEST(Utf16SurrogateTest, OrphanHighSurrogateKeepsFollowingChar) {
+  auto put = [](std::string& s, int unit) {
+    s += Byte(unit & 0xFF);
+    s += Byte((unit >> 8) & 0xFF);
+  };
+
+  std::string input;
+  put(input, 0xFEFF);  // UTF-16LE BOM
+  put(input, 'x');
+  put(input, ':');
+  put(input, ' ');
+  put(input, 0xD800);  // high surrogate with no trailing low surrogate
+  put(input, 'A');
+
+  std::stringstream stream(input);
+  Node node = Load(stream);
+  EXPECT_EQ(node["x"].as<std::string>(), "\xEF\xBF\xBD" "A");
+}
 }
 }