diff --git a/json/lex_string.mbt b/json/lex_string.mbt index b124804fa..250492132 100644 --- a/json/lex_string.mbt +++ b/json/lex_string.mbt @@ -37,87 +37,124 @@ fn @lexbuf.StringScanner::lex_string( } ///| +/// Slow path for strings that contain escapes: re-scans from the start of the +/// string contents, copying ordinary runs and decoding one escape at a time. +/// Each `lexscan` case is one form from RFC 8259 §7; anything else falls to +/// the wildcard, which leaves the cursor at the failure point and raises with +/// the same positions as the per-character lexer this replaced. fn @lexbuf.StringScanner::lex_string_slow( ctx : @lexbuf.StringScanner, ) -> String raise ParseError { let buf = StringBuilder() - let mut start = ctx.cursor - fn flush(end : Int) { - if start > 0 && end > start { - buf.write_view(ctx.data[start:end]) + for ;; { + lexscan ctx { + re"^\"" => break + // The longest run of ordinary characters: anything but the closing + // quote, a backslash, or an unescaped control character. Lone + // surrogates in the input pass through here unchanged, as they did + // with the per-character scan. + re"^[^\"\\\x00-\x1f]+" as run => buf.write_view(run) + re"^\\\"" => buf.write_char('"') + re"^\\\\" => buf.write_char('\\') + re"^\\/" => buf.write_char('/') + re"^\\b" => buf.write_char('\b') + re"^\\f" => buf.write_char('\u{0C}') + re"^\\n" => buf.write_char('\n') + re"^\\r" => buf.write_char('\r') + re"^\\t" => buf.write_char('\t') + // A `\uXXXX` escape denoting a BMP scalar value: the first digit rules + // out the surrogate range D800–DFFF. + re"^\\u([0-9A-Ca-cE-Fe-f][0-9A-Fa-f]{3}|[Dd][0-7][0-9A-Fa-f]{2})" as esc => + buf.write_char(hex_quad_value(esc[2:]).unsafe_to_char()) + // An escaped surrogate pair — a leading half immediately followed by a + // trailing half — decodes to the one character it stands for. An + // escaped half that cannot pair up this way matches no case and is + // rejected by the wildcard, because decoding it would manufacture a + // string containing an unpaired surrogate, which MoonBit strings + // disallow (RFC 8259 calls the behavior for such escapes + // unpredictable; I-JSON forbids them). + re"^\\u[Dd][89ABab][0-9A-Fa-f]{2}\\u[Dd][C-Fc-f][0-9A-Fa-f]{2}" as esc => { + let lead = hex_quad_value(esc[2:6]) + let trail = hex_quad_value(esc[8:12]) + buf.write_char(((lead << 10) + trail - 0x35fdc00).unsafe_to_char()) + } + _ => ctx.lex_string_failure() } } + buf.to_string() +} - for ;; { - match ctx.read_char() { - Some('"') => { - flush(ctx.cursor - 1) - break - } - Some('\\') => { - flush(ctx.cursor - 1) - match ctx.read_char() { - Some('b') => buf.write_char('\b') - Some('f') => buf.write_char('\u{0C}') - Some('n') => buf.write_char('\n') - Some('r') => buf.write_char('\r') - Some('t') => buf.write_char('\t') - Some('"') => buf.write_char('"') - Some('\\') => buf.write_char('\\') - Some('/') => buf.write_char('/') - Some('u') => { - // The backslash that opened this escape. `ctx.cursor` is just - // past the `u`, and `\` and `u` are one code unit each. - let escape_start = ctx.cursor - 2 - let c = ctx.lex_hex_digits(4) - if c is (0xD800..=0xDBFF) { - // A leading-surrogate escape is only meaningful as the first - // half of an escaped surrogate pair; combine it with the - // immediately following trailing-surrogate escape into one - // Unicode scalar value. Anything else would manufacture a - // string containing an unpaired surrogate, which MoonBit - // strings disallow (RFC 8259 calls the behavior for such - // escapes unpredictable; I-JSON forbids them). - match ctx.read_char() { - Some('\\') => () - Some(_) => ctx.unpaired_surrogate(escape_start) - None => raise InvalidEof - } - match ctx.read_char() { - Some('u') => () - Some(_) => ctx.unpaired_surrogate(escape_start) - None => raise InvalidEof - } - let c2 = ctx.lex_hex_digits(4) - if c2 is (0xDC00..=0xDFFF) { - let combined = (c << 10) + c2 - 0x35fdc00 - buf.write_char(combined.unsafe_to_char()) - } else { - ctx.unpaired_surrogate(escape_start) - } - } else if c is (0xDC00..=0xDFFF) { - // A bare trailing-surrogate escape can never form a scalar - // value. - ctx.unpaired_surrogate(escape_start) - } else { - buf.write_char(c.unsafe_to_char()) +///| +/// Decodes exactly four UTF-16 code units of hex digits. The regexes in +/// `lex_string_slow` guarantee the digits, so this cannot fail. +fn hex_quad_value(quad : StringView) -> Int { + for c in quad; r = 0 { + let d = match c { + '0'..='9' => c.to_int() - '0' + 'A'..='F' => c.to_int() - 'A' + 10 + 'a'..='f' => c.to_int() - 'a' + 10 + _ => abort("hex_quad_value: caller guarantees hex digits") + } + continue (r << 4) | d + } nobreak { + r + } +} + +///| +/// Diagnoses the point where `lex_string_slow`'s scan stopped and raises the +/// matching error. The regex cases cover every valid form, so the cursor can +/// only be at the end of input, at an unescaped control character, or at a +/// malformed escape. Re-reading with the same per-character primitives as the +/// old lexer keeps every error position unchanged: a bad escape character or +/// hex digit is blamed itself, while an escape rejected as an unpaired +/// surrogate is blamed at its opening backslash (see `unpaired_surrogate`). +fn[T] @lexbuf.StringScanner::lex_string_failure( + ctx : @lexbuf.StringScanner, +) -> T raise ParseError { + let escape_start = ctx.cursor + match ctx.read_char() { + Some('\\') => + match ctx.read_char() { + Some('u') => { + let c = ctx.lex_hex_digits(4) + if c is (0xD800..=0xDBFF) { + // A leading half whose pair failed to close. Whatever follows + // cannot complete it — the pair case would have matched — but + // scanning it finds where things went wrong: a malformed second + // escape is still reported as its own error, anything else as + // the unpaired surrogate it leaves behind. + match ctx.read_char() { + Some('\\') => () + Some(_) => ctx.unpaired_surrogate(escape_start) + None => raise InvalidEof + } + match ctx.read_char() { + Some('u') => () + Some(_) => ctx.unpaired_surrogate(escape_start) + None => raise InvalidEof } + ctx.lex_hex_digits(4) |> ignore + ctx.unpaired_surrogate(escape_start) + } else if c is (0xDC00..=0xDFFF) { + // A bare trailing-surrogate escape can never form a scalar + // value. + ctx.unpaired_surrogate(escape_start) + } else { + abort("unreachable: the BMP escape case matches this") } - Some(c) => ctx.invalid_char(shift=-c.utf16_len()) - None => raise InvalidEof } - start = ctx.cursor + Some(c) => ctx.invalid_char(shift=-c.utf16_len()) + None => raise InvalidEof } - Some(ch) => - if ch.to_int() < 32 { - ctx.invalid_char(shift=-1) - } else { - continue - } - None => raise InvalidEof - } + Some(ch) => + if ch.to_int() < 32 { + ctx.invalid_char(shift=-1) + } else { + abort("unreachable: ordinary characters match the run case") + } + None => raise InvalidEof } - buf.to_string() } ///| diff --git a/json/lex_string_test.mbt b/json/lex_string_test.mbt index 57cd6b9af..a61cdd314 100644 --- a/json/lex_string_test.mbt +++ b/json/lex_string_test.mbt @@ -231,6 +231,22 @@ test "valid surrogate pairs decode to one scalar value" { assert_true(@json.parse("\"\\u0041\\uFFFD\"") == Json::string("A\u{FFFD}")) } +///| +test "slow path copies astral characters in plain runs" { + // The escapes force the slow path; the runs around them contain characters + // outside the BMP, which the scan must traverse as surrogate pairs rather + // than stopping at (or splitting) either half. + assert_true( + @json.parse("\"\u{1F600}a\\n\u{1D11E}b\u{1F600}\"") == + Json::string("\u{1F600}a\n\u{1D11E}b\u{1F600}"), + ) + // A run that is nothing but astral characters, delimited by escapes. + assert_true( + @json.parse("\"\\t\u{1F600}\u{1F601}\\t\"") == + Json::string("\t\u{1F600}\u{1F601}\t"), + ) +} + ///| test "escaped and raw surrogate halves do not pair up" { // Kept last: before the surrogate rule this input aborted the process diff --git a/json/parse_bench_test.mbt b/json/parse_bench_test.mbt index f90e86c21..4f59542b0 100644 --- a/json/parse_bench_test.mbt +++ b/json/parse_bench_test.mbt @@ -46,3 +46,21 @@ test "bench json parse mixed object array n=1000" (it : @bench.T) { let input = repeat_parse_bench_item(1000) it.bench(fn() { it.keep(try! @json.parse(input)) }) } + +///| +/// Exercises the string slow path: every string carries escapes, so none of +/// them can be returned as a plain slice of the input. +test "bench json parse escape-heavy strings n=2000" (it : @bench.T) { + let item = "\"plain prefix \\n\\t\\\"quoted\\\" \\u0041\\u00e9 pair \\uD83D\\uDE00 tail\"" + let buf = StringBuilder(size_hint=2000 * (item.length() + 1) + 2) + buf.write_char('[') + for i in 0..<2000 { + if i > 0 { + buf.write_char(',') + } + buf.write_string(item) + } + buf.write_char(']') + let input = buf.to_string() + it.bench(fn() { it.keep(try! @json.parse(input)) }) +}