diff --git a/src/parsers/toml/lexer.rs b/src/parsers/toml/lexer.rs index cfe8c61a5cad..ca971fed13b3 100644 --- a/src/parsers/toml/lexer.rs +++ b/src/parsers/toml/lexer.rs @@ -656,8 +656,9 @@ impl<'a> Lexer<'a> { } self.step(); self.token = T::t_string_literal; - self.string_literal_slice = - &self.source.contents[start + 2..end]; + self.string_literal_slice = &self.source.contents + [trim_multiline_start(&self.source.contents, start + 2) + ..end]; return Ok(()); } _ => {} @@ -747,13 +748,15 @@ impl<'a> Lexer<'a> { self.step(); self.token = T::t_string_literal; + let content_start = + trim_multiline_start(&self.source.contents, start + 2); if needs_slow_pass { - slice_lo = start + 2; + slice_lo = content_start; slice_hi = end; break; } self.string_literal_slice = - &self.source.contents[start + 2..end]; + &self.source.contents[content_start..end]; return Ok(()); } _ => {} @@ -807,10 +810,14 @@ impl<'a> Lexer<'a> { let text = &self.source.contents[slice_lo..slice_hi]; let mut array_list = bun_alloc::ArenaVec::with_capacity_in(text.len(), self.bump); + // Diagnostic positions inside decode_escape_sequences are computed as + // base + iter.i, where iter.i is an offset into `text`. Pass `slice_lo` + // (which equals `start` for single-line and `start + 2 + trimmed LF/CRLF` + // for multi-line) so the column points at the real source byte. if is_multiline_string_literal { - self.decode_escape_sequences::(start, text, &mut array_list)?; + self.decode_escape_sequences::(slice_lo, text, &mut array_list)?; } else { - self.decode_escape_sequences::(start, text, &mut array_list)?; + self.decode_escape_sequences::(slice_lo, text, &mut array_list)?; } self.string_literal_slice = array_list.into_bump_slice(); self.string_literal_is_ascii = false; @@ -887,31 +894,31 @@ impl<'a> Lexer<'a> { match c2 { // https://mathiasbynens.be/notes/javascript-escapes#single c if c == 'b' as CodePoint => { - buf.push(8); + buf.push(0x08); continue; } c if c == 'f' as CodePoint => { // Form feed: U+000C - buf.push(12); + buf.push(0x0C); continue; } c if c == 'n' as CodePoint => { - buf.push(10); + buf.push(0x0A); continue; } c if c == 'v' as CodePoint => { // Vertical tab is invalid JSON // We're going to allow it. - buf.push(11); + buf.push(0x0B); continue; } c if c == 't' as CodePoint => { // Horizontal tab: U+0009 - buf.push(9); + buf.push(0x09); continue; } c if c == 'r' as CodePoint => { - buf.push(13); + buf.push(0x0D); continue; } @@ -1132,28 +1139,68 @@ impl<'a> Lexer<'a> { self.add_default_error(b"Unexpected end of line")?; } - // Ignore line continuations. A line continuation is not an escaped newline. - // Match the JS lexer (js_parser/lexer.rs:660-661, 937-939): guard on - // the index we actually read (`iter.i + 1`), not `iter.i`. Without - // this, a multiline basic string ending in `\` right before `"""` - // reads `text[len]` and panics even in release (slice bounds checks - // always run). - let next_i: usize = iter.i as usize + 1; - if next_i < text.len() && text[next_i] == b'\n' { - // Make sure Windows CRLF counts as a single newline - iter.i += 1; + // Line ending backslash: skip \r (and \n if CRLF) plus all subsequent + // whitespace. `iter.i` points at the start of `\r`; the `\n` (if any) + // is at `iter.i + 1` — reading `text[iter.i]` here would always be `\r`. + let mut pos = iter.i as usize + 1; + if pos < text.len() && text[pos] == b'\n' { + pos += 1; + } + // Consume all subsequent whitespace (spaces, tabs, newlines) + while pos < text.len() { + match text[pos] { + b' ' | b'\t' | b'\n' | b'\r' => pos += 1, + _ => break, + } } + iter.i = pos as u32; + // Reset width so the next iterator.next() starts at iter.i + iter.width = 0; continue; } c if c == '\n' as CodePoint || c == 0x2028 || c == 0x2029 => { - // Ignore line continuations. A line continuation is not an escaped newline. + // Line ending backslash: skip newline and all subsequent whitespace if !ALLOW_MULTILINE { self.end = (start + iter.i as usize).saturating_sub(width2 as usize); self.add_default_error(b"Unexpected end of line")?; } + iter.i += iter.width as u32; + // Consume all subsequent whitespace (spaces, tabs, newlines) + while (iter.i as usize) < text.len() { + match text[iter.i as usize] { + b' ' | b'\t' | b'\n' | b'\r' => iter.i += 1, + _ => break, + } + } + // Reset width so the next iterator.next() starts at iter.i + iter.width = 0; continue; } + c if ALLOW_MULTILINE + && (c == ' ' as CodePoint || c == '\t' as CodePoint) => + { + // TOML v1.0.0 ABNF: `mlb-escaped-nl = escape ws newline *( wschar / newline )` + // — a backslash may have trailing spaces/tabs before the newline. + // Peek past them: if the run ends in a newline it's a line-ending + // backslash; otherwise fall through and emit the whitespace char. + let mut peek = iter.i as usize + iter.width as usize; + while peek < text.len() && (text[peek] == b' ' || text[peek] == b'\t') { + peek += 1; + } + if peek < text.len() && (text[peek] == b'\n' || text[peek] == b'\r') { + iter.i = peek as u32; + iter.width = 0; + while (iter.i as usize) < text.len() { + match text[iter.i as usize] { + b' ' | b'\t' | b'\n' | b'\r' => iter.i += 1, + _ => break, + } + } + continue; + } + iter.c = c2; + } _ => { iter.c = c2; } @@ -1282,6 +1329,23 @@ pub(crate) fn is_identifier_part(code_point: CodePoint) -> bool { // The `(0..=127)` bound is required for the byte cast above to be sound. } +/// Per TOML spec: "A newline immediately following the opening delimiter will be trimmed." +/// Applies to both multi-line basic strings (""") and multi-line literal strings ('''). +#[inline] +fn trim_multiline_start(contents: &[u8], content_start: usize) -> usize { + if content_start < contents.len() && contents[content_start] == b'\n' { + return content_start + 1; + } + if content_start < contents.len() + && contents[content_start] == b'\r' + && content_start + 1 < contents.len() + && contents[content_start + 1] == b'\n' + { + return content_start + 2; + } + content_start +} + #[inline] fn float64(num: CodePoint) -> f64 { num as f64 diff --git a/test/js/bun/resolve/toml/toml.test.js b/test/js/bun/resolve/toml/toml.test.js index c244e1be31ef..35470a49d32b 100644 --- a/test/js/bun/resolve/toml/toml.test.js +++ b/test/js/bun/resolve/toml/toml.test.js @@ -112,3 +112,95 @@ it("Bun.TOML.parse throws on deeply nested inline tables instead of crashing", ( "a = " + Buffer.alloc(depth * 6, "{ b = ").toString() + "1" + Buffer.alloc(depth * 2, " }").toString(); expect(() => Bun.TOML.parse(deepToml)).toThrow(RangeError); }); + +// #28680 — TOML multi-line strings must trim the first newline after the +// opening delimiter and line-continuation backslashes must consume all +// subsequent whitespace. +it("TOML multi-line basic strings trim leading newline and handle line ending backslash", () => { + const toml = ` +str1 = "The quick brown fox jumps over the lazy dog." + +str2 = """ +The quick brown \\ + + + fox jumps over \\ + the lazy dog.""" + +str3 = """\\ + The quick brown \\ + fox jumps over \\ + the lazy dog.\\ + """ +`; + + const result = Bun.TOML.parse(toml); + expect(result.str1).toBe("The quick brown fox jumps over the lazy dog."); + expect(result.str2).toBe("The quick brown fox jumps over the lazy dog."); + expect(result.str3).toBe("The quick brown fox jumps over the lazy dog."); +}); + +it("TOML multi-line literal strings trim leading newline", () => { + const toml = ` +str1 = 'The quick brown fox jumps over the lazy dog.' + +str2 = ''' +The quick brown fox jumps over the lazy dog.''' +`; + + const result = Bun.TOML.parse(toml); + expect(result.str1).toBe("The quick brown fox jumps over the lazy dog."); + expect(result.str2).toBe("The quick brown fox jumps over the lazy dog."); +}); + +it("TOML multi-line basic string with only backslash continuation", () => { + const toml = ` +str = """\\ + hello\\ + world\\ + """ +`; + + const result = Bun.TOML.parse(toml); + expect(result.str).toBe("helloworld"); +}); + +// TOML v1.0.0 ABNF: `mlb-escaped-nl = escape ws newline *( wschar / newline )` +// — trailing spaces/tabs between the backslash and the newline are part of +// the line-ending continuation. +it("TOML multi-line basic string allows whitespace between backslash and newline", () => { + const toml = 'str = """hello\\ \n world"""'; + + const result = Bun.TOML.parse(toml); + expect(result.str).toBe("helloworld"); +}); + +it("TOML multi-line strings without leading newline are unchanged", () => { + const toml = ` +str1 = """no leading newline""" +str2 = '''no leading newline''' +`; + + const result = Bun.TOML.parse(toml); + expect(result.str1).toBe("no leading newline"); + expect(result.str2).toBe("no leading newline"); +}); + +it("TOML multi-line strings handle CRLF line endings", () => { + // Template literals normalize to LF, so build CRLF inputs explicitly. + expect(Bun.TOML.parse('str = """\r\nhello"""').str).toBe("hello"); + expect(Bun.TOML.parse("str = '''\r\nhello'''").str).toBe("hello"); + // Line-ending backslash followed by CRLF consumes the pair plus indentation. + expect(Bun.TOML.parse('str = """a\\\r\n b"""').str).toBe("ab"); +}); + +it("TOML escape sequences produce correct character codes", () => { + const toml = ` +tab = "hello\\tworld" +ff = "hello\\fworld" +`; + + const result = Bun.TOML.parse(toml); + expect(result.tab).toBe("hello\tworld"); + expect(result.ff).toBe("hello\fworld"); +});