diff --git a/Cargo.toml b/Cargo.toml index 2d0c04d..52fc71c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "yamd" description = "Yet Another Markdown Document (flavour)" -version = "0.19.0" +version = "0.20.0" edition = "2024" license = "MIT OR Apache-2.0" repository = "https://github.com/Lurk/yamd" diff --git a/readme.md b/readme.md index 9901d47..1e15560 100644 --- a/readme.md +++ b/readme.md @@ -49,8 +49,10 @@ distinction), and escaping is context-independent. ### Escaping -Escaping is handled at the [`lexer`](https://docs.rs/yamd/latest/yamd/lexer/) level: any character following `\` is treated as a -[literal](https://docs.rs/yamd/latest/yamd/lexer/token/enum.TokenKind.html#variant.Literal). +Escaping is recognized at the [`lexer`](https://docs.rs/yamd/latest/yamd/lexer/) level: `\` forces the following character into the +surrounding [literal](https://docs.rs/yamd/latest/yamd/lexer/token/enum.TokenKind.html#variant.Literal) run instead of its usual meaning. The `\` +itself is stripped later, lazily, when the content is resolved to text (e.g. via +[`Content::as_str`](https://docs.rs/yamd/latest/yamd/op/struct.Content.html#method.as_str), or transparently as part of [`deserialize`](https://docs.rs/yamd/latest/yamd/fn.deserialize.html)). Example: diff --git a/src/lexer/mod.rs b/src/lexer/mod.rs index 39ef055..2a5a5e3 100644 --- a/src/lexer/mod.rs +++ b/src/lexer/mod.rs @@ -23,7 +23,7 @@ pub use token::{Position, Token, TokenKind}; pub struct Lexer<'input> { literal_start: Option, len: usize, - escaped: bool, + escaped: u32, position: Position, iter: Peekable>, queue: VecDeque, @@ -38,7 +38,7 @@ impl<'input> Lexer<'input> { len: input.len(), iter: input.char_indices().peekable(), literal_start: None, - escaped: false, + escaped: 0, queue: VecDeque::with_capacity(2), token: None, } @@ -54,7 +54,7 @@ impl<'input> Lexer<'input> { }) { self.queue.push_back(token); } - self.escaped = false; + self.escaped = 0; } } @@ -91,19 +91,17 @@ impl<'input> Lexer<'input> { return false; }; if *next_char == char { - self.next_char(false); + self.next_char(); return true; } false } - fn next_char(&mut self, escaped: bool) -> Option<(Position, char)> { + fn next_char(&mut self) -> Option<(Position, char)> { if let Some((byte_offset, char)) = self.iter.next() { self.position.byte_index = byte_offset; let res = Some((self.position.clone(), char)); - if char != '\\' || escaped { - self.position.column += 1; - } + self.position.column += 1; return res; } None @@ -117,6 +115,13 @@ impl<'input> Lexer<'input> { ) } + fn escape(&mut self, position: Position) { + self.literal_start.get_or_insert(position); + if self.next_char().is_some() { + self.escaped += 1; + } + } + fn take_while(&mut self, c: char, kind: TokenKind, start: Position) { while self.next_is(c) {} self.emit(Token::new( @@ -136,13 +141,7 @@ impl<'input> Lexer<'input> { '%' if self.next_is('}') => { self.emit(self.to_token(TokenKind::CollapsibleEnd, position, 2)) } - '\\' => { - self.emit_literal_if_started(position.byte_index); - if let Some((pos, _)) = self.next_char(true) { - self.escaped = true; - self.literal_start.get_or_insert(pos); - } - } + '\\' => self.escape(position), '~' => self.take_while('~', TokenKind::Tilde, position), '*' => self.take_while('*', TokenKind::Star, position), '}' => self.take_while('}', TokenKind::RightCurlyBrace, position), @@ -168,7 +167,7 @@ impl<'input> Lexer<'input> { fn advance(&mut self) { while self.queue.is_empty() { - if let Some((position, char)) = self.next_char(false) { + if let Some((position, char)) = self.next_char() { self.parse(position, char); } else { self.position.byte_index = self.len; @@ -254,20 +253,20 @@ mod tests { vec![ Token { kind: TokenKind::Literal, - range: 1..2, + range: 0..2, position: Position { - byte_index: 1, + byte_index: 0, column: 0, row: 0 }, - escaped: true + escaped: 1 }, Token::new( TokenKind::LeftSquareBracket, 2..3, Position { byte_index: 2, - column: 1, + column: 2, row: 0 } ) @@ -329,13 +328,13 @@ mod tests { Token::new(TokenKind::Hash, 0..3, Position::default()), Token { kind: TokenKind::Literal, - range: 4..7, + range: 3..7, position: Position { - byte_index: 4, + byte_index: 3, column: 3, row: 0, }, - escaped: true + escaped: 1 }, ] ); @@ -345,28 +344,16 @@ mod tests { fn escaped_space_compression() { assert_eq!( Lexer::new("\\ \\ ").collect::>(), - vec![ - Token { - kind: TokenKind::Literal, - range: 1..2, - position: Position { - byte_index: 1, - column: 0, - row: 0 - }, - escaped: true - }, - Token { - kind: TokenKind::Literal, - range: 3..5, - position: Position { - byte_index: 3, - column: 1, - row: 0 - }, - escaped: true + vec![Token { + kind: TokenKind::Literal, + range: 0..5, + position: Position { + byte_index: 0, + column: 0, + row: 0 }, - ] + escaped: 2 + }] ); } @@ -531,13 +518,13 @@ mod tests { Lexer::new("\\\\").collect::>(), vec![Token { kind: TokenKind::Literal, - range: 1..2, + range: 0..2, position: Position { - byte_index: 1, + byte_index: 0, column: 0, row: 0 }, - escaped: true + escaped: 1 },] ) } @@ -547,23 +534,18 @@ mod tests { assert_eq!( Lexer::new("literal\\[[").collect::>(), vec![ - Token::new(TokenKind::Literal, 0..7, Position::default()), Token { kind: TokenKind::Literal, - range: 8..9, - position: Position { - byte_index: 8, - column: 7, - row: 0, - }, - escaped: true + range: 0..9, + position: Position::default(), + escaped: 1 }, Token::new( TokenKind::LeftSquareBracket, 9..10, Position { byte_index: 9, - column: 8, + column: 9, row: 0, }, ), @@ -733,6 +715,9 @@ mod tests { #[test] fn dangling_backslash_at_eof() { - assert_eq!(Lexer::new("\\").collect::>(), vec![]); + assert_eq!( + Lexer::new("\\").collect::>(), + vec![Token::new(TokenKind::Literal, 0..1, Position::default())] + ); } } diff --git a/src/lexer/token.rs b/src/lexer/token.rs index 099d52a..2e92774 100644 --- a/src/lexer/token.rs +++ b/src/lexer/token.rs @@ -101,18 +101,19 @@ pub struct Token { pub range: Range, /// The position of the token in the input string. pub position: Position, - /// Indicates if the token is escaped. - pub escaped: bool, + /// How many `\`-escapes are folded into this token's range. `0` means the range can be used + /// as-is; otherwise the text needs unescaping (see `Content::as_str`) before use. + pub escaped: u32, } impl Token { - /// Creates a new non escaped `Token` instance. + /// Creates a new `Token` instance with no escapes. pub fn new(kind: TokenKind, range: Range, position: Position) -> Self { Self { kind, range, position, - escaped: false, + escaped: 0, } } } @@ -158,6 +159,6 @@ mod tests { let token = Token::new(TokenKind::Literal, 0..5, Position::default()); assert_eq!(token.kind, TokenKind::Literal); assert_eq!(token.range, 0..5); - assert!(!token.escaped); + assert_eq!(token.escaped, 0); } } diff --git a/src/lib.rs b/src/lib.rs index 366fa40..e101ced 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -41,8 +41,10 @@ //! //! ## Escaping //! -//! Escaping is handled at the [lexer] level: any character following `\` is treated as a -//! [literal](lexer::TokenKind::Literal). +//! Escaping is recognized at the [lexer] level: `\` forces the following character into the +//! surrounding [literal](lexer::TokenKind::Literal) run instead of its usual meaning. The `\` +//! itself is stripped later, lazily, when the content is resolved to text (e.g. via +//! [`Content::as_str`](op::Content::as_str), or transparently as part of [`deserialize`]). //! //! Example: //! diff --git a/src/op/anchor.rs b/src/op/anchor.rs index e3c7db1..01568fe 100644 --- a/src/op/anchor.rs +++ b/src/op/anchor.rs @@ -3,13 +3,13 @@ use crate::op::{Content, Node, Op, Parser, destination::destination, title::titl pub fn anchor(p: &mut Parser) -> bool { let start = p.pos; let snap = p.ops.len(); - p.ops.push(Op::new_start(Node::Anchor, Content::Span(0..0))); + p.ops.push(Op::new_start(Node::Anchor, Content::empty())); if !title(p) || !destination(p) { p.pos = start; p.ops.truncate(snap); return false; } - p.ops.push(Op::new_end(Node::Anchor, Content::Span(0..0))); + p.ops.push(Op::new_end(Node::Anchor, Content::empty())); true } @@ -27,14 +27,14 @@ mod tests { assert_eq!( p.ops, vec![ - Op::new_start(Node::Anchor, Content::Span(0..0)), + Op::new_start(Node::Anchor, Content::empty()), Op::new_start(Node::Title, p.span(0..1)), Op::new_value(p.span(1..2)), Op::new_end(Node::Title, p.span(2..3)), Op::new_start(Node::Destination, p.span(3..4)), Op::new_value(p.span(4..5)), Op::new_end(Node::Destination, p.span(5..6)), - Op::new_end(Node::Anchor, Content::Span(0..0)) + Op::new_end(Node::Anchor, Content::empty()) ] ); } @@ -74,14 +74,14 @@ mod tests { assert_eq!( p.ops, vec![ - Op::new_start(Node::Anchor, Content::Span(0..0)), + Op::new_start(Node::Anchor, Content::empty()), Op::new_start(Node::Title, p.span(0..1)), - Op::new_value(p.span(1..4)), - Op::new_end(Node::Title, p.span(4..5)), - Op::new_start(Node::Destination, p.span(5..6)), - Op::new_value(p.span(6..7)), - Op::new_end(Node::Destination, p.span(7..8)), - Op::new_end(Node::Anchor, Content::Span(0..0)) + Op::new_value(p.span(1..3)), + Op::new_end(Node::Title, p.span(3..4)), + Op::new_start(Node::Destination, p.span(4..5)), + Op::new_value(p.span(5..6)), + Op::new_end(Node::Destination, p.span(6..7)), + Op::new_end(Node::Anchor, Content::empty()) ] ); } @@ -93,14 +93,14 @@ mod tests { assert_eq!( p.ops, vec![ - Op::new_start(Node::Anchor, Content::Span(0..0)), + Op::new_start(Node::Anchor, Content::empty()), Op::new_start(Node::Title, p.span(0..1)), Op::new_value(p.span(1..2)), Op::new_end(Node::Title, p.span(2..3)), Op::new_start(Node::Destination, p.span(3..4)), Op::new_value(p.span(4..8)), Op::new_end(Node::Destination, p.span(8..9)), - Op::new_end(Node::Anchor, Content::Span(0..0)) + Op::new_end(Node::Anchor, Content::empty()) ] ); } @@ -112,14 +112,14 @@ mod tests { assert_eq!( p.ops, vec![ - Op::new_start(Node::Anchor, Content::Span(0..0)), + Op::new_start(Node::Anchor, Content::empty()), Op::new_start(Node::Title, p.span(0..1)), Op::new_value(p.span(1..2)), Op::new_end(Node::Title, p.span(2..3)), Op::new_start(Node::Destination, p.span(3..4)), Op::new_value(p.span(4..6)), Op::new_end(Node::Destination, p.span(6..7)), - Op::new_end(Node::Anchor, Content::Span(0..0)) + Op::new_end(Node::Anchor, Content::empty()) ] ); } diff --git a/src/op/code.rs b/src/op/code.rs index a20e05f..c65dccd 100644 --- a/src/op/code.rs +++ b/src/op/code.rs @@ -11,15 +11,11 @@ fn is_backtick3(t: &Token) -> bool { t.kind == TokenKind::Backtick && t.position.column == 0 && t.range.len() == 3 } -fn is_eol(t: &Token) -> bool { - t.kind == TokenKind::Eol -} - pub fn code(p: &mut Parser) -> bool { let start_pos = p.pos; let snap = p.ops.len(); - let first_range = eat_seq!(p, is_backtick3, is_eol).or_else(|| p.eat(is_backtick3)); + let first_range = eat_seq!(p, is_backtick3, eol).or_else(|| p.eat(is_backtick3)); let Some(first_range) = first_range else { return false; }; @@ -42,7 +38,7 @@ pub fn code(p: &mut Parser) -> bool { return false; }; - let end_range = if let Some(eol_range) = p.eat(is_eol) { + let end_range = if let Some(eol_range) = p.eat(eol) { close_range.start..eol_range.end } else if p.at_block_boundary() { close_range diff --git a/src/op/collapsible.rs b/src/op/collapsible.rs index 310f83d..3d9d51d 100644 --- a/src/op/collapsible.rs +++ b/src/op/collapsible.rs @@ -4,7 +4,7 @@ use crate::{ Node, Op, Parser, document::document, modifier::modifier, - parser::{StopCondition, eat_seq}, + parser::{StopCondition, eat_seq, eol}, }, }; @@ -20,10 +20,6 @@ fn is_space_or_eol(t: &Token) -> bool { t.kind == TokenKind::Space || t.kind == TokenKind::Eol } -fn is_eol(t: &Token) -> bool { - t.kind == TokenKind::Eol -} - pub fn collapsible(p: &mut Parser) -> bool { let start = p.pos; let snap = p.ops.len(); @@ -48,8 +44,8 @@ pub fn collapsible(p: &mut Parser) -> bool { document(p); }); - p.eat(|t: &Token| t.kind == TokenKind::Eol); - let end_range = eat_seq!(p, is_collapsible_end, is_eol).or_else(|| p.eat(is_collapsible_end)); + p.eat(eol); + let end_range = eat_seq!(p, is_collapsible_end, eol).or_else(|| p.eat(is_collapsible_end)); let Some(end_range) = end_range else { p.pos = start; @@ -78,23 +74,23 @@ mod tests { p.ops, vec![ Op::new_start(Node::Collapsible, p.span(0..2)), - Op::new_start(Node::Modifier, Content::Span(0..0)), + Op::new_start(Node::Modifier, Content::empty()), Op::new_value(p.span(2..3)), Op::new_end(Node::Modifier, p.span(3..4)), - Op::new_start(Node::Document, Content::Span(0..0)), + Op::new_start(Node::Document, Content::empty()), Op::new_start(Node::Heading, p.span(4..6)), Op::new_value(p.span(6..7)), - Op::new_end(Node::Heading, Content::Span(0..0)), + Op::new_end(Node::Heading, Content::empty()), Op::new_value(p.span(7..8)), - Op::new_start(Node::Paragraph, Content::Span(0..0)), + Op::new_start(Node::Paragraph, Content::empty()), Op::new_value(p.span(8..9)), - Op::new_end(Node::Paragraph, Content::Span(0..0)), + Op::new_end(Node::Paragraph, Content::empty()), Op::new_value(p.span(9..10)), Op::new_start(Node::Collapsible, p.span(10..12)), - Op::new_start(Node::Modifier, Content::Span(0..0)), + Op::new_start(Node::Modifier, Content::empty()), Op::new_value(p.span(12..13)), Op::new_end(Node::Modifier, p.span(13..14)), - Op::new_start(Node::Document, Content::Span(0..0)), + Op::new_start(Node::Document, Content::empty()), Op::new_start(Node::Image, p.span(14..15)), Op::new_start(Node::Title, p.span(15..16)), Op::new_value(p.span(16..17)), @@ -103,9 +99,9 @@ mod tests { Op::new_value(p.span(19..20)), Op::new_end(Node::Destination, p.span(20..21)), Op::new_end(Node::Image, p.span(21..22)), - Op::new_end(Node::Document, Content::Span(0..0)), + Op::new_end(Node::Document, Content::empty()), Op::new_end(Node::Collapsible, p.span(22..24)), - Op::new_end(Node::Document, Content::Span(0..0)), + Op::new_end(Node::Document, Content::empty()), Op::new_end(Node::Collapsible, p.span(24..25)), ] ); @@ -147,14 +143,14 @@ mod tests { p.ops, vec![ Op::new_start(Node::Collapsible, p.span(0..2)), - Op::new_start(Node::Modifier, Content::Span(0..0)), + Op::new_start(Node::Modifier, Content::empty()), Op::new_value(p.span(2..3)), Op::new_end(Node::Modifier, p.span(3..4)), - Op::new_start(Node::Document, Content::Span(0..0)), + Op::new_start(Node::Document, Content::empty()), Op::new_start(Node::Heading, p.span(4..6)), Op::new_value(p.span(6..7)), - Op::new_end(Node::Heading, Content::Span(0..0)), - Op::new_end(Node::Document, Content::Span(0..0)), + Op::new_end(Node::Heading, Content::empty()), + Op::new_end(Node::Document, Content::empty()), Op::new_end(Node::Collapsible, p.span(8..9)), ] ); @@ -181,17 +177,17 @@ mod tests { assert_eq!( p.ops, vec![ - Op::new_start(Node::Collapsible, p.span(0..2)), // {% - Op::new_start(Node::Modifier, Content::Span(0..0)), // - Op::new_value(p.span(2..3)), // Title - Op::new_end(Node::Modifier, p.span(3..4)), // \n - Op::new_start(Node::Document, Content::Span(0..0)), // - Op::new_start(Node::Embed, p.span(4..5)), // {{ - Op::new_value(p.span(5..6)), // foo - Op::new_value(p.span(6..7)), // | - Op::new_value(p.span(7..8)), // bar - Op::new_end(Node::Embed, p.span(8..10)), // }}\n - Op::new_end(Node::Document, Content::Span(0..0)), + Op::new_start(Node::Collapsible, p.span(0..2)), // {% + Op::new_start(Node::Modifier, Content::empty()), // + Op::new_value(p.span(2..3)), // Title + Op::new_end(Node::Modifier, p.span(3..4)), // \n + Op::new_start(Node::Document, Content::empty()), // + Op::new_start(Node::Embed, p.span(4..5)), // {{ + Op::new_value(p.span(5..6)), // foo + Op::new_value(p.span(6..7)), // | + Op::new_value(p.span(7..8)), // bar + Op::new_end(Node::Embed, p.span(8..10)), // }}\n + Op::new_end(Node::Document, Content::empty()), Op::new_end(Node::Collapsible, p.span(10..11)), // %} ] ); diff --git a/src/op/document.rs b/src/op/document.rs index 20d172a..695e8f2 100644 --- a/src/op/document.rs +++ b/src/op/document.rs @@ -49,8 +49,7 @@ fn is_terminator(t: &Token) -> bool { } pub fn document(p: &mut Parser) { - p.ops - .push(Op::new_start(Node::Document, Content::Span(0..0))); + p.ops.push(Op::new_start(Node::Document, Content::empty())); while !p.at_eof() { let before = p.pos; @@ -72,7 +71,7 @@ pub fn document(p: &mut Parser) { "document loop made no progress at token {before}" ); } - p.ops.push(Op::new_end(Node::Document, Content::Span(0..0))); + p.ops.push(Op::new_end(Node::Document, Content::empty())); } #[cfg(test)] diff --git a/src/op/embed.rs b/src/op/embed.rs index 3e01187..3c3408e 100644 --- a/src/op/embed.rs +++ b/src/op/embed.rs @@ -1,6 +1,6 @@ use crate::{ lexer::{Token, TokenKind}, - op::{Node, Op, Parser}, + op::{Node, Op, Parser, parser::eol}, }; fn is_left_curly2(t: &Token) -> bool { @@ -15,10 +15,6 @@ fn is_right_curly2(t: &Token) -> bool { t.kind == TokenKind::RightCurlyBrace && t.range.len() == 2 } -fn is_eol(t: &Token) -> bool { - t.kind == TokenKind::Eol -} - pub fn embed(p: &mut Parser) -> bool { let start = p.pos; let Some(start_range) = p.eat(is_left_curly2) else { @@ -35,7 +31,7 @@ pub fn embed(p: &mut Parser) -> bool { return false; }; - let end_range = if let Some(eol_range) = p.eat(is_eol) { + let end_range = if let Some(eol_range) = p.eat(eol) { close_range.start..eol_range.end } else if p.at_block_boundary() { close_range diff --git a/src/op/heading.rs b/src/op/heading.rs index ebfc4a7..b3df4b2 100644 --- a/src/op/heading.rs +++ b/src/op/heading.rs @@ -40,7 +40,7 @@ pub fn heading(p: &mut Parser) -> bool { let content = p.span(s..p.pos); p.ops.push(Op::new_value(content)); } - p.ops.push(Op::new_end(Node::Heading, Content::Span(0..0))); + p.ops.push(Op::new_end(Node::Heading, Content::empty())); true } @@ -61,16 +61,16 @@ mod tests { vec![ Op::new_start(Node::Heading, p.span(0..2)), Op::new_value(p.span(2..3)), - Op::new_start(Node::Anchor, Content::Span(0..0)), + Op::new_start(Node::Anchor, Content::empty()), Op::new_start(Node::Title, p.span(3..4)), Op::new_value(p.span(4..5)), Op::new_end(Node::Title, p.span(5..6)), Op::new_start(Node::Destination, p.span(6..7)), Op::new_value(p.span(7..8)), Op::new_end(Node::Destination, p.span(8..9)), - Op::new_end(Node::Anchor, Content::Span(0..0)), + Op::new_end(Node::Anchor, Content::empty()), Op::new_value(p.span(9..11)), - Op::new_end(Node::Heading, Content::Span(0..0)), + Op::new_end(Node::Heading, Content::empty()), ] ); } @@ -83,16 +83,16 @@ mod tests { p.ops, vec![ Op::new_start(Node::Heading, p.span(0..2)), - Op::new_start(Node::Anchor, Content::Span(0..0)), + Op::new_start(Node::Anchor, Content::empty()), Op::new_start(Node::Title, p.span(2..3)), Op::new_value(p.span(3..4)), Op::new_end(Node::Title, p.span(4..5)), Op::new_start(Node::Destination, p.span(5..6)), Op::new_value(p.span(6..7)), Op::new_end(Node::Destination, p.span(7..8)), - Op::new_end(Node::Anchor, Content::Span(0..0)), + Op::new_end(Node::Anchor, Content::empty()), Op::new_value(p.span(8..10)), - Op::new_end(Node::Heading, Content::Span(0..0)), + Op::new_end(Node::Heading, Content::empty()), ] ); } @@ -106,7 +106,7 @@ mod tests { vec![ Op::new_start(Node::Heading, p.span(0..2)), Op::new_value(p.span(2..8)), - Op::new_end(Node::Heading, Content::Span(0..0)), + Op::new_end(Node::Heading, Content::empty()), ] ); } @@ -121,7 +121,7 @@ mod tests { vec![ Op::new_start(Node::Heading, p.span(0..2)), Op::new_value(p.span(2..3)), - Op::new_end(Node::Heading, Content::Span(0..0)), + Op::new_end(Node::Heading, Content::empty()), ] ); }); @@ -146,7 +146,7 @@ mod tests { p.ops, vec![ Op::new_start(Node::Heading, p.span(0..2)), - Op::new_end(Node::Heading, Content::Span(0..0)), + Op::new_end(Node::Heading, Content::empty()), ] ); } @@ -160,16 +160,16 @@ mod tests { vec![ Op::new_start(Node::Heading, p.span(0..2)), Op::new_value(p.span(2..3)), - Op::new_start(Node::Anchor, Content::Span(0..0)), + Op::new_start(Node::Anchor, Content::empty()), Op::new_start(Node::Title, p.span(3..4)), Op::new_value(p.span(4..5)), Op::new_end(Node::Title, p.span(5..6)), Op::new_start(Node::Destination, p.span(6..7)), Op::new_value(p.span(7..8)), Op::new_end(Node::Destination, p.span(8..9)), - Op::new_end(Node::Anchor, Content::Span(0..0)), + Op::new_end(Node::Anchor, Content::empty()), Op::new_value(p.span(9..13)), - Op::new_end(Node::Heading, Content::Span(0..0)), + Op::new_end(Node::Heading, Content::empty()), ] ); } diff --git a/src/op/highlight.rs b/src/op/highlight.rs index 5057568..cbf865c 100644 --- a/src/op/highlight.rs +++ b/src/op/highlight.rs @@ -20,10 +20,6 @@ fn is_space(t: &Token) -> bool { t.kind == TokenKind::Space && t.range.len() == 1 } -fn is_eol(t: &Token) -> bool { - t.kind == TokenKind::Eol -} - fn is_terminator(t: &Token) -> bool { t.kind == TokenKind::Terminator } @@ -35,7 +31,7 @@ fn icon(p: &mut Parser) -> bool { return false; }; - let Some((body_range, end_range)) = p.eat_until(is_eol) else { + let Some((body_range, end_range)) = p.eat_until(eol) else { p.pos = start; p.ops.truncate(snap); return false; @@ -54,7 +50,7 @@ pub fn highlight(p: &mut Parser) -> bool { let start = p.pos; let snap = p.ops.len(); - let Some(start_range) = eat_seq!(p, is_two_bangs, |t: &Token| is_space(t) || is_eol(t)) else { + let Some(start_range) = eat_seq!(p, is_two_bangs, |t: &Token| is_space(t) || eol(t)) else { return false; }; @@ -81,7 +77,7 @@ pub fn highlight(p: &mut Parser) -> bool { let before = p.pos; if let Some(close_range) = p.eat(is_two_bangs) { - let end_range = if let Some(eol_range) = p.eat(is_eol) { + let end_range = if let Some(eol_range) = p.eat(eol) { close_range.start..eol_range.end } else { close_range @@ -126,13 +122,13 @@ mod tests { p.ops, vec![ Op::new_start(Node::Highlight, p.span(0..2)), - Op::new_start(Node::Modifier, Content::Span(0..0)), + Op::new_start(Node::Modifier, Content::empty()), Op::new_value(p.span(2..3)), Op::new_end(Node::Modifier, p.span(3..4)), Op::new_start(Node::Icon, p.span(4..6)), Op::new_value(p.span(6..7)), Op::new_end(Node::Icon, p.span(7..8)), - Op::new_start(Node::Paragraph, Content::Span(0..0)), + Op::new_start(Node::Paragraph, Content::empty()), Op::new_start(Node::Italic, p.span(8..9)), Op::new_value(p.span(9..10)), Op::new_end(Node::Italic, p.span(10..11)), @@ -140,15 +136,15 @@ mod tests { Op::new_start(Node::Bold, p.span(12..13)), Op::new_value(p.span(13..14)), Op::new_end(Node::Bold, p.span(14..15)), - Op::new_end(Node::Paragraph, Content::Span(0..0)), + Op::new_end(Node::Paragraph, Content::empty()), Op::new_value(p.span(15..16)), - Op::new_start(Node::Paragraph, Content::Span(0..0)), + Op::new_start(Node::Paragraph, Content::empty()), Op::new_value(p.span(16..17)), Op::new_start(Node::Strikethrough, p.span(17..18)), Op::new_value(p.span(18..19)), Op::new_end(Node::Strikethrough, p.span(19..20)), Op::new_value(p.span(20..22)), - Op::new_end(Node::Paragraph, Content::Span(0..0)), + Op::new_end(Node::Paragraph, Content::empty()), Op::new_end(Node::Highlight, p.span(22..23)), ] ); @@ -165,7 +161,7 @@ mod tests { Op::new_start(Node::Icon, p.span(2..4)), Op::new_value(p.span(4..5)), Op::new_end(Node::Icon, p.span(5..6)), - Op::new_start(Node::Paragraph, Content::Span(0..0)), + Op::new_start(Node::Paragraph, Content::empty()), Op::new_start(Node::Italic, p.span(6..7)), Op::new_value(p.span(7..8)), Op::new_end(Node::Italic, p.span(8..9)), @@ -173,15 +169,15 @@ mod tests { Op::new_start(Node::Bold, p.span(10..11)), Op::new_value(p.span(11..12)), Op::new_end(Node::Bold, p.span(12..13)), - Op::new_end(Node::Paragraph, Content::Span(0..0)), + Op::new_end(Node::Paragraph, Content::empty()), Op::new_value(p.span(13..14)), - Op::new_start(Node::Paragraph, Content::Span(0..0)), + Op::new_start(Node::Paragraph, Content::empty()), Op::new_value(p.span(14..15)), Op::new_start(Node::Strikethrough, p.span(15..16)), Op::new_value(p.span(16..17)), Op::new_end(Node::Strikethrough, p.span(17..18)), Op::new_value(p.span(18..20)), - Op::new_end(Node::Paragraph, Content::Span(0..0)), + Op::new_end(Node::Paragraph, Content::empty()), Op::new_end(Node::Highlight, p.span(20..21)), ] ) @@ -195,10 +191,10 @@ mod tests { p.ops, vec![ Op::new_start(Node::Highlight, p.span(0..2)), - Op::new_start(Node::Modifier, Content::Span(0..0)), + Op::new_start(Node::Modifier, Content::empty()), Op::new_value(p.span(2..3)), Op::new_end(Node::Modifier, p.span(3..4)), - Op::new_start(Node::Paragraph, Content::Span(0..0)), + Op::new_start(Node::Paragraph, Content::empty()), Op::new_start(Node::Italic, p.span(4..5)), Op::new_value(p.span(5..6)), Op::new_end(Node::Italic, p.span(6..7)), @@ -206,15 +202,15 @@ mod tests { Op::new_start(Node::Bold, p.span(8..9)), Op::new_value(p.span(9..10)), Op::new_end(Node::Bold, p.span(10..11)), - Op::new_end(Node::Paragraph, Content::Span(0..0)), + Op::new_end(Node::Paragraph, Content::empty()), Op::new_value(p.span(11..12)), - Op::new_start(Node::Paragraph, Content::Span(0..0)), + Op::new_start(Node::Paragraph, Content::empty()), Op::new_value(p.span(12..13)), Op::new_start(Node::Strikethrough, p.span(13..14)), Op::new_value(p.span(14..15)), Op::new_end(Node::Strikethrough, p.span(15..16)), Op::new_value(p.span(16..18)), - Op::new_end(Node::Paragraph, Content::Span(0..0)), + Op::new_end(Node::Paragraph, Content::empty()), Op::new_end(Node::Highlight, p.span(18..19)), ] ) @@ -263,11 +259,11 @@ mod tests { p.ops, vec![ Op::new_start(Node::Highlight, p.span(0..2)), - Op::new_start(Node::Paragraph, Content::Span(0..0)), + Op::new_start(Node::Paragraph, Content::empty()), Op::new_value(p.span(2..5)), - Op::new_end(Node::Paragraph, Content::Span(0..0)), + Op::new_end(Node::Paragraph, Content::empty()), Op::new_value(p.span(5..6)), - Op::new_start(Node::Paragraph, Content::Span(0..0)), + Op::new_start(Node::Paragraph, Content::empty()), Op::new_start(Node::Italic, p.span(6..7)), Op::new_value(p.span(7..8)), Op::new_end(Node::Italic, p.span(8..9)), @@ -275,15 +271,15 @@ mod tests { Op::new_start(Node::Bold, p.span(10..11)), Op::new_value(p.span(11..12)), Op::new_end(Node::Bold, p.span(12..13)), - Op::new_end(Node::Paragraph, Content::Span(0..0)), + Op::new_end(Node::Paragraph, Content::empty()), Op::new_value(p.span(13..14)), - Op::new_start(Node::Paragraph, Content::Span(0..0)), + Op::new_start(Node::Paragraph, Content::empty()), Op::new_value(p.span(14..15)), Op::new_start(Node::Strikethrough, p.span(15..16)), Op::new_value(p.span(16..17)), Op::new_end(Node::Strikethrough, p.span(17..18)), Op::new_value(p.span(18..20)), - Op::new_end(Node::Paragraph, Content::Span(0..0)), + Op::new_end(Node::Paragraph, Content::empty()), Op::new_end(Node::Highlight, p.span(20..21)), ] ); @@ -297,12 +293,12 @@ mod tests { p.ops, vec![ Op::new_start(Node::Highlight, p.span(0..2)), - Op::new_start(Node::Modifier, Content::Span(0..0)), + Op::new_start(Node::Modifier, Content::empty()), Op::new_value(p.span(2..3)), Op::new_end(Node::Modifier, p.span(3..4)), - Op::new_start(Node::Paragraph, Content::Span(0..0)), + Op::new_start(Node::Paragraph, Content::empty()), Op::new_value(p.span(4..7)), - Op::new_end(Node::Paragraph, Content::Span(0..0)), + Op::new_end(Node::Paragraph, Content::empty()), Op::new_end(Node::Highlight, p.span(7..8)), ] ); diff --git a/src/op/image.rs b/src/op/image.rs index b2f7a16..b3a570b 100644 --- a/src/op/image.rs +++ b/src/op/image.rs @@ -1,16 +1,12 @@ use crate::{ lexer::{Token, TokenKind}, - op::{Content, Node, Op, Parser, destination::destination, title::title}, + op::{Content, Node, Op, Parser, destination::destination, parser::eol, title::title}, }; fn is_bang(t: &Token) -> bool { t.kind == TokenKind::Bang && t.range.len() == 1 } -fn is_eol(t: &Token) -> bool { - t.kind == TokenKind::Eol -} - pub fn image(p: &mut Parser) -> bool { let start = p.pos; let snap = p.ops.len(); @@ -33,12 +29,12 @@ pub fn image(p: &mut Parser) -> bool { return false; } - if let Some(eol_range) = p.eat(is_eol) { + if let Some(eol_range) = p.eat(eol) { let end_content = p.span(eol_range); p.ops.push(Op::new_end(Node::Image, end_content)); return true; } else if p.at_eof() { - p.ops.push(Op::new_end(Node::Image, Content::Span(0..0))); + p.ops.push(Op::new_end(Node::Image, Content::empty())); return true; } diff --git a/src/op/images.rs b/src/op/images.rs index b7fce06..c6dc656 100644 --- a/src/op/images.rs +++ b/src/op/images.rs @@ -18,8 +18,8 @@ pub fn images(p: &mut Parser) -> bool { if count > 1 { p.ops - .insert(snap, Op::new_start(Node::Images, Content::Span(0..0))); - p.ops.push(Op::new_end(Node::Images, Content::Span(0..0))); + .insert(snap, Op::new_start(Node::Images, Content::empty())); + p.ops.push(Op::new_end(Node::Images, Content::empty())); } count > 0 } @@ -38,7 +38,7 @@ mod tests { assert_eq!( p.ops, vec![ - Op::new_start(Node::Images, Content::Span(0..0)), + Op::new_start(Node::Images, Content::empty()), Op::new_start(Node::Image, p.span(0..1)), Op::new_start(Node::Title, p.span(1..2)), Op::new_value(p.span(2..3)), @@ -54,8 +54,8 @@ mod tests { Op::new_start(Node::Destination, p.span(12..13)), Op::new_value(p.span(13..14)), Op::new_end(Node::Destination, p.span(14..15)), - Op::new_end(Node::Image, Content::Span(0..0)), - Op::new_end(Node::Images, Content::Span(0..0)), + Op::new_end(Node::Image, Content::empty()), + Op::new_end(Node::Images, Content::empty()), ] ); } @@ -97,7 +97,7 @@ mod tests { assert_eq!( p.ops, vec![ - Op::new_start(Node::Images, Content::Span(0..0)), + Op::new_start(Node::Images, Content::empty()), Op::new_start(Node::Image, p.span(0..1)), Op::new_start(Node::Title, p.span(1..2)), Op::new_value(p.span(2..3)), @@ -114,7 +114,7 @@ mod tests { Op::new_value(p.span(13..14)), Op::new_end(Node::Destination, p.span(14..15)), Op::new_end(Node::Image, p.span(15..16)), - Op::new_end(Node::Images, Content::Span(0..0)), + Op::new_end(Node::Images, Content::empty()), ] ); } diff --git a/src/op/list.rs b/src/op/list.rs index c490a64..c3ee2c7 100644 --- a/src/op/list.rs +++ b/src/op/list.rs @@ -47,7 +47,7 @@ fn list_item(p: &mut Parser, level: usize, kind: Option) -> Option Option { let list_start_idx = snap; p.ops.insert( list_start_idx, - Op::new_start(list_kind.node(), Content::Span(0..0)), + Op::new_start(list_kind.node(), Content::empty()), ); while list_item(p, level, Some(list_kind)).is_some() {} - p.ops - .push(Op::new_end(list_kind.node(), Content::Span(0..0))); + p.ops.push(Op::new_end(list_kind.node(), Content::empty())); Some(list_kind) } @@ -84,18 +83,18 @@ mod tests { assert_eq!( p.ops, vec![ - Op::new_start(Node::UnorderedList, Content::Span(0..0)), + Op::new_start(Node::UnorderedList, Content::empty()), Op::new_start(Node::ListItem, p.span(0..2)), - Op::new_start(Node::Paragraph, Content::Span(0..0)), + Op::new_start(Node::Paragraph, Content::empty()), Op::new_value(p.span(2..4)), - Op::new_end(Node::Paragraph, Content::Span(0..0)), - Op::new_end(Node::ListItem, Content::Span(0..0)), + Op::new_end(Node::Paragraph, Content::empty()), + Op::new_end(Node::ListItem, Content::empty()), Op::new_start(Node::ListItem, p.span(4..6)), - Op::new_start(Node::Paragraph, Content::Span(0..0)), + Op::new_start(Node::Paragraph, Content::empty()), Op::new_value(p.span(6..7)), - Op::new_end(Node::Paragraph, Content::Span(0..0)), - Op::new_end(Node::ListItem, Content::Span(0..0)), - Op::new_end(Node::UnorderedList, Content::Span(0..0)), + Op::new_end(Node::Paragraph, Content::empty()), + Op::new_end(Node::ListItem, Content::empty()), + Op::new_end(Node::UnorderedList, Content::empty()), ] ); } @@ -108,18 +107,18 @@ mod tests { assert_eq!( p.ops, vec![ - Op::new_start(Node::OrderedList, Content::Span(0..0)), + Op::new_start(Node::OrderedList, Content::empty()), Op::new_start(Node::ListItem, p.span(0..2)), - Op::new_start(Node::Paragraph, Content::Span(0..0)), + Op::new_start(Node::Paragraph, Content::empty()), Op::new_value(p.span(2..4)), - Op::new_end(Node::Paragraph, Content::Span(0..0)), - Op::new_end(Node::ListItem, Content::Span(0..0)), + Op::new_end(Node::Paragraph, Content::empty()), + Op::new_end(Node::ListItem, Content::empty()), Op::new_start(Node::ListItem, p.span(4..6)), - Op::new_start(Node::Paragraph, Content::Span(0..0)), + Op::new_start(Node::Paragraph, Content::empty()), Op::new_value(p.span(6..7)), - Op::new_end(Node::Paragraph, Content::Span(0..0)), - Op::new_end(Node::ListItem, Content::Span(0..0)), - Op::new_end(Node::OrderedList, Content::Span(0..0)), + Op::new_end(Node::Paragraph, Content::empty()), + Op::new_end(Node::ListItem, Content::empty()), + Op::new_end(Node::OrderedList, Content::empty()), ] ); } @@ -132,20 +131,20 @@ mod tests { assert_eq!( p.ops, vec![ - Op::new_start(Node::OrderedList, Content::Span(0..0)), + Op::new_start(Node::OrderedList, Content::empty()), Op::new_start(Node::ListItem, p.span(0..2)), - Op::new_start(Node::Paragraph, Content::Span(0..0)), + Op::new_start(Node::Paragraph, Content::empty()), Op::new_value(p.span(2..4)), - Op::new_end(Node::Paragraph, Content::Span(0..0)), - Op::new_start(Node::UnorderedList, Content::Span(0..0)), + Op::new_end(Node::Paragraph, Content::empty()), + Op::new_start(Node::UnorderedList, Content::empty()), Op::new_start(Node::ListItem, p.span(4..7)), - Op::new_start(Node::Paragraph, Content::Span(0..0)), + Op::new_start(Node::Paragraph, Content::empty()), Op::new_value(p.span(7..8)), - Op::new_end(Node::Paragraph, Content::Span(0..0)), - Op::new_end(Node::ListItem, Content::Span(0..0)), - Op::new_end(Node::UnorderedList, Content::Span(0..0)), - Op::new_end(Node::ListItem, Content::Span(0..0)), - Op::new_end(Node::OrderedList, Content::Span(0..0)), + Op::new_end(Node::Paragraph, Content::empty()), + Op::new_end(Node::ListItem, Content::empty()), + Op::new_end(Node::UnorderedList, Content::empty()), + Op::new_end(Node::ListItem, Content::empty()), + Op::new_end(Node::OrderedList, Content::empty()), ] ); } @@ -158,20 +157,20 @@ mod tests { assert_eq!( p.ops, vec![ - Op::new_start(Node::UnorderedList, Content::Span(0..0)), + Op::new_start(Node::UnorderedList, Content::empty()), Op::new_start(Node::ListItem, p.span(0..2)), - Op::new_start(Node::Paragraph, Content::Span(0..0)), + Op::new_start(Node::Paragraph, Content::empty()), Op::new_value(p.span(2..4)), - Op::new_end(Node::Paragraph, Content::Span(0..0)), - Op::new_start(Node::UnorderedList, Content::Span(0..0)), + Op::new_end(Node::Paragraph, Content::empty()), + Op::new_start(Node::UnorderedList, Content::empty()), Op::new_start(Node::ListItem, p.span(4..7)), - Op::new_start(Node::Paragraph, Content::Span(0..0)), + Op::new_start(Node::Paragraph, Content::empty()), Op::new_value(p.span(7..8)), - Op::new_end(Node::Paragraph, Content::Span(0..0)), - Op::new_end(Node::ListItem, Content::Span(0..0)), - Op::new_end(Node::UnorderedList, Content::Span(0..0)), - Op::new_end(Node::ListItem, Content::Span(0..0)), - Op::new_end(Node::UnorderedList, Content::Span(0..0)), + Op::new_end(Node::Paragraph, Content::empty()), + Op::new_end(Node::ListItem, Content::empty()), + Op::new_end(Node::UnorderedList, Content::empty()), + Op::new_end(Node::ListItem, Content::empty()), + Op::new_end(Node::UnorderedList, Content::empty()), ] ); } @@ -184,20 +183,20 @@ mod tests { assert_eq!( p.ops, vec![ - Op::new_start(Node::UnorderedList, Content::Span(0..0)), + Op::new_start(Node::UnorderedList, Content::empty()), Op::new_start(Node::ListItem, p.span(0..2)), - Op::new_start(Node::Paragraph, Content::Span(0..0)), + Op::new_start(Node::Paragraph, Content::empty()), Op::new_value(p.span(2..4)), - Op::new_end(Node::Paragraph, Content::Span(0..0)), - Op::new_start(Node::UnorderedList, Content::Span(0..0)), + Op::new_end(Node::Paragraph, Content::empty()), + Op::new_start(Node::UnorderedList, Content::empty()), Op::new_start(Node::ListItem, p.span(4..7)), - Op::new_start(Node::Paragraph, Content::Span(0..0)), + Op::new_start(Node::Paragraph, Content::empty()), Op::new_value(p.span(7..11)), - Op::new_end(Node::Paragraph, Content::Span(0..0)), - Op::new_end(Node::ListItem, Content::Span(0..0)), - Op::new_end(Node::UnorderedList, Content::Span(0..0)), - Op::new_end(Node::ListItem, Content::Span(0..0)), - Op::new_end(Node::UnorderedList, Content::Span(0..0)), + Op::new_end(Node::Paragraph, Content::empty()), + Op::new_end(Node::ListItem, Content::empty()), + Op::new_end(Node::UnorderedList, Content::empty()), + Op::new_end(Node::ListItem, Content::empty()), + Op::new_end(Node::UnorderedList, Content::empty()), ] ); } @@ -210,13 +209,13 @@ mod tests { assert_eq!( p.ops, vec![ - Op::new_start(Node::OrderedList, Content::Span(0..0)), + Op::new_start(Node::OrderedList, Content::empty()), Op::new_start(Node::ListItem, p.span(0..2)), - Op::new_start(Node::Paragraph, Content::Span(0..0)), + Op::new_start(Node::Paragraph, Content::empty()), Op::new_value(p.span(2..7)), - Op::new_end(Node::Paragraph, Content::Span(0..0)), - Op::new_end(Node::ListItem, Content::Span(0..0)), - Op::new_end(Node::OrderedList, Content::Span(0..0)), + Op::new_end(Node::Paragraph, Content::empty()), + Op::new_end(Node::ListItem, Content::empty()), + Op::new_end(Node::OrderedList, Content::empty()), ] ); } @@ -229,13 +228,13 @@ mod tests { assert_eq!( p.ops, vec![ - Op::new_start(Node::UnorderedList, Content::Span(0..0)), + Op::new_start(Node::UnorderedList, Content::empty()), Op::new_start(Node::ListItem, p.span(0..2)), - Op::new_start(Node::Paragraph, Content::Span(0..0)), + Op::new_start(Node::Paragraph, Content::empty()), Op::new_value(p.span(2..7)), - Op::new_end(Node::Paragraph, Content::Span(0..0)), - Op::new_end(Node::ListItem, Content::Span(0..0)), - Op::new_end(Node::UnorderedList, Content::Span(0..0)), + Op::new_end(Node::Paragraph, Content::empty()), + Op::new_end(Node::ListItem, Content::empty()), + Op::new_end(Node::UnorderedList, Content::empty()), ] ); } @@ -250,56 +249,56 @@ mod tests { assert_eq!( p.ops, vec![ - Op::new_start(Node::UnorderedList, Content::Span(0..0)), + Op::new_start(Node::UnorderedList, Content::empty()), Op::new_start(Node::ListItem, p.span(0..2)), - Op::new_start(Node::Paragraph, Content::Span(0..0)), + Op::new_start(Node::Paragraph, Content::empty()), Op::new_value(p.span(2..4)), - Op::new_end(Node::Paragraph, Content::Span(0..0)), - Op::new_start(Node::UnorderedList, Content::Span(0..0)), + Op::new_end(Node::Paragraph, Content::empty()), + Op::new_start(Node::UnorderedList, Content::empty()), Op::new_start(Node::ListItem, p.span(4..7)), - Op::new_start(Node::Paragraph, Content::Span(0..0)), + Op::new_start(Node::Paragraph, Content::empty()), Op::new_value(p.span(7..9)), - Op::new_end(Node::Paragraph, Content::Span(0..0)), - Op::new_start(Node::UnorderedList, Content::Span(0..0)), + Op::new_end(Node::Paragraph, Content::empty()), + Op::new_start(Node::UnorderedList, Content::empty()), Op::new_start(Node::ListItem, p.span(9..12)), - Op::new_start(Node::Paragraph, Content::Span(0..0)), + Op::new_start(Node::Paragraph, Content::empty()), Op::new_value(p.span(12..14)), - Op::new_end(Node::Paragraph, Content::Span(0..0)), - Op::new_end(Node::ListItem, Content::Span(0..0)), - Op::new_end(Node::UnorderedList, Content::Span(0..0)), - Op::new_end(Node::ListItem, Content::Span(0..0)), + Op::new_end(Node::Paragraph, Content::empty()), + Op::new_end(Node::ListItem, Content::empty()), + Op::new_end(Node::UnorderedList, Content::empty()), + Op::new_end(Node::ListItem, Content::empty()), Op::new_start(Node::ListItem, p.span(14..17)), - Op::new_start(Node::Paragraph, Content::Span(0..0)), + Op::new_start(Node::Paragraph, Content::empty()), Op::new_value(p.span(17..19)), - Op::new_end(Node::Paragraph, Content::Span(0..0)), - Op::new_end(Node::ListItem, Content::Span(0..0)), - Op::new_end(Node::UnorderedList, Content::Span(0..0)), - Op::new_end(Node::ListItem, Content::Span(0..0)), + Op::new_end(Node::Paragraph, Content::empty()), + Op::new_end(Node::ListItem, Content::empty()), + Op::new_end(Node::UnorderedList, Content::empty()), + Op::new_end(Node::ListItem, Content::empty()), Op::new_start(Node::ListItem, p.span(19..21)), - Op::new_start(Node::Paragraph, Content::Span(0..0)), + Op::new_start(Node::Paragraph, Content::empty()), Op::new_value(p.span(21..23)), - Op::new_end(Node::Paragraph, Content::Span(0..0)), - Op::new_start(Node::UnorderedList, Content::Span(0..0)), + Op::new_end(Node::Paragraph, Content::empty()), + Op::new_start(Node::UnorderedList, Content::empty()), Op::new_start(Node::ListItem, p.span(23..26)), - Op::new_start(Node::Paragraph, Content::Span(0..0)), + Op::new_start(Node::Paragraph, Content::empty()), Op::new_value(p.span(26..28)), - Op::new_end(Node::Paragraph, Content::Span(0..0)), - Op::new_start(Node::UnorderedList, Content::Span(0..0)), + Op::new_end(Node::Paragraph, Content::empty()), + Op::new_start(Node::UnorderedList, Content::empty()), Op::new_start(Node::ListItem, p.span(28..31)), - Op::new_start(Node::Paragraph, Content::Span(0..0)), + Op::new_start(Node::Paragraph, Content::empty()), Op::new_value(p.span(31..33)), - Op::new_end(Node::Paragraph, Content::Span(0..0)), - Op::new_end(Node::ListItem, Content::Span(0..0)), - Op::new_end(Node::UnorderedList, Content::Span(0..0)), - Op::new_end(Node::ListItem, Content::Span(0..0)), - Op::new_end(Node::UnorderedList, Content::Span(0..0)), - Op::new_end(Node::ListItem, Content::Span(0..0)), + Op::new_end(Node::Paragraph, Content::empty()), + Op::new_end(Node::ListItem, Content::empty()), + Op::new_end(Node::UnorderedList, Content::empty()), + Op::new_end(Node::ListItem, Content::empty()), + Op::new_end(Node::UnorderedList, Content::empty()), + Op::new_end(Node::ListItem, Content::empty()), Op::new_start(Node::ListItem, p.span(33..35)), - Op::new_start(Node::Paragraph, Content::Span(0..0)), + Op::new_start(Node::Paragraph, Content::empty()), Op::new_value(p.span(35..36)), - Op::new_end(Node::Paragraph, Content::Span(0..0)), - Op::new_end(Node::ListItem, Content::Span(0..0)), - Op::new_end(Node::UnorderedList, Content::Span(0..0)), + Op::new_end(Node::Paragraph, Content::empty()), + Op::new_end(Node::ListItem, Content::empty()), + Op::new_end(Node::UnorderedList, Content::empty()), ] ); } @@ -314,56 +313,56 @@ mod tests { assert_eq!( p.ops, vec![ - Op::new_start(Node::OrderedList, Content::Span(0..0)), + Op::new_start(Node::OrderedList, Content::empty()), Op::new_start(Node::ListItem, p.span(0..2)), - Op::new_start(Node::Paragraph, Content::Span(0..0)), + Op::new_start(Node::Paragraph, Content::empty()), Op::new_value(p.span(2..4)), - Op::new_end(Node::Paragraph, Content::Span(0..0)), - Op::new_start(Node::OrderedList, Content::Span(0..0)), + Op::new_end(Node::Paragraph, Content::empty()), + Op::new_start(Node::OrderedList, Content::empty()), Op::new_start(Node::ListItem, p.span(4..7)), - Op::new_start(Node::Paragraph, Content::Span(0..0)), + Op::new_start(Node::Paragraph, Content::empty()), Op::new_value(p.span(7..9)), - Op::new_end(Node::Paragraph, Content::Span(0..0)), - Op::new_start(Node::OrderedList, Content::Span(0..0)), + Op::new_end(Node::Paragraph, Content::empty()), + Op::new_start(Node::OrderedList, Content::empty()), Op::new_start(Node::ListItem, p.span(9..12)), - Op::new_start(Node::Paragraph, Content::Span(0..0)), + Op::new_start(Node::Paragraph, Content::empty()), Op::new_value(p.span(12..14)), - Op::new_end(Node::Paragraph, Content::Span(0..0)), - Op::new_end(Node::ListItem, Content::Span(0..0)), - Op::new_end(Node::OrderedList, Content::Span(0..0)), - Op::new_end(Node::ListItem, Content::Span(0..0)), + Op::new_end(Node::Paragraph, Content::empty()), + Op::new_end(Node::ListItem, Content::empty()), + Op::new_end(Node::OrderedList, Content::empty()), + Op::new_end(Node::ListItem, Content::empty()), Op::new_start(Node::ListItem, p.span(14..17)), - Op::new_start(Node::Paragraph, Content::Span(0..0)), + Op::new_start(Node::Paragraph, Content::empty()), Op::new_value(p.span(17..19)), - Op::new_end(Node::Paragraph, Content::Span(0..0)), - Op::new_end(Node::ListItem, Content::Span(0..0)), - Op::new_end(Node::OrderedList, Content::Span(0..0)), - Op::new_end(Node::ListItem, Content::Span(0..0)), + Op::new_end(Node::Paragraph, Content::empty()), + Op::new_end(Node::ListItem, Content::empty()), + Op::new_end(Node::OrderedList, Content::empty()), + Op::new_end(Node::ListItem, Content::empty()), Op::new_start(Node::ListItem, p.span(19..21)), - Op::new_start(Node::Paragraph, Content::Span(0..0)), + Op::new_start(Node::Paragraph, Content::empty()), Op::new_value(p.span(21..23)), - Op::new_end(Node::Paragraph, Content::Span(0..0)), - Op::new_start(Node::OrderedList, Content::Span(0..0)), + Op::new_end(Node::Paragraph, Content::empty()), + Op::new_start(Node::OrderedList, Content::empty()), Op::new_start(Node::ListItem, p.span(23..26)), - Op::new_start(Node::Paragraph, Content::Span(0..0)), + Op::new_start(Node::Paragraph, Content::empty()), Op::new_value(p.span(26..28)), - Op::new_end(Node::Paragraph, Content::Span(0..0)), - Op::new_start(Node::OrderedList, Content::Span(0..0)), + Op::new_end(Node::Paragraph, Content::empty()), + Op::new_start(Node::OrderedList, Content::empty()), Op::new_start(Node::ListItem, p.span(28..31)), - Op::new_start(Node::Paragraph, Content::Span(0..0)), + Op::new_start(Node::Paragraph, Content::empty()), Op::new_value(p.span(31..33)), - Op::new_end(Node::Paragraph, Content::Span(0..0)), - Op::new_end(Node::ListItem, Content::Span(0..0)), - Op::new_end(Node::OrderedList, Content::Span(0..0)), - Op::new_end(Node::ListItem, Content::Span(0..0)), - Op::new_end(Node::OrderedList, Content::Span(0..0)), - Op::new_end(Node::ListItem, Content::Span(0..0)), + Op::new_end(Node::Paragraph, Content::empty()), + Op::new_end(Node::ListItem, Content::empty()), + Op::new_end(Node::OrderedList, Content::empty()), + Op::new_end(Node::ListItem, Content::empty()), + Op::new_end(Node::OrderedList, Content::empty()), + Op::new_end(Node::ListItem, Content::empty()), Op::new_start(Node::ListItem, p.span(33..35)), - Op::new_start(Node::Paragraph, Content::Span(0..0)), + Op::new_start(Node::Paragraph, Content::empty()), Op::new_value(p.span(35..36)), - Op::new_end(Node::Paragraph, Content::Span(0..0)), - Op::new_end(Node::ListItem, Content::Span(0..0)), - Op::new_end(Node::OrderedList, Content::Span(0..0)), + Op::new_end(Node::Paragraph, Content::empty()), + Op::new_end(Node::ListItem, Content::empty()), + Op::new_end(Node::OrderedList, Content::empty()), ] ); } @@ -376,12 +375,12 @@ mod tests { assert_eq!( p.ops, vec![ - Op::new_start(Node::UnorderedList, Content::Span(0..0)), + Op::new_start(Node::UnorderedList, Content::empty()), Op::new_start(Node::ListItem, p.span(0..2)), - Op::new_start(Node::Paragraph, Content::Span(0..0)), - Op::new_end(Node::Paragraph, Content::Span(0..0)), - Op::new_end(Node::ListItem, Content::Span(0..0)), - Op::new_end(Node::UnorderedList, Content::Span(0..0)), + Op::new_start(Node::Paragraph, Content::empty()), + Op::new_end(Node::Paragraph, Content::empty()), + Op::new_end(Node::ListItem, Content::empty()), + Op::new_end(Node::UnorderedList, Content::empty()), ] ); } @@ -394,19 +393,19 @@ mod tests { assert_eq!( p.ops, vec![ - Op::new_start(Node::OrderedList, Content::Span(0..0)), + Op::new_start(Node::OrderedList, Content::empty()), Op::new_start(Node::ListItem, p.span(0..2)), - Op::new_start(Node::Paragraph, Content::Span(0..0)), + Op::new_start(Node::Paragraph, Content::empty()), Op::new_value(p.span(2..4)), - Op::new_end(Node::Paragraph, Content::Span(0..0)), - Op::new_start(Node::OrderedList, Content::Span(0..0)), + Op::new_end(Node::Paragraph, Content::empty()), + Op::new_start(Node::OrderedList, Content::empty()), Op::new_start(Node::ListItem, p.span(4..7)), - Op::new_start(Node::Paragraph, Content::Span(0..0)), - Op::new_end(Node::Paragraph, Content::Span(0..0)), - Op::new_end(Node::ListItem, Content::Span(0..0)), - Op::new_end(Node::OrderedList, Content::Span(0..0)), - Op::new_end(Node::ListItem, Content::Span(0..0)), - Op::new_end(Node::OrderedList, Content::Span(0..0)), + Op::new_start(Node::Paragraph, Content::empty()), + Op::new_end(Node::Paragraph, Content::empty()), + Op::new_end(Node::ListItem, Content::empty()), + Op::new_end(Node::OrderedList, Content::empty()), + Op::new_end(Node::ListItem, Content::empty()), + Op::new_end(Node::OrderedList, Content::empty()), ] ); } @@ -419,19 +418,19 @@ mod tests { assert_eq!( p.ops, vec![ - Op::new_start(Node::OrderedList, Content::Span(0..0)), + Op::new_start(Node::OrderedList, Content::empty()), Op::new_start(Node::ListItem, p.span(0..2)), - Op::new_start(Node::Paragraph, Content::Span(0..0)), + Op::new_start(Node::Paragraph, Content::empty()), Op::new_value(p.span(2..4)), - Op::new_end(Node::Paragraph, Content::Span(0..0)), - Op::new_start(Node::UnorderedList, Content::Span(0..0)), + Op::new_end(Node::Paragraph, Content::empty()), + Op::new_start(Node::UnorderedList, Content::empty()), Op::new_start(Node::ListItem, p.span(4..7)), - Op::new_start(Node::Paragraph, Content::Span(0..0)), - Op::new_end(Node::Paragraph, Content::Span(0..0)), - Op::new_end(Node::ListItem, Content::Span(0..0)), - Op::new_end(Node::UnorderedList, Content::Span(0..0)), - Op::new_end(Node::ListItem, Content::Span(0..0)), - Op::new_end(Node::OrderedList, Content::Span(0..0)), + Op::new_start(Node::Paragraph, Content::empty()), + Op::new_end(Node::Paragraph, Content::empty()), + Op::new_end(Node::ListItem, Content::empty()), + Op::new_end(Node::UnorderedList, Content::empty()), + Op::new_end(Node::ListItem, Content::empty()), + Op::new_end(Node::OrderedList, Content::empty()), ] ); } diff --git a/src/op/mod.rs b/src/op/mod.rs index 0df4eb5..a1985c3 100644 --- a/src/op/mod.rs +++ b/src/op/mod.rs @@ -5,13 +5,12 @@ //! - [`OpKind::Start`]`(`[`Node`]`)` / [`OpKind::End`]`(`[`Node`]`)` bracket a node; everything //! between belongs to it. [`OpKind::Value`] is leaf content belonging to the innermost open //! node. -//! - [`Content`] borrows from the source ([`Content::Span`]) when possible, and only allocates -//! ([`Content::Materialized`]) when text was assembled from non-contiguous tokens (e.g. after -//! escape processing). +//! - [`Content`] is a byte range into the source plus a count of the `\`-escapes it contains. //! //! [`to_yamd`] consumes an event stream and promotes it to the [`Yamd`](crate::nodes::Yamd) tree //! form used by [`deserialize`](crate::deserialize). +use std::borrow::Cow; use std::ops::Range; #[cfg(feature = "serde")] @@ -45,92 +44,86 @@ mod title; mod to_yamd; pub use to_yamd::{UnbalancedOpStream, to_yamd, try_to_yamd}; -/// Text content extracted from the source input. -/// -/// `Content` provides zero-copy access to source text when possible. Use [`Span`](Content::Span) -/// when the text maps to a contiguous byte range in the source. Use -/// [`Materialized`](Content::Materialized) when the text was assembled from non-contiguous tokens -/// (e.g., after escape processing removed backslashes). +/// Text content extracted from the source input: a byte range plus how many `\`-escapes it +/// contains. #[derive(Debug, PartialEq)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] -#[cfg_attr(feature = "serde", serde(tag = "type", content = "value"))] -pub enum Content { - /// A contiguous byte range in the source string. Avoids allocation by referencing the original input directly. - Span(Range), - /// An owned string for text assembled from non-contiguous tokens. - Materialized(String), +pub struct Content { + span: Range, + escaped: usize, } impl Content { - /// Returns the text this content represents, borrowing from `source` for [`Span`](Content::Span) variants. - pub fn as_str<'a>(&'a self, source: &'a str) -> &'a str { - match self { - Content::Span(range) => { - if range.is_empty() { - "" - } else { - &source[range.clone()] - } - } - Content::Materialized(s) => s.as_str(), + /// General constructor + pub(crate) fn new(span: Range, escaped: usize) -> Self { + Self { span, escaped } + } + + /// Constructor for an unescaped span (the common case). + pub fn span(range: Range) -> Self { + Content::new(range, 0) + } + + /// Constructor for empty case + pub fn empty() -> Self { + Content::new(0..0, 0) + } + + /// Returns the text this content represents. Borrows directly from `source` when there's + /// nothing to unescape; allocates and strips `\` otherwise. + pub fn as_str<'a>(&'a self, source: &'a str) -> Cow<'a, str> { + let raw = if self.span.is_empty() { + "" + } else { + &source[self.span.clone()] + }; + if self.escaped == 0 { + Cow::Borrowed(raw) + } else { + Cow::Owned(unescape(raw, self.escaped)) } } /// Returns an owned copy of the text this content represents. pub fn to_string(&self, source: &str) -> String { - self.as_str(source).to_owned() + self.as_str(source).into_owned() } /// Returns `true` if this content represents an empty string. pub fn is_empty(&self) -> bool { - match self { - Content::Span(range) => range.is_empty(), - Content::Materialized(s) => s.is_empty(), - } + self.span.is_empty() } - /// Builds `Content` from a token slice. Produces a [`Span`](Content::Span) when tokens are contiguous in `source`, - /// or [`Materialized`](Content::Materialized) when gaps exist (e.g., escape characters were removed). - pub fn from_tokens(tokens: &[Token], source: &str) -> Self { + /// Builds `Content` from a token slice. + pub fn from_tokens(tokens: &[Token]) -> Self { if tokens.is_empty() { - return Content::Span(0..0); - } - let is_contiguous = tokens - .windows(2) - .all(|w| w[0].range.end == w[1].range.start); - if is_contiguous { - let start = tokens.first().unwrap().range.start; - let end = tokens.last().unwrap().range.end; - Content::Span(start..end) - } else { - let s: String = tokens.iter().map(|t| &source[t.range.clone()]).collect(); - Content::Materialized(s) - } - } -} - -impl From<&[Token]> for Content { - fn from(tokens: &[Token]) -> Self { - if tokens.is_empty() { - Content::Span(0..0) - } else { - let start = tokens.first().unwrap().range.start; - let end = tokens.last().unwrap().range.end; - Content::Span(start..end) + return Content::empty(); } + let start = tokens.first().unwrap().range.start; + let end = tokens.last().unwrap().range.end; + let escaped = tokens.iter().map(|t| t.escaped as usize).sum(); + Content::new(start..end, escaped) } } -impl From<&[Token; N]> for Content { - fn from(tokens: &[Token; N]) -> Self { - Content::from(tokens.as_slice()) - } -} - -impl From> for Content { - fn from(tokens: Vec) -> Self { - Content::from(tokens.as_slice()) +fn unescape(raw: &str, escaped_count: usize) -> String { + let mut out = String::with_capacity(raw.len().saturating_sub(escaped_count)); + let mut rest = raw; + for _ in 0..escaped_count { + let Some(idx) = rest.find('\\') else { + break; + }; + out.push_str(&rest[..idx]); + rest = &rest[idx + 1..]; + let Some(c) = rest.chars().next() else { + out.push('\\'); + return out; + }; + out.push(c); + rest = &rest[c.len_utf8()..]; } + out.push_str(rest); + out } /// Identifies which AST node type an [`Op`] refers to. @@ -200,26 +193,26 @@ pub struct Op { impl Op { /// Creates a [`Value`](OpKind::Value) operation with the given content. - pub fn new_value>(tokens: T) -> Self { + pub fn new_value(content: Content) -> Self { Self { kind: OpKind::Value, - content: tokens.into(), + content, } } /// Creates a [`Start`](OpKind::Start) operation for the given node type. - pub fn new_start>(node: Node, tokens: T) -> Self { + pub fn new_start(node: Node, content: Content) -> Self { Self { kind: OpKind::Start(node), - content: tokens.into(), + content, } } /// Creates an [`End`](OpKind::End) operation for the given node type. - pub fn new_end>(node: Node, tokens: T) -> Self { + pub fn new_end(node: Node, content: Content) -> Self { Self { kind: OpKind::End(node), - content: tokens.into(), + content, } } } @@ -313,31 +306,20 @@ end let ops = parse(TEST_CASE); let mut covered = vec![false; TEST_CASE.len()]; - let esc_start = TEST_CASE - .find("\\*escaped\\*") - .expect("escape marker must be present"); - let esc_end = esc_start + "\\*escaped\\*".len(); - covered[esc_start..esc_end].fill(true); - for op in &ops { - match &op.content { - Content::Span(range) => { - for i in range.clone() { - assert!( - !covered[i], - "byte {i} covered by multiple ops (char: {:?})", - &TEST_CASE[i..i + 1] - ); - covered[i] = true; - } - } - Content::Materialized(_) => {} + for i in op.content.span.clone() { + assert!( + !covered[i], + "byte {i} covered by multiple ops (char: {:?})", + &TEST_CASE[i..i + 1] + ); + covered[i] = true; } } let uncovered: Vec = covered .iter() .enumerate() - .filter(|&(_, b)| !b) + .filter(|&(_, is_covered)| !is_covered) .map(|(i, _)| i) .collect(); assert!( @@ -446,118 +428,130 @@ end #[test] fn content_span_as_str() { let source = "hello world"; - let content = Content::Span(0..5); + let content = Content::span(0..5); assert_eq!(content.as_str(source), "hello"); } #[test] - fn content_materialized_as_str() { - let content = Content::Materialized(String::from("hello")); - assert_eq!(content.as_str("ignored source"), "hello"); + fn content_escaped_as_str() { + let source = "a\\!b"; + let content = Content::new(0..source.len(), 1); + assert_eq!(content.as_str(source), "a!b"); } #[test] fn content_span_to_string() { let source = "hello world"; - let content = Content::Span(0..5); + let content = Content::span(0..5); assert_eq!(content.to_string(source), "hello"); } #[test] - fn content_from_non_contiguous_tokens_materializes() { - let source = "a\\!b"; + fn content_from_tokens_sums_escaped_across_slice() { + let source = "a\\!b c\\!d"; let tokens = vec![ - Token::new(TokenKind::Literal, 0..1, Position::default()), Token { kind: TokenKind::Literal, - range: 2..3, - position: Position { - byte_index: 2, - column: 1, - row: 0, - }, - escaped: true, + range: 0..4, + position: Position::default(), + escaped: 1, }, Token::new( - TokenKind::Literal, - 3..4, + TokenKind::Space, + 4..5, Position { - byte_index: 3, - column: 2, + byte_index: 4, + column: 4, row: 0, }, ), + Token { + kind: TokenKind::Literal, + range: 5..9, + position: Position { + byte_index: 5, + column: 5, + row: 0, + }, + escaped: 1, + }, ]; - let content = Content::from_tokens(&tokens, source); - assert_eq!(content, Content::Materialized(String::from("a!b"))); + let content = Content::from_tokens(&tokens); + assert_eq!(content, Content::new(0..9, 2)); + assert_eq!(content.as_str(source), "a!b c!d"); } #[test] - fn content_from_contiguous_tokens_stays_span() { - let source = "hello"; - let tokens = vec![Token::new(TokenKind::Literal, 0..5, Position::default())]; - let content = Content::from_tokens(&tokens, source); - assert_eq!(content, Content::Span(0..5)); + fn unescape_no_escapes() { + assert_eq!(unescape("hello", 0), "hello"); } #[test] - fn content_empty_span_as_str() { - let content = Content::Span(0..0); - assert_eq!(content.as_str("anything"), ""); + fn unescape_single_escape() { + assert_eq!(unescape("a\\!b", 1), "a!b"); } #[test] - fn content_is_empty() { - assert!(Content::Span(0..0).is_empty()); - assert!(!Content::Span(0..5).is_empty()); - assert!(Content::Materialized(String::new()).is_empty()); - assert!(!Content::Materialized(String::from("hi")).is_empty()); + fn unescape_double_backslash_is_one() { + assert_eq!(unescape("\\\\", 1), "\\"); } #[test] - fn content_from_empty_tokens() { - let content = Content::from_tokens(&[], "source"); - assert_eq!(content, Content::Span(0..0)); + fn unescape_multibyte_escaped_char() { + assert_eq!(unescape("\\ツ", 1), "ツ"); } #[test] - fn content_from_token_slice() { - let tokens = vec![Token::new(TokenKind::Literal, 0..5, Position::default())]; - let content = Content::from(tokens.as_slice()); - assert_eq!(content, Content::Span(0..5)); + fn unescape_dangling_backslash_after_budget_spent_is_kept() { + assert_eq!(unescape("a\\!\\", 1), "a!\\"); } #[test] - fn content_from_empty_token_slice() { - let empty: &[Token] = &[]; - let content = Content::from(empty); - assert_eq!(content, Content::Span(0..0)); + fn unescape_count_too_high_with_no_backslash_left_stops_gracefully() { + assert_eq!(unescape("ab", 2), "ab"); } #[test] - fn content_from_token_array() { - let tokens = [Token::new(TokenKind::Literal, 0..3, Position::default())]; - let content = Content::from(&tokens); - assert_eq!(content, Content::Span(0..3)); + fn unescape_count_too_high_with_dangling_backslash_stops_gracefully() { + assert_eq!(unescape("a\\", 2), "a\\"); } #[test] - fn content_from_token_vec() { - let tokens = vec![Token::new(TokenKind::Literal, 0..5, Position::default())]; - let content = Content::from(tokens); - assert_eq!(content, Content::Span(0..5)); + fn unescape_count_too_low_leaves_remainder_untouched() { + assert_eq!(unescape("a\\!b\\!c", 1), "a!b\\!c"); + } + + #[test] + fn content_empty_span_as_str() { + let content = Content::empty(); + assert_eq!(content.as_str("anything"), ""); + } + + #[test] + fn content_is_empty() { + assert!(Content::empty().is_empty()); + assert!(!Content::span(0..5).is_empty()); + } + + #[test] + fn content_from_empty_tokens() { + let content = Content::from_tokens(&[]); + assert_eq!(content, Content::empty()); } #[test] fn escape() { + let source = "¯\\\\\\_(ツ)\\_/¯"; + let ops = parse(source); + assert_eq!(ops[2].content.as_str(source), "¯\\_(ツ)_/¯"); assert_eq!( - parse("¯\\\\\\_(ツ)\\_/¯"), + ops, vec![ - Op::new_start(Node::Document, Content::Span(0..0)), - Op::new_start(Node::Paragraph, Content::Span(0..0)), - Op::new_value(Content::Materialized(String::from("¯\\_(ツ)_/¯"))), - Op::new_end(Node::Paragraph, Content::Span(0..0)), - Op::new_end(Node::Document, Content::Span(0..0)) + Op::new_start(Node::Document, Content::empty()), + Op::new_start(Node::Paragraph, Content::empty()), + Op::new_value(Content::new(0..16, 3)), + Op::new_end(Node::Paragraph, Content::empty()), + Op::new_end(Node::Document, Content::empty()) ] ); } diff --git a/src/op/modifier.rs b/src/op/modifier.rs index 34a1951..964e665 100644 --- a/src/op/modifier.rs +++ b/src/op/modifier.rs @@ -18,8 +18,7 @@ pub fn modifier(p: &mut Parser) -> bool { let body_content = p.span(body_range); let end_content = p.span(end_range); - p.ops - .push(Op::new_start(Node::Modifier, Content::Span(0..0))); + p.ops.push(Op::new_start(Node::Modifier, Content::empty())); p.ops.push(Op::new_value(body_content)); p.ops.push(Op::new_end(Node::Modifier, end_content)); true diff --git a/src/op/paragraph.rs b/src/op/paragraph.rs index 690138b..5f34b19 100644 --- a/src/op/paragraph.rs +++ b/src/op/paragraph.rs @@ -4,8 +4,7 @@ use crate::op::{ }; pub fn paragraph(p: &mut Parser) { - p.ops - .push(Op::new_start(Node::Paragraph, Content::Span(0..0))); + p.ops.push(Op::new_start(Node::Paragraph, Content::empty())); let mut text_start: Option = None; while !p.at_eof() { let pos = p.pos; @@ -27,8 +26,7 @@ pub fn paragraph(p: &mut Parser) { let content = p.span(start..p.pos); p.ops.push(Op::new_value(content)); } - p.ops - .push(Op::new_end(Node::Paragraph, Content::Span(0..0))); + p.ops.push(Op::new_end(Node::Paragraph, Content::empty())); } #[cfg(test)] @@ -73,9 +71,9 @@ mod tests { assert_eq!( p.ops, vec![ - Op::new_start(Node::Paragraph, Content::Span(0..0)), - Op::new_value(Content::Span(0..input.len())), - Op::new_end(Node::Paragraph, Content::Span(0..0)), + Op::new_start(Node::Paragraph, Content::empty()), + Op::new_value(Content::span(0..input.len())), + Op::new_end(Node::Paragraph, Content::empty()), ] ); } diff --git a/src/op/parser.rs b/src/op/parser.rs index 3987ae1..fde5157 100644 --- a/src/op/parser.rs +++ b/src/op/parser.rs @@ -129,18 +129,16 @@ impl StopCondition { /// Node-specific parsing functions (e.g., `heading`, `paragraph`) receive `&mut Parser`, use /// [`eat`](Parser::eat)/[`at`](Parser::at) to match tokens, and push results to [`ops`](Parser::ops). /// On mismatch they restore [`pos`](Parser::pos) and truncate `ops` to backtrack. -pub(crate) struct Parser<'a> { - pub(crate) source: &'a str, +pub(crate) struct Parser { tokens: Vec, pub(crate) pos: usize, eof_stack: Vec, pub(crate) ops: Vec, } -impl<'a> From<&'a str> for Parser<'a> { - fn from(input: &'a str) -> Self { +impl From<&str> for Parser { + fn from(input: &str) -> Self { Self { - source: input, tokens: Lexer::new(input).collect(), pos: 0, eof_stack: Vec::new(), @@ -149,7 +147,7 @@ impl<'a> From<&'a str> for Parser<'a> { } } -impl Parser<'_> { +impl Parser { /// Returns the token at `index`, or `None` if out of bounds. #[inline] pub(crate) fn get(&self, index: usize) -> Option<&Token> { @@ -274,22 +272,10 @@ impl Parser<'_> { None } - /// Converts a token index range into [`Content`] using the tokens' byte ranges. - /// Returns [`Content::Materialized`] when any token in the range is escaped (has gaps from - /// removed backslashes), otherwise returns [`Content::Span`]. + /// Converts a token index range into [`Content`] #[inline] pub(crate) fn span(&self, range: Range) -> Content { - if range.is_empty() { - return Content::Span(0..0); - } - let tokens = &self.tokens[range]; - if tokens.iter().any(|t| t.escaped) { - Content::from_tokens(tokens, self.source) - } else { - let byte_start = tokens.first().unwrap().range.start; - let byte_end = tokens.last().unwrap().range.end; - Content::Span(byte_start..byte_end) - } + Content::from_tokens(&self.tokens[range]) } /// Consumes the parser and returns the accumulated operations. @@ -414,7 +400,7 @@ mod tests { #[test] fn eat_until_finds_match() { let mut p = Parser::from("hello\nworld"); - let result = p.eat_until(|t: &Token| t.kind == TokenKind::Eol); + let result = p.eat_until(eol); assert!(result.is_some()); let (before, matched) = result.unwrap(); assert_eq!(before, 0..1); diff --git a/src/op/thematic_break.rs b/src/op/thematic_break.rs index 04db1b8..874a97a 100644 --- a/src/op/thematic_break.rs +++ b/src/op/thematic_break.rs @@ -12,9 +12,9 @@ pub fn thematic_break(p: &mut Parser) -> bool { return false; }; let content = p.span(range); - let empty = Content::Span(0..0); + let empty = Content::empty(); p.ops - .push(Op::new_start(Node::ThematicBreak, Content::Span(0..0))); + .push(Op::new_start(Node::ThematicBreak, Content::empty())); p.ops.push(Op::new_value(content)); p.ops.push(Op::new_end(Node::ThematicBreak, empty)); true @@ -31,9 +31,9 @@ mod tests { assert_eq!( p.ops, vec![ - Op::new_start(Node::ThematicBreak, Content::Span(0..0)), + Op::new_start(Node::ThematicBreak, Content::empty()), Op::new_value(p.span(0..1)), - Op::new_end(Node::ThematicBreak, Content::Span(0..0)), + Op::new_end(Node::ThematicBreak, Content::empty()), ] ); } diff --git a/src/op/to_yamd.rs b/src/op/to_yamd.rs index 4cc0629..ec1301f 100644 --- a/src/op/to_yamd.rs +++ b/src/op/to_yamd.rs @@ -485,7 +485,7 @@ mod tests { use pretty_assertions::assert_eq; use crate::nodes::*; - use crate::op::{Node, Op, UnbalancedOpStream, parse, to_yamd, try_to_yamd}; + use crate::op::{Content, Node, Op, UnbalancedOpStream, parse, to_yamd, try_to_yamd}; #[test] fn single_paragraph() { @@ -1056,7 +1056,7 @@ end"#; #[test] fn unclosed_frame_errors() { - let ops = vec![Op::new_start(Node::Heading, &[] as &[crate::lexer::Token])]; + let ops = vec![Op::new_start(Node::Heading, Content::empty())]; assert_eq!(try_to_yamd(&ops, ""), Err(UnbalancedOpStream)); } } diff --git a/tests/round_trip.rs b/tests/round_trip.rs index e1a8039..89468b7 100644 --- a/tests/round_trip.rs +++ b/tests/round_trip.rs @@ -52,6 +52,39 @@ fn paragraph_text_with_double_backslash() { round_trip_inline(ParagraphNodes::from("text \\\\ more".to_string())); } +#[test] +fn escaped_heading_marker_stays_paragraph() { + assert_eq!( + Yamd::new( + None, + vec![Paragraph::new(vec![ParagraphNodes::from("# not heading".to_string())]).into()], + ), + deserialize("\\# not heading") + ); +} + +#[test] +fn escaped_list_marker_stays_paragraph() { + assert_eq!( + Yamd::new( + None, + vec![Paragraph::new(vec![ParagraphNodes::from("- not a list".to_string())]).into()], + ), + deserialize("\\- not a list") + ); +} + +#[test] +fn escaped_thematic_break_stays_paragraph() { + assert_eq!( + Yamd::new( + None, + vec![Paragraph::new(vec![ParagraphNodes::from("---".to_string())]).into()], + ), + deserialize("\\---") + ); +} + #[test] fn paragraph_text_with_special_chars() { round_trip_inline(ParagraphNodes::from( @@ -121,6 +154,33 @@ fn backslash_at_end_of_text() { round_trip_inline(ParagraphNodes::from("text ending with \\".to_string())); } +#[test] +fn dangling_backslash_at_eof_is_kept_literal() { + assert_eq!( + Yamd::new( + None, + vec![Paragraph::new(vec![ParagraphNodes::from("\\".to_string())]).into()], + ), + deserialize("\\") + ); +} + +#[test] +fn dangling_backslash_at_eof_after_text_is_kept_literal() { + assert_eq!( + Yamd::new( + None, + vec![Paragraph::new(vec![ParagraphNodes::from("plain text\\".to_string())]).into()], + ), + deserialize("plain text\\") + ); +} + +#[test] +fn dangling_backslash_at_eof_round_trips() { + round_trip_inline(ParagraphNodes::from("plain text\\".to_string())); +} + #[test] fn consecutive_special_chars() { round_trip_inline(ParagraphNodes::from("***___~~~```###!!!".to_string()));