Move escape from omit and flag to keep and count

Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "yamd"
 description = "Yet Another Markdown Document (flavour)"
-version = "0.19.0"
+version = "0.20.0"
 edition = "2024"
 license = "MIT OR Apache-2.0"
 repository = "https://github.com/Lurk/yamd"

readme.md

@@ -49,8 +49,10 @@ distinction), and escaping is context-independent.
 
 ### Escaping
 
-Escaping is handled at the [`lexer`](https://docs.rs/yamd/latest/yamd/lexer/) level: any character following `\` is treated as a
-[literal](https://docs.rs/yamd/latest/yamd/lexer/token/enum.TokenKind.html#variant.Literal).
+Escaping is recognized at the [`lexer`](https://docs.rs/yamd/latest/yamd/lexer/) level: `\` forces the following character into the
+surrounding [literal](https://docs.rs/yamd/latest/yamd/lexer/token/enum.TokenKind.html#variant.Literal) run instead of its usual meaning. The `\`
+itself is stripped later, lazily, when the content is resolved to text (e.g. via
+[`Content::as_str`](https://docs.rs/yamd/latest/yamd/op/struct.Content.html#method.as_str), or transparently as part of [`deserialize`](https://docs.rs/yamd/latest/yamd/fn.deserialize.html)).
 
 Example:
 

src/lexer/mod.rs

@@ -23,7 +23,7 @@ pub use token::{Position, Token, TokenKind};
 pub struct Lexer<'input> {
     literal_start: Option<Position>,
     len: usize,
-    escaped: bool,
+    escaped: u32,
     position: Position,
     iter: Peekable<CharIndices<'input>>,
     queue: VecDeque<Token>,
@@ -38,7 +38,7 @@ impl<'input> Lexer<'input> {
             len: input.len(),
             iter: input.char_indices().peekable(),
             literal_start: None,
-            escaped: false,
+            escaped: 0,
             queue: VecDeque::with_capacity(2),
             token: None,
         }
@@ -54,7 +54,7 @@ impl<'input> Lexer<'input> {
             }) {
                 self.queue.push_back(token);
             }
-            self.escaped = false;
+            self.escaped = 0;
         }
     }
 
@@ -91,19 +91,17 @@ impl<'input> Lexer<'input> {
             return false;
         };
         if *next_char == char {
-            self.next_char(false);
+            self.next_char();
             return true;
         }
         false
     }
 
-    fn next_char(&mut self, escaped: bool) -> Option<(Position, char)> {
+    fn next_char(&mut self) -> Option<(Position, char)> {
         if let Some((byte_offset, char)) = self.iter.next() {
             self.position.byte_index = byte_offset;
             let res = Some((self.position.clone(), char));
-            if char != '\\' || escaped {
-                self.position.column += 1;
-            }
+            self.position.column += 1;
             return res;
         }
         None
@@ -117,6 +115,13 @@ impl<'input> Lexer<'input> {
         )
     }
 
+    fn escape(&mut self, position: Position) {
+        self.literal_start.get_or_insert(position);
+        if self.next_char().is_some() {
+            self.escaped += 1;
+        }
+    }
+
     fn take_while(&mut self, c: char, kind: TokenKind, start: Position) {
         while self.next_is(c) {}
         self.emit(Token::new(
@@ -136,13 +141,7 @@ impl<'input> Lexer<'input> {
             '%' if self.next_is('}') => {
                 self.emit(self.to_token(TokenKind::CollapsibleEnd, position, 2))
             }
-            '\\' => {
-                self.emit_literal_if_started(position.byte_index);
-                if let Some((pos, _)) = self.next_char(true) {
-                    self.escaped = true;
-                    self.literal_start.get_or_insert(pos);
-                }
-            }
+            '\\' => self.escape(position),
             '~' => self.take_while('~', TokenKind::Tilde, position),
             '*' => self.take_while('*', TokenKind::Star, position),
             '}' => self.take_while('}', TokenKind::RightCurlyBrace, position),
@@ -168,7 +167,7 @@ impl<'input> Lexer<'input> {
 
     fn advance(&mut self) {
         while self.queue.is_empty() {
-            if let Some((position, char)) = self.next_char(false) {
+            if let Some((position, char)) = self.next_char() {
                 self.parse(position, char);
             } else {
                 self.position.byte_index = self.len;
@@ -254,20 +253,20 @@ mod tests {
             vec![
                 Token {
                     kind: TokenKind::Literal,
-                    range: 1..2,
+                    range: 0..2,
                     position: Position {
-                        byte_index: 1,
+                        byte_index: 0,
                         column: 0,
                         row: 0
                     },
-                    escaped: true
+                    escaped: 1
                 },
                 Token::new(
                     TokenKind::LeftSquareBracket,
                     2..3,
                     Position {
                         byte_index: 2,
-                        column: 1,
+                        column: 2,
                         row: 0
                     }
                 )
@@ -329,13 +328,13 @@ mod tests {
                 Token::new(TokenKind::Hash, 0..3, Position::default()),
                 Token {
                     kind: TokenKind::Literal,
-                    range: 4..7,
+                    range: 3..7,
                     position: Position {
-                        byte_index: 4,
+                        byte_index: 3,
                         column: 3,
                         row: 0,
                     },
-                    escaped: true
+                    escaped: 1
                 },
             ]
         );
@@ -345,28 +344,16 @@ mod tests {
     fn escaped_space_compression() {
         assert_eq!(
             Lexer::new("\\ \\  ").collect::<Vec<_>>(),
-            vec![
-                Token {
-                    kind: TokenKind::Literal,
-                    range: 1..2,
-                    position: Position {
-                        byte_index: 1,
-                        column: 0,
-                        row: 0
-                    },
-                    escaped: true
-                },
-                Token {
-                    kind: TokenKind::Literal,
-                    range: 3..5,
-                    position: Position {
-                        byte_index: 3,
-                        column: 1,
-                        row: 0
-                    },
-                    escaped: true
+            vec![Token {
+                kind: TokenKind::Literal,
+                range: 0..5,
+                position: Position {
+                    byte_index: 0,
+                    column: 0,
+                    row: 0
                 },
-            ]
+                escaped: 2
+            }]
         );
     }
 
@@ -531,13 +518,13 @@ mod tests {
             Lexer::new("\\\\").collect::<Vec<_>>(),
             vec![Token {
                 kind: TokenKind::Literal,
-                range: 1..2,
+                range: 0..2,
                 position: Position {
-                    byte_index: 1,
+                    byte_index: 0,
                     column: 0,
                     row: 0
                 },
-                escaped: true
+                escaped: 1
             },]
         )
     }
@@ -547,23 +534,18 @@ mod tests {
         assert_eq!(
             Lexer::new("literal\\[[").collect::<Vec<_>>(),
             vec![
-                Token::new(TokenKind::Literal, 0..7, Position::default()),
                 Token {
                     kind: TokenKind::Literal,
-                    range: 8..9,
-                    position: Position {
-                        byte_index: 8,
-                        column: 7,
-                        row: 0,
-                    },
-                    escaped: true
+                    range: 0..9,
+                    position: Position::default(),
+                    escaped: 1
                 },
                 Token::new(
                     TokenKind::LeftSquareBracket,
                     9..10,
                     Position {
                         byte_index: 9,
-                        column: 8,
+                        column: 9,
                         row: 0,
                     },
                 ),
@@ -733,6 +715,9 @@ mod tests {
 
     #[test]
     fn dangling_backslash_at_eof() {
-        assert_eq!(Lexer::new("\\").collect::<Vec<_>>(), vec![]);
+        assert_eq!(
+            Lexer::new("\\").collect::<Vec<_>>(),
+            vec![Token::new(TokenKind::Literal, 0..1, Position::default())]
+        );
     }
 }

src/lexer/token.rs

@@ -101,18 +101,19 @@ pub struct Token {
     pub range: Range<usize>,
     /// The position of the token in the input string.
     pub position: Position,
-    /// Indicates if the token is escaped.
-    pub escaped: bool,
+    /// How many `\`-escapes are folded into this token's range. `0` means the range can be used
+    /// as-is; otherwise the text needs unescaping (see `Content::as_str`) before use.
+    pub escaped: u32,
 }
 
 impl Token {
-    /// Creates a new non escaped `Token` instance.
+    /// Creates a new `Token` instance with no escapes.
     pub fn new(kind: TokenKind, range: Range<usize>, position: Position) -> Self {
         Self {
             kind,
             range,
             position,
-            escaped: false,
+            escaped: 0,
         }
     }
 }
@@ -158,6 +159,6 @@ mod tests {
         let token = Token::new(TokenKind::Literal, 0..5, Position::default());
         assert_eq!(token.kind, TokenKind::Literal);
         assert_eq!(token.range, 0..5);
-        assert!(!token.escaped);
+        assert_eq!(token.escaped, 0);
     }
 }

src/lib.rs

@@ -41,8 +41,10 @@
 //!
 //! ## Escaping
 //!
-//! Escaping is handled at the [lexer] level: any character following `\` is treated as a
-//! [literal](lexer::TokenKind::Literal).
+//! Escaping is recognized at the [lexer] level: `\` forces the following character into the
+//! surrounding [literal](lexer::TokenKind::Literal) run instead of its usual meaning. The `\`
+//! itself is stripped later, lazily, when the content is resolved to text (e.g. via
+//! [`Content::as_str`](op::Content::as_str), or transparently as part of [`deserialize`]).
 //!
 //! Example:
 //!

src/op/anchor.rs

@@ -3,13 +3,13 @@ use crate::op::{Content, Node, Op, Parser, destination::destination, title::titl
 pub fn anchor(p: &mut Parser) -> bool {
     let start = p.pos;
     let snap = p.ops.len();
-    p.ops.push(Op::new_start(Node::Anchor, Content::Span(0..0)));
+    p.ops.push(Op::new_start(Node::Anchor, Content::empty()));
     if !title(p) || !destination(p) {
         p.pos = start;
         p.ops.truncate(snap);
         return false;
     }
-    p.ops.push(Op::new_end(Node::Anchor, Content::Span(0..0)));
+    p.ops.push(Op::new_end(Node::Anchor, Content::empty()));
     true
 }
 
@@ -27,14 +27,14 @@ mod tests {
         assert_eq!(
             p.ops,
             vec![
-                Op::new_start(Node::Anchor, Content::Span(0..0)),
+                Op::new_start(Node::Anchor, Content::empty()),
                 Op::new_start(Node::Title, p.span(0..1)),
                 Op::new_value(p.span(1..2)),
                 Op::new_end(Node::Title, p.span(2..3)),
                 Op::new_start(Node::Destination, p.span(3..4)),
                 Op::new_value(p.span(4..5)),
                 Op::new_end(Node::Destination, p.span(5..6)),
-                Op::new_end(Node::Anchor, Content::Span(0..0))
+                Op::new_end(Node::Anchor, Content::empty())
             ]
         );
     }
@@ -74,14 +74,14 @@ mod tests {
         assert_eq!(
             p.ops,
             vec![
-                Op::new_start(Node::Anchor, Content::Span(0..0)),
+                Op::new_start(Node::Anchor, Content::empty()),
                 Op::new_start(Node::Title, p.span(0..1)),
-                Op::new_value(p.span(1..4)),
-                Op::new_end(Node::Title, p.span(4..5)),
-                Op::new_start(Node::Destination, p.span(5..6)),
-                Op::new_value(p.span(6..7)),
-                Op::new_end(Node::Destination, p.span(7..8)),
-                Op::new_end(Node::Anchor, Content::Span(0..0))
+                Op::new_value(p.span(1..3)),
+                Op::new_end(Node::Title, p.span(3..4)),
+                Op::new_start(Node::Destination, p.span(4..5)),
+                Op::new_value(p.span(5..6)),
+                Op::new_end(Node::Destination, p.span(6..7)),
+                Op::new_end(Node::Anchor, Content::empty())
             ]
         );
     }
@@ -93,14 +93,14 @@ mod tests {
         assert_eq!(
             p.ops,
             vec![
-                Op::new_start(Node::Anchor, Content::Span(0..0)),
+                Op::new_start(Node::Anchor, Content::empty()),
                 Op::new_start(Node::Title, p.span(0..1)),
                 Op::new_value(p.span(1..2)),
                 Op::new_end(Node::Title, p.span(2..3)),
                 Op::new_start(Node::Destination, p.span(3..4)),
                 Op::new_value(p.span(4..8)),
                 Op::new_end(Node::Destination, p.span(8..9)),
-                Op::new_end(Node::Anchor, Content::Span(0..0))
+                Op::new_end(Node::Anchor, Content::empty())
             ]
         );
     }
@@ -112,14 +112,14 @@ mod tests {
         assert_eq!(
             p.ops,
             vec![
-                Op::new_start(Node::Anchor, Content::Span(0..0)),
+                Op::new_start(Node::Anchor, Content::empty()),
                 Op::new_start(Node::Title, p.span(0..1)),
                 Op::new_value(p.span(1..2)),
                 Op::new_end(Node::Title, p.span(2..3)),
                 Op::new_start(Node::Destination, p.span(3..4)),
                 Op::new_value(p.span(4..6)),
                 Op::new_end(Node::Destination, p.span(6..7)),
-                Op::new_end(Node::Anchor, Content::Span(0..0))
+                Op::new_end(Node::Anchor, Content::empty())
             ]
         );
     }