[WJ-1060] Avoid exponential parsing of nested unclosed blocks

src/parsing/parser.rs

@@ -30,6 +30,7 @@ use crate::tree::{
 };
 use std::borrow::Cow;
 use std::cell::RefCell;
+use std::collections::HashMap;
 use std::rc::Rc;
 use std::{mem, ptr};
 
@@ -79,6 +80,11 @@ pub struct Parser<'r, 't> {
     // overriding later ones.
     bibliographies: Rc<RefCell<BibliographyList<'t>>>,
 
+    // Failed block parses can be retried after an enclosing block falls back
+    // to text. Keep terminal failures so malformed nested blocks do not cause
+    // the same suffix to be parsed exponentially many times.
+    block_failures: Rc<RefCell<HashMap<BlockFailureKey, CachedBlockFailure>>>,
+
     // Flags
     accepts_partial: AcceptsPartial,
     in_footnote: bool, // Whether we're currently inside [[footnote]] ... [[/footnote]].
@@ -115,6 +121,7 @@ impl<'r, 't> Parser<'r, 't> {
             code_blocks: make_shared_vec(),
             footnotes: make_shared_vec(),
             bibliographies: Rc::new(RefCell::new(BibliographyList::new())),
+            block_failures: Rc::new(RefCell::new(HashMap::new())),
             accepts_partial: AcceptsPartial::None,
             in_footnote: false,
             has_footnote_block: false,
@@ -216,6 +223,7 @@ impl<'r, 't> Parser<'r, 't> {
             html_block_index: self.html_blocks.borrow().len(),
             code_block_index: self.code_blocks.borrow().len(),
             table_of_contents_index: self.table_of_contents.borrow().len(),
+            bibliography_index: self.bibliographies.borrow().next_index(),
         }
     }
 
@@ -229,6 +237,7 @@ impl<'r, 't> Parser<'r, 't> {
             html_block_index,
             code_block_index,
             table_of_contents_index,
+            bibliography_index,
         }: ParserMutableState,
     ) {
         self.footnotes.borrow_mut().truncate(footnote_index);
@@ -237,6 +246,9 @@ impl<'r, 't> Parser<'r, 't> {
         self.table_of_contents
             .borrow_mut()
             .truncate(table_of_contents_index);
+        self.bibliographies
+            .borrow_mut()
+            .truncate(bibliography_index);
     }
 
     // Parse settings helpers
@@ -292,6 +304,26 @@ impl<'r, 't> Parser<'r, 't> {
         self.footnotes.borrow_mut().truncate(count);
     }
 
+    pub(crate) fn cached_block_failure(&self, rule: &'static str) -> Option<ParseError> {
+        let key = self.block_failure_key(rule);
+        self.block_failures
+            .borrow()
+            .get(&key)
+            .filter(|failure| self.depth <= failure.depth)
+            .map(|failure| failure.error.clone())
+    }
+
+    pub(crate) fn cache_block_failure(&mut self, rule: &'static str, error: &ParseError) {
+        let key = self.block_failure_key(rule);
+        self.block_failures.borrow_mut().insert(
+            key,
+            CachedBlockFailure {
+                error: error.clone(),
+                depth: self.depth,
+            },
+        );
+    }
+
     #[cold]
     pub fn remove_footnotes(&mut self) -> Vec<Vec<Element<'t>>> {
         mem::take(&mut self.footnotes.borrow_mut())
@@ -597,6 +629,52 @@ impl<'r, 't> Parser<'r, 't> {
     }
 }
 
+/// State which affects whether a terminal block parse can be retried.
+///
+/// The recursion depth is intentionally not part of this key. The cached
+/// failure records it separately and is only reused at an equal or shallower
+/// depth, so recursion-limit errors retain their original behavior.
+#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
+struct BlockFailureKey {
+    token: usize,
+    rule: &'static str,
+    accepts_partial: AcceptsPartial,
+    in_footnote: bool,
+    has_footnote_block: bool,
+    start_of_line: bool,
+    bibliography_index: usize,
+    footnote_index: usize,
+    html_block_index: usize,
+    code_block_index: usize,
+    table_of_contents_index: usize,
+}
+
+#[derive(Debug, Clone)]
+struct CachedBlockFailure {
+    error: ParseError,
+    depth: usize,
+}
+
+impl<'r, 't> Parser<'r, 't> {
+    fn block_failure_key(&self, rule: &'static str) -> BlockFailureKey {
+        let mutable_state = self.get_mutable_state();
+
+        BlockFailureKey {
+            token: self.current as *const ExtractedToken<'t> as usize,
+            rule,
+            accepts_partial: self.accepts_partial,
+            in_footnote: self.in_footnote,
+            has_footnote_block: self.has_footnote_block,
+            start_of_line: self.start_of_line,
+            bibliography_index: mutable_state.bibliography_index,
+            footnote_index: mutable_state.footnote_index,
+            html_block_index: mutable_state.html_block_index,
+            code_block_index: mutable_state.code_block_index,
+            table_of_contents_index: mutable_state.table_of_contents_index,
+        }
+    }
+}
+
 /// This struct stores the state of the mutable fields in `Parser`.
 ///
 /// This way, on rule failure, we can revert to the state these
@@ -614,6 +692,7 @@ pub struct ParserMutableState {
     html_block_index: usize,
     code_block_index: usize,
     table_of_contents_index: usize,
+    bibliography_index: usize,
 }
 
 #[inline]
@@ -663,3 +742,28 @@ fn parser_newline_flag() {
         [true, true, false, true, false, true, false, false],
     );
 }
+
+#[test]
+fn block_failure_cache_uses_rolled_back_state() {
+    use super::consume::consume;
+    use crate::layout::Layout;
+    use crate::settings::WikitextMode;
+
+    let page_info = PageInfo::dummy();
+    let settings = WikitextSettings::from_mode(WikitextMode::Page, Layout::Wikidot);
+    let mut input = String::from("[[div]]\n[[code]]\ncode\n[[/code]]\n");
+
+    crate::preprocess(&mut input);
+    let tokens = crate::tokenize(&input);
+    let mut parser = Parser::new(&tokens, &page_info, &settings);
+
+    parser.step().expect("expected the first input token");
+    let _ = consume(&mut parser).expect("unclosed div should fall back to text");
+
+    let failures = parser.block_failures.borrow();
+    let failure = failures
+        .keys()
+        .next()
+        .expect("expected a cached block failure");
+    assert_eq!(failure.code_block_index, 0);
+}

src/parsing/rule/mod.rs

@@ -66,6 +66,11 @@ impl Rule {
             }
         }
 
+        let memoize_failure = matches!(self.name, "block" | "block-star");
+        if memoize_failure && let Some(error) = parser.cached_block_failure(self.name) {
+            return Err(error);
+        }
+
         // Fork parser and try running the rule.
         let parser_state = parser.get_mutable_state();
         let mut sub_parser = parser.clone_with_rule(self);
@@ -85,7 +90,13 @@ impl Rule {
             //
             // While normally discarding the subparser is sufficient,
             // some annoying mutable fields are
-            Err(_) => parser.reset_mutable_state(parser_state),
+            Err(ref error) => {
+                parser.reset_mutable_state(parser_state);
+
+                if memoize_failure && error.kind() == ParseErrorKind::EndOfInput {
+                    parser.cache_block_failure(self.name, error);
+                }
+            }
         }
 
         result

src/test/large.rs

@@ -70,6 +70,53 @@ fn recursion_depth() {
     assert_eq!(element, &Element::Text(input_cow));
 }
 
+/// Unclosed nested blocks used to retry the same suffix exponentially often.
+#[test]
+fn nested_unclosed_divs() {
+    const ITERATIONS: usize = 22;
+
+    let page_info = PageInfo::dummy();
+    let settings = WikitextSettings::from_mode(WikitextMode::Page, Layout::Wikidot);
+
+    let mut input = String::new();
+    for _ in 0..ITERATIONS {
+        input.push_str("[[div]]\n");
+    }
+
+    crate::preprocess(&mut input);
+    let tokens = crate::tokenize(&input);
+    let (tree, errors) = crate::parse(&tokens, &page_info, &settings).into();
+
+    assert_eq!(errors.len(), ITERATIONS * 3);
+    assert_eq!(tree.elements.len(), 1);
+}
+
+/// Failed nested blocks must roll back bibliography state before caching.
+#[test]
+fn nested_unclosed_blocks_preserve_bibliography_indices() {
+    const ITERATIONS: usize = 22;
+
+    let page_info = PageInfo::dummy();
+    let settings = WikitextSettings::from_mode(WikitextMode::Page, Layout::Wikidot);
+
+    let mut input = String::new();
+    for _ in 0..ITERATIONS {
+        input.push_str("[[div]]\n");
+    }
+    input.push_str("[[bibliography]]\n: foo : bar\n[[/bibliography]]\n");
+
+    crate::preprocess(&mut input);
+    let tokens = crate::tokenize(&input);
+    let (tree, errors) = crate::parse(&tokens, &page_info, &settings).into();
+
+    assert_eq!(errors.len(), ITERATIONS * 3);
+    assert!(matches!(
+        tree.elements.last(),
+        Some(Element::BibliographyBlock { index: 0, .. })
+    ));
+    assert_eq!(tree.bibliographies.next_index(), 1);
+}
+
 /// Test the parser's ability to process large bodies
 #[test]
 #[ignore = "slow test"]

src/tree/bibliography.rs

@@ -102,6 +102,10 @@ impl<'t> BibliographyList<'t> {
         self.0.append(&mut other.0);
     }
 
+    pub(crate) fn truncate(&mut self, length: usize) {
+        self.0.truncate(length);
+    }
+
     #[inline]
     pub fn is_empty(&self) -> bool {
         self.0.is_empty()