Enhancement: add more readable parser error snapshots

Cargo.toml

@@ -60,6 +60,7 @@ built = { version = "0.8", features = ["chrono", "git2"] }
 clap = "4"
 proptest = "1"
 termcolor = "1"
+codespan-reporting = "0.13.1"
 
 [target.'cfg(target_arch = "wasm32")'.dependencies]
 getrandom = { version = "0.4", features = ["wasm_js"] }

src/test/ast/loader.rs

@@ -235,6 +235,7 @@ impl Test {
         let mut input = None;
         let mut tree = None;
         let mut errors = None;
+        let mut errors_stderr = None;
         let mut wikidot_output = None;
         let mut html_output = None;
         let mut text_output = None;
@@ -272,6 +273,7 @@ impl Test {
                     ),
                     "tree.json" => tree = Some(empty_syntax_tree()),
                     "errors.json" => errors = Some(Vec::new()),
+                    "errors.stderr" => errors_stderr = Some(String::new()),
                     "wikidot.html" => wikidot_output = Some(String::new()),
                     "output.html" => html_output = Some(String::new()),
                     "output.txt" => text_output = Some(String::new()),
@@ -286,6 +288,7 @@ impl Test {
                 "input.ftml" => input = Some(read_text_file(&path)),
                 "tree.json" => tree = Some(read_json(&path)),
                 "errors.json" => errors = Some(read_json(&path)),
+                "errors.stderr" => errors_stderr = Some(read_text_file(&path)),
                 "wikidot.html" => wikidot_output = Some(read_text_file(&path)),
                 "output.html" => html_output = Some(read_text_file(&path)),
                 "output.txt" => text_output = Some(read_text_file(&path)),
@@ -313,6 +316,7 @@ impl Test {
             input,
             tree,
             errors,
+            errors_stderr,
             wikidot_output,
             html_output,
             text_output,

src/test/ast/mod.rs

@@ -56,7 +56,7 @@ const UPDATE_TESTS: bool = false;
 
 /// The directory where all test files are located.
 /// This is the directory `test` under the repository root.
-static TEST_DIRECTORY: LazyLock<PathBuf> = LazyLock::new(|| {
+pub(crate) static TEST_DIRECTORY: LazyLock<PathBuf> = LazyLock::new(|| {
     let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
     path.push("test");
     path
@@ -147,6 +147,10 @@ pub struct Test {
     /// Read from `errors.json`.
     pub errors: Option<Vec<ParseError>>,
 
+    /// The readable error message from this test.
+    /// Read from `errors.stderr`.
+    pub errors_stderr: Option<String>,
+
     /// The Wikidot-layout HTML expected to be generated from this input.
     /// Read from `wikidot.html`.
     pub wikidot_output: Option<String>,

src/test/ast/runner.rs

@@ -23,15 +23,23 @@
 use super::{Test, TestResult, TestStats, TestUniverse};
 use crate::data::{PageInfo, ScoreValue};
 use crate::layout::Layout;
+use crate::parsing::ParseError;
 use crate::render::Render;
 use crate::render::html::HtmlRender;
 use crate::render::text::TextRender;
 use crate::settings::{WikitextMode, WikitextSettings};
+use crate::test::ast::TEST_DIRECTORY;
 use crate::test::includer::TestIncluder;
+use codespan_reporting::{
+    diagnostic::{Diagnostic, Label},
+    files::SimpleFiles,
+    term,
+};
 use std::borrow::Cow;
 use std::fs::{self, File};
 use std::io::Write;
 use std::path::{Path, PathBuf};
+use termcolor::Buffer;
 
 macro_rules! cow {
     ($value:expr $(,)?) => {
@@ -189,6 +197,24 @@ impl Test {
             }
         }
 
+        // Only run stderr generate and check when file exists.
+        let stderr_path = TEST_DIRECTORY.join("errors.stderr");
+
+        if stderr_path.exists() {
+            // Render parser errors as compiler-style diagnostics with source locations.
+            let stderr =
+                render_errors_to_stderr("input.ftml", &self.input, &actual_errors);
+            // Run and check stderr
+            if let Some(expected_stderr) = &self.errors_stderr
+                && &stderr != expected_stderr
+            {
+                result = TestResult::Fail;
+                eprintln!("Parse stderr did not match:");
+                eprintln!("Expected:\n{}", expected_stderr);
+                eprintln!("Actual:\n{}", stderr);
+            }
+        }
+
         result
     }
 
@@ -283,6 +309,19 @@ impl Test {
                 update!(write_text, actual_text, "output.txt");
             }
         }
+
+        // Only run stderr generate and check when file exists.
+        let stderr_path = TEST_DIRECTORY.join("errors.stderr");
+
+        if stderr_path.exists() {
+            let stderr = render_errors_to_stderr("input.ftml", &self.input, &errors); // see run()
+            // Run and check stderr
+            if let Some(expected_stderr) = &self.errors_stderr
+                && &stderr != expected_stderr
+            {
+                update!(write_text, stderr, "errors.stderr");
+            }
+        }
     }
 }
 
@@ -354,3 +393,33 @@ fn write_text(path: &Path, contents: &str) {
     file.write_all(b"\n")
         .expect("Unable to write final newline to file");
 }
+
+/// Helper function for rendering errors to stderr.
+fn render_errors_to_stderr(
+    source_name: &str,
+    source: &str,
+    errors: &[ParseError],
+) -> String {
+    let mut files = SimpleFiles::new();
+
+    let file_id = files.add(source_name, source);
+
+    let config = term::Config::default();
+
+    // no color because `.stderr` is raw text
+    let mut output = Buffer::no_color();
+
+    for error in errors {
+        let diagnostic = Diagnostic::error()
+            .with_message(error.kind().name())
+            .with_labels(vec![
+                Label::primary(file_id, error.span()).with_message(error.rule()),
+            ]);
+
+        term::emit_to_write_style(&mut output, &config, &files, &diagnostic)
+            .expect("failed to emit diagnostic");
+    }
+
+    String::from_utf8(output.as_slice().to_vec())
+        .expect("diagnostic output was not utf-8")
+}

test/color/fail/errors.stderr

@@ -0,0 +1,28 @@
+error: RuleFailed
+  ┌─ input.ftml:1:12
+  │  
+1 │   ##not color
+  │ ╭───────────^
+2 │ │ 
+3 │ │ ##|no spec##
+  │ ╰^ color
+
+error: NoRulesMatch
+  ┌─ input.ftml:1:1
+  │
+1 │ ##not color
+  │ ^^ fallback
+
+error: EndOfInput
+  ┌─ input.ftml:5:14
+  │
+5 │ ##only color|
+  │              ^ color
+
+error: NoRulesMatch
+  ┌─ input.ftml:5:1
+  │
+5 │ ##only color|
+  │ ^^ fallback
+
+

test/footnote/block-inside-fail/errors.stderr

@@ -0,0 +1,19 @@
+error: FootnotesNested
+  ┌─ input.ftml:1:35
+  │
+1 │ Apple[[footnote]][[footnoteblock]][[/footnote]]Banana
+  │                                   ^^^ block-footnote-block
+
+error: NoRulesMatch
+  ┌─ input.ftml:1:18
+  │
+1 │ Apple[[footnote]][[footnoteblock]][[/footnote]]Banana
+  │                  ^^ fallback
+
+error: NoRulesMatch
+  ┌─ input.ftml:1:33
+  │
+1 │ Apple[[footnote]][[footnoteblock]][[/footnote]]Banana
+  │                                 ^^ fallback
+
+

test/footnote/revert/errors.stderr

@@ -0,0 +1,55 @@
+error: EndOfInput
+  ┌─ input.ftml:9:31
+  │
+9 │ 2[[footnote]]Beta[[/footnote]]
+  │                               ^ block-div
+
+error: NoRulesMatch
+  ┌─ input.ftml:1:1
+  │
+1 │ [[div]]
+  │ ^^ fallback
+
+error: NoRulesMatch
+  ┌─ input.ftml:1:6
+  │
+1 │ [[div]]
+  │      ^^ fallback
+
+error: EndOfInput
+  ┌─ input.ftml:9:31
+  │
+9 │ 2[[footnote]]Beta[[/footnote]]
+  │                               ^ block-div
+
+error: NoRulesMatch
+  ┌─ input.ftml:2:1
+  │
+2 │ [[div]]
+  │ ^^ fallback
+
+error: NoRulesMatch
+  ┌─ input.ftml:2:6
+  │
+2 │ [[div]]
+  │      ^^ fallback
+
+error: EndOfInput
+  ┌─ input.ftml:9:31
+  │
+9 │ 2[[footnote]]Beta[[/footnote]]
+  │                               ^ block-div
+
+error: NoRulesMatch
+  ┌─ input.ftml:3:1
+  │
+3 │ [[div]]
+  │ ^^ fallback
+
+error: NoRulesMatch
+  ┌─ input.ftml:3:6
+  │
+3 │ [[div]]
+  │      ^^ fallback
+
+