From 9e965c6fe1bd1fd3801a84d8e4e8ad6a541ad359 Mon Sep 17 00:00:00 2001 From: wasd243 Date: Wed, 29 Jul 2026 12:30:08 -0400 Subject: [PATCH 1/3] feat(stderr): allow `.stderr` file in `test/` directory to make parse errors more readable --- Cargo.toml | 1 + src/test/ast/loader.rs | 4 +++ src/test/ast/mod.rs | 4 +++ src/test/ast/runner.rs | 57 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 66 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index 90c28d2b6..5e4fd7941 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -60,6 +60,7 @@ built = { version = "0.8", features = ["chrono", "git2"] } clap = "4" proptest = "1" termcolor = "1" +codespan-reporting = "0.13.1" [target.'cfg(target_arch = "wasm32")'.dependencies] getrandom = { version = "0.4", features = ["wasm_js"] } diff --git a/src/test/ast/loader.rs b/src/test/ast/loader.rs index 008470a3a..098f76988 100644 --- a/src/test/ast/loader.rs +++ b/src/test/ast/loader.rs @@ -235,6 +235,7 @@ impl Test { let mut input = None; let mut tree = None; let mut errors = None; + let mut errors_stderr = None; let mut wikidot_output = None; let mut html_output = None; let mut text_output = None; @@ -272,6 +273,7 @@ impl Test { ), "tree.json" => tree = Some(empty_syntax_tree()), "errors.json" => errors = Some(Vec::new()), + "errors.stderr" => errors_stderr = Some(String::new()), "wikidot.html" => wikidot_output = Some(String::new()), "output.html" => html_output = Some(String::new()), "output.txt" => text_output = Some(String::new()), @@ -286,6 +288,7 @@ impl Test { "input.ftml" => input = Some(read_text_file(&path)), "tree.json" => tree = Some(read_json(&path)), "errors.json" => errors = Some(read_json(&path)), + "errors.stderr" => errors_stderr = Some(read_text_file(&path)), "wikidot.html" => wikidot_output = Some(read_text_file(&path)), "output.html" => html_output = Some(read_text_file(&path)), "output.txt" => text_output = Some(read_text_file(&path)), @@ -313,6 +316,7 @@ impl Test { input, tree, errors, + errors_stderr, wikidot_output, html_output, text_output, diff --git a/src/test/ast/mod.rs b/src/test/ast/mod.rs index 8b54bb7c9..3f5ebf26f 100644 --- a/src/test/ast/mod.rs +++ b/src/test/ast/mod.rs @@ -147,6 +147,10 @@ pub struct Test { /// Read from `errors.json`. pub errors: Option>, + /// The readable error message for this test. + /// Read from `errors.stderr`. + pub errors_stderr: Option, + /// The Wikidot-layout HTML expected to be generated from this input. /// Read from `wikidot.html`. pub wikidot_output: Option, diff --git a/src/test/ast/runner.rs b/src/test/ast/runner.rs index 2563518c0..5a7a13497 100644 --- a/src/test/ast/runner.rs +++ b/src/test/ast/runner.rs @@ -23,15 +23,22 @@ use super::{Test, TestResult, TestStats, TestUniverse}; use crate::data::{PageInfo, ScoreValue}; use crate::layout::Layout; +use crate::parsing::ParseError; use crate::render::Render; use crate::render::html::HtmlRender; use crate::render::text::TextRender; use crate::settings::{WikitextMode, WikitextSettings}; use crate::test::includer::TestIncluder; +use codespan_reporting::{ + diagnostic::{Diagnostic, Label}, + files::SimpleFiles, + term, +}; use std::borrow::Cow; use std::fs::{self, File}; use std::io::Write; use std::path::{Path, PathBuf}; +use termcolor::Buffer; macro_rules! cow { ($value:expr $(,)?) => { @@ -125,6 +132,8 @@ impl Test { let (mut tree, actual_errors) = result.into(); tree.wikitext_len = 0; // not stored in the JSON, need for correct eq + let stderr = render_errors_stderr("input.ftml", &self.input, &actual_errors); + let mut result = TestResult::Pass; // Check abstract syntax tree @@ -189,6 +198,16 @@ impl Test { } } + // Run and check stderr + if let Some(expected_stderr) = &self.errors_stderr { + if &stderr != expected_stderr { + result = TestResult::Fail; + eprintln!("Parse stderr did not match:"); + eprintln!("Expected:\n{}", expected_stderr); + eprintln!("Actual:\n{}", stderr); + } + } + result } @@ -213,6 +232,8 @@ impl Test { let (mut tree, errors) = result.into(); tree.wikitext_len = 0; // see run() + let stderr = render_errors_stderr("input.ftml", &self.input, &errors); + macro_rules! update { ($write_func:ident, $object:expr, $filename:expr $(,)?) => {{ println!("= {}/{}", self.name, $filename); @@ -283,6 +304,13 @@ impl Test { update!(write_text, actual_text, "output.txt"); } } + + // Run and check stderr + if let Some(expected_stderr) = &self.errors_stderr { + if &stderr != expected_stderr { + update!(write_text, stderr, "errors.stderr"); + } + } } } @@ -354,3 +382,32 @@ fn write_text(path: &Path, contents: &str) { file.write_all(b"\n") .expect("Unable to write final newline to file"); } + +/// Helper function for rendering errors to stderr. +fn render_errors_stderr( + source_name: &str, + source: &str, + errors: &[ParseError], +) -> String { + let mut files = SimpleFiles::new(); + + let file_id = files.add(source_name, source); + + let config = term::Config::default(); + + // no color because `.stderr` is raw text + let mut output = Buffer::no_color(); + + for error in errors { + let diagnostic = Diagnostic::error() + .with_message(error.kind().name()) + .with_labels(vec![ + Label::primary(file_id, error.span()).with_message(error.rule()), + ]); + + term::emit_to_write_style(&mut output, &config, &files, &diagnostic) + .expect("failed to emit diagnostic"); + } + + String::from_utf8(output.as_slice().to_vec()).expect("diagnostic output was not utf8") +} From b7747c79db211a16215c012dbfcc31d68f0a4e27 Mon Sep 17 00:00:00 2001 From: wasd243 Date: Wed, 29 Jul 2026 13:02:21 -0400 Subject: [PATCH 2/3] Add multiple `.stderr` snapshots for parser error cases --- test/color/fail/errors.stderr | 28 ++++++++++ test/definition-list/fail/errors.stderr | 0 test/footnote/block-inside-fail/errors.stderr | 19 +++++++ test/footnote/revert/errors.stderr | 55 +++++++++++++++++++ 4 files changed, 102 insertions(+) create mode 100644 test/color/fail/errors.stderr create mode 100644 test/definition-list/fail/errors.stderr create mode 100644 test/footnote/block-inside-fail/errors.stderr create mode 100644 test/footnote/revert/errors.stderr diff --git a/test/color/fail/errors.stderr b/test/color/fail/errors.stderr new file mode 100644 index 000000000..a07f596f7 --- /dev/null +++ b/test/color/fail/errors.stderr @@ -0,0 +1,28 @@ +error: RuleFailed + ┌─ input.ftml:1:12 + │ +1 │ ##not color + │ ╭───────────^ +2 │ │ +3 │ │ ##|no spec## + │ ╰^ color + +error: NoRulesMatch + ┌─ input.ftml:1:1 + │ +1 │ ##not color + │ ^^ fallback + +error: EndOfInput + ┌─ input.ftml:5:14 + │ +5 │ ##only color| + │ ^ color + +error: NoRulesMatch + ┌─ input.ftml:5:1 + │ +5 │ ##only color| + │ ^^ fallback + + diff --git a/test/definition-list/fail/errors.stderr b/test/definition-list/fail/errors.stderr new file mode 100644 index 000000000..e69de29bb diff --git a/test/footnote/block-inside-fail/errors.stderr b/test/footnote/block-inside-fail/errors.stderr new file mode 100644 index 000000000..8b007d3aa --- /dev/null +++ b/test/footnote/block-inside-fail/errors.stderr @@ -0,0 +1,19 @@ +error: FootnotesNested + ┌─ input.ftml:1:35 + │ +1 │ Apple[[footnote]][[footnoteblock]][[/footnote]]Banana + │ ^^^ block-footnote-block + +error: NoRulesMatch + ┌─ input.ftml:1:18 + │ +1 │ Apple[[footnote]][[footnoteblock]][[/footnote]]Banana + │ ^^ fallback + +error: NoRulesMatch + ┌─ input.ftml:1:33 + │ +1 │ Apple[[footnote]][[footnoteblock]][[/footnote]]Banana + │ ^^ fallback + + diff --git a/test/footnote/revert/errors.stderr b/test/footnote/revert/errors.stderr new file mode 100644 index 000000000..a06f03fe7 --- /dev/null +++ b/test/footnote/revert/errors.stderr @@ -0,0 +1,55 @@ +error: EndOfInput + ┌─ input.ftml:9:31 + │ +9 │ 2[[footnote]]Beta[[/footnote]] + │ ^ block-div + +error: NoRulesMatch + ┌─ input.ftml:1:1 + │ +1 │ [[div]] + │ ^^ fallback + +error: NoRulesMatch + ┌─ input.ftml:1:6 + │ +1 │ [[div]] + │ ^^ fallback + +error: EndOfInput + ┌─ input.ftml:9:31 + │ +9 │ 2[[footnote]]Beta[[/footnote]] + │ ^ block-div + +error: NoRulesMatch + ┌─ input.ftml:2:1 + │ +2 │ [[div]] + │ ^^ fallback + +error: NoRulesMatch + ┌─ input.ftml:2:6 + │ +2 │ [[div]] + │ ^^ fallback + +error: EndOfInput + ┌─ input.ftml:9:31 + │ +9 │ 2[[footnote]]Beta[[/footnote]] + │ ^ block-div + +error: NoRulesMatch + ┌─ input.ftml:3:1 + │ +3 │ [[div]] + │ ^^ fallback + +error: NoRulesMatch + ┌─ input.ftml:3:6 + │ +3 │ [[div]] + │ ^^ fallback + + From 4de347eb315b3c85b931b1ad363bf20417104ee7 Mon Sep 17 00:00:00 2001 From: wasd243 Date: Wed, 29 Jul 2026 13:44:23 -0400 Subject: [PATCH 3/3] chore(stderr): skip stderr generation without `errors.stderr` --- src/test/ast/mod.rs | 4 ++-- src/test/ast/runner.rs | 36 ++++++++++++++++++++++++------------ 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/src/test/ast/mod.rs b/src/test/ast/mod.rs index 3f5ebf26f..839835bac 100644 --- a/src/test/ast/mod.rs +++ b/src/test/ast/mod.rs @@ -56,7 +56,7 @@ const UPDATE_TESTS: bool = false; /// The directory where all test files are located. /// This is the directory `test` under the repository root. -static TEST_DIRECTORY: LazyLock = LazyLock::new(|| { +pub(crate) static TEST_DIRECTORY: LazyLock = LazyLock::new(|| { let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); path.push("test"); path @@ -147,7 +147,7 @@ pub struct Test { /// Read from `errors.json`. pub errors: Option>, - /// The readable error message for this test. + /// The readable error message from this test. /// Read from `errors.stderr`. pub errors_stderr: Option, diff --git a/src/test/ast/runner.rs b/src/test/ast/runner.rs index 5a7a13497..dffc4b9d1 100644 --- a/src/test/ast/runner.rs +++ b/src/test/ast/runner.rs @@ -28,6 +28,7 @@ use crate::render::Render; use crate::render::html::HtmlRender; use crate::render::text::TextRender; use crate::settings::{WikitextMode, WikitextSettings}; +use crate::test::ast::TEST_DIRECTORY; use crate::test::includer::TestIncluder; use codespan_reporting::{ diagnostic::{Diagnostic, Label}, @@ -132,8 +133,6 @@ impl Test { let (mut tree, actual_errors) = result.into(); tree.wikitext_len = 0; // not stored in the JSON, need for correct eq - let stderr = render_errors_stderr("input.ftml", &self.input, &actual_errors); - let mut result = TestResult::Pass; // Check abstract syntax tree @@ -198,9 +197,17 @@ impl Test { } } - // Run and check stderr - if let Some(expected_stderr) = &self.errors_stderr { - if &stderr != expected_stderr { + // Only run stderr generate and check when file exists. + let stderr_path = TEST_DIRECTORY.join("errors.stderr"); + + if stderr_path.exists() { + // Render parser errors as compiler-style diagnostics with source locations. + let stderr = + render_errors_to_stderr("input.ftml", &self.input, &actual_errors); + // Run and check stderr + if let Some(expected_stderr) = &self.errors_stderr + && &stderr != expected_stderr + { result = TestResult::Fail; eprintln!("Parse stderr did not match:"); eprintln!("Expected:\n{}", expected_stderr); @@ -232,8 +239,6 @@ impl Test { let (mut tree, errors) = result.into(); tree.wikitext_len = 0; // see run() - let stderr = render_errors_stderr("input.ftml", &self.input, &errors); - macro_rules! update { ($write_func:ident, $object:expr, $filename:expr $(,)?) => {{ println!("= {}/{}", self.name, $filename); @@ -305,9 +310,15 @@ impl Test { } } - // Run and check stderr - if let Some(expected_stderr) = &self.errors_stderr { - if &stderr != expected_stderr { + // Only run stderr generate and check when file exists. + let stderr_path = TEST_DIRECTORY.join("errors.stderr"); + + if stderr_path.exists() { + let stderr = render_errors_to_stderr("input.ftml", &self.input, &errors); // see run() + // Run and check stderr + if let Some(expected_stderr) = &self.errors_stderr + && &stderr != expected_stderr + { update!(write_text, stderr, "errors.stderr"); } } @@ -384,7 +395,7 @@ fn write_text(path: &Path, contents: &str) { } /// Helper function for rendering errors to stderr. -fn render_errors_stderr( +fn render_errors_to_stderr( source_name: &str, source: &str, errors: &[ParseError], @@ -409,5 +420,6 @@ fn render_errors_stderr( .expect("failed to emit diagnostic"); } - String::from_utf8(output.as_slice().to_vec()).expect("diagnostic output was not utf8") + String::from_utf8(output.as_slice().to_vec()) + .expect("diagnostic output was not utf-8") }