diff --git a/Cargo.toml b/Cargo.toml index 90c28d2b6..5e4fd7941 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -60,6 +60,7 @@ built = { version = "0.8", features = ["chrono", "git2"] } clap = "4" proptest = "1" termcolor = "1" +codespan-reporting = "0.13.1" [target.'cfg(target_arch = "wasm32")'.dependencies] getrandom = { version = "0.4", features = ["wasm_js"] } diff --git a/src/test/ast/loader.rs b/src/test/ast/loader.rs index 008470a3a..098f76988 100644 --- a/src/test/ast/loader.rs +++ b/src/test/ast/loader.rs @@ -235,6 +235,7 @@ impl Test { let mut input = None; let mut tree = None; let mut errors = None; + let mut errors_stderr = None; let mut wikidot_output = None; let mut html_output = None; let mut text_output = None; @@ -272,6 +273,7 @@ impl Test { ), "tree.json" => tree = Some(empty_syntax_tree()), "errors.json" => errors = Some(Vec::new()), + "errors.stderr" => errors_stderr = Some(String::new()), "wikidot.html" => wikidot_output = Some(String::new()), "output.html" => html_output = Some(String::new()), "output.txt" => text_output = Some(String::new()), @@ -286,6 +288,7 @@ impl Test { "input.ftml" => input = Some(read_text_file(&path)), "tree.json" => tree = Some(read_json(&path)), "errors.json" => errors = Some(read_json(&path)), + "errors.stderr" => errors_stderr = Some(read_text_file(&path)), "wikidot.html" => wikidot_output = Some(read_text_file(&path)), "output.html" => html_output = Some(read_text_file(&path)), "output.txt" => text_output = Some(read_text_file(&path)), @@ -313,6 +316,7 @@ impl Test { input, tree, errors, + errors_stderr, wikidot_output, html_output, text_output, diff --git a/src/test/ast/mod.rs b/src/test/ast/mod.rs index 8b54bb7c9..839835bac 100644 --- a/src/test/ast/mod.rs +++ b/src/test/ast/mod.rs @@ -56,7 +56,7 @@ const UPDATE_TESTS: bool = false; /// The directory where all test files are located. /// This is the directory `test` under the repository root. -static TEST_DIRECTORY: LazyLock = LazyLock::new(|| { +pub(crate) static TEST_DIRECTORY: LazyLock = LazyLock::new(|| { let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); path.push("test"); path @@ -147,6 +147,10 @@ pub struct Test { /// Read from `errors.json`. pub errors: Option>, + /// The readable error message from this test. + /// Read from `errors.stderr`. + pub errors_stderr: Option, + /// The Wikidot-layout HTML expected to be generated from this input. /// Read from `wikidot.html`. pub wikidot_output: Option, diff --git a/src/test/ast/runner.rs b/src/test/ast/runner.rs index 2563518c0..dffc4b9d1 100644 --- a/src/test/ast/runner.rs +++ b/src/test/ast/runner.rs @@ -23,15 +23,23 @@ use super::{Test, TestResult, TestStats, TestUniverse}; use crate::data::{PageInfo, ScoreValue}; use crate::layout::Layout; +use crate::parsing::ParseError; use crate::render::Render; use crate::render::html::HtmlRender; use crate::render::text::TextRender; use crate::settings::{WikitextMode, WikitextSettings}; +use crate::test::ast::TEST_DIRECTORY; use crate::test::includer::TestIncluder; +use codespan_reporting::{ + diagnostic::{Diagnostic, Label}, + files::SimpleFiles, + term, +}; use std::borrow::Cow; use std::fs::{self, File}; use std::io::Write; use std::path::{Path, PathBuf}; +use termcolor::Buffer; macro_rules! cow { ($value:expr $(,)?) => { @@ -189,6 +197,24 @@ impl Test { } } + // Only run stderr generate and check when file exists. + let stderr_path = TEST_DIRECTORY.join("errors.stderr"); + + if stderr_path.exists() { + // Render parser errors as compiler-style diagnostics with source locations. + let stderr = + render_errors_to_stderr("input.ftml", &self.input, &actual_errors); + // Run and check stderr + if let Some(expected_stderr) = &self.errors_stderr + && &stderr != expected_stderr + { + result = TestResult::Fail; + eprintln!("Parse stderr did not match:"); + eprintln!("Expected:\n{}", expected_stderr); + eprintln!("Actual:\n{}", stderr); + } + } + result } @@ -283,6 +309,19 @@ impl Test { update!(write_text, actual_text, "output.txt"); } } + + // Only run stderr generate and check when file exists. + let stderr_path = TEST_DIRECTORY.join("errors.stderr"); + + if stderr_path.exists() { + let stderr = render_errors_to_stderr("input.ftml", &self.input, &errors); // see run() + // Run and check stderr + if let Some(expected_stderr) = &self.errors_stderr + && &stderr != expected_stderr + { + update!(write_text, stderr, "errors.stderr"); + } + } } } @@ -354,3 +393,33 @@ fn write_text(path: &Path, contents: &str) { file.write_all(b"\n") .expect("Unable to write final newline to file"); } + +/// Helper function for rendering errors to stderr. +fn render_errors_to_stderr( + source_name: &str, + source: &str, + errors: &[ParseError], +) -> String { + let mut files = SimpleFiles::new(); + + let file_id = files.add(source_name, source); + + let config = term::Config::default(); + + // no color because `.stderr` is raw text + let mut output = Buffer::no_color(); + + for error in errors { + let diagnostic = Diagnostic::error() + .with_message(error.kind().name()) + .with_labels(vec![ + Label::primary(file_id, error.span()).with_message(error.rule()), + ]); + + term::emit_to_write_style(&mut output, &config, &files, &diagnostic) + .expect("failed to emit diagnostic"); + } + + String::from_utf8(output.as_slice().to_vec()) + .expect("diagnostic output was not utf-8") +} diff --git a/test/color/fail/errors.stderr b/test/color/fail/errors.stderr new file mode 100644 index 000000000..a07f596f7 --- /dev/null +++ b/test/color/fail/errors.stderr @@ -0,0 +1,28 @@ +error: RuleFailed + ┌─ input.ftml:1:12 + │ +1 │ ##not color + │ ╭───────────^ +2 │ │ +3 │ │ ##|no spec## + │ ╰^ color + +error: NoRulesMatch + ┌─ input.ftml:1:1 + │ +1 │ ##not color + │ ^^ fallback + +error: EndOfInput + ┌─ input.ftml:5:14 + │ +5 │ ##only color| + │ ^ color + +error: NoRulesMatch + ┌─ input.ftml:5:1 + │ +5 │ ##only color| + │ ^^ fallback + + diff --git a/test/definition-list/fail/errors.stderr b/test/definition-list/fail/errors.stderr new file mode 100644 index 000000000..e69de29bb diff --git a/test/footnote/block-inside-fail/errors.stderr b/test/footnote/block-inside-fail/errors.stderr new file mode 100644 index 000000000..8b007d3aa --- /dev/null +++ b/test/footnote/block-inside-fail/errors.stderr @@ -0,0 +1,19 @@ +error: FootnotesNested + ┌─ input.ftml:1:35 + │ +1 │ Apple[[footnote]][[footnoteblock]][[/footnote]]Banana + │ ^^^ block-footnote-block + +error: NoRulesMatch + ┌─ input.ftml:1:18 + │ +1 │ Apple[[footnote]][[footnoteblock]][[/footnote]]Banana + │ ^^ fallback + +error: NoRulesMatch + ┌─ input.ftml:1:33 + │ +1 │ Apple[[footnote]][[footnoteblock]][[/footnote]]Banana + │ ^^ fallback + + diff --git a/test/footnote/revert/errors.stderr b/test/footnote/revert/errors.stderr new file mode 100644 index 000000000..a06f03fe7 --- /dev/null +++ b/test/footnote/revert/errors.stderr @@ -0,0 +1,55 @@ +error: EndOfInput + ┌─ input.ftml:9:31 + │ +9 │ 2[[footnote]]Beta[[/footnote]] + │ ^ block-div + +error: NoRulesMatch + ┌─ input.ftml:1:1 + │ +1 │ [[div]] + │ ^^ fallback + +error: NoRulesMatch + ┌─ input.ftml:1:6 + │ +1 │ [[div]] + │ ^^ fallback + +error: EndOfInput + ┌─ input.ftml:9:31 + │ +9 │ 2[[footnote]]Beta[[/footnote]] + │ ^ block-div + +error: NoRulesMatch + ┌─ input.ftml:2:1 + │ +2 │ [[div]] + │ ^^ fallback + +error: NoRulesMatch + ┌─ input.ftml:2:6 + │ +2 │ [[div]] + │ ^^ fallback + +error: EndOfInput + ┌─ input.ftml:9:31 + │ +9 │ 2[[footnote]]Beta[[/footnote]] + │ ^ block-div + +error: NoRulesMatch + ┌─ input.ftml:3:1 + │ +3 │ [[div]] + │ ^^ fallback + +error: NoRulesMatch + ┌─ input.ftml:3:6 + │ +3 │ [[div]] + │ ^^ fallback + +