diff --git a/parser/src/lib.rs b/parser/src/lib.rs index 4bdb03add..557ea7c6b 100644 --- a/parser/src/lib.rs +++ b/parser/src/lib.rs @@ -2,7 +2,9 @@ pub mod command; pub mod error; mod ignore_block; mod mentions; +mod text; mod token; pub use ignore_block::replace_all_outside_ignore_blocks; pub use mentions::get_mentions; +pub use text::strip_markdown; diff --git a/parser/src/text.rs b/parser/src/text.rs new file mode 100644 index 000000000..5bae3aaf8 --- /dev/null +++ b/parser/src/text.rs @@ -0,0 +1,83 @@ +use pulldown_cmark::{Event, Options, Parser, TagEnd}; + +pub fn strip_markdown(text: &str) -> String { + let mut buffer = String::new(); + + let mut parser = Parser::new_ext( + text, + Options::ENABLE_TABLES + | Options::ENABLE_GFM + | Options::ENABLE_MATH + | Options::ENABLE_STRIKETHROUGH + | Options::ENABLE_FOOTNOTES + | Options::ENABLE_HEADING_ATTRIBUTES, + ) + .into_iter(); + + while let Some(event) = parser.next() { + match event { + // Text and inline code are the content we want to keep + Event::Text(t) | Event::Code(t) => { + let stripped = t.replace("@", ""); // Strip mentions as well + buffer.push_str(&stripped); + } + // Add a newline when a block-level element ends to maintain spacing + Event::End(tag) => { + if is_block_tag(&tag) { + buffer.push('\n'); + } + } + _ => {} + } + } + + buffer.trim().to_string() +} + +fn is_block_tag(tag: &TagEnd) -> bool { + matches!( + tag, + TagEnd::Paragraph | TagEnd::Heading { .. } | TagEnd::Item | TagEnd::CodeBlock + ) +} + +#[test] +fn basic_formatting() { + let input = "This is **bold**, *italic* and ~~strikethrough~~ text."; + let expected = "This is bold, italic and strikethrough text."; + assert_eq!(strip_markdown(input), expected); +} + +#[test] +fn links_and_images() { + let input = "Check out [Google](https://google.com) and this ![alt text](image.png)."; + let expected = "Check out Google and this alt text."; + assert_eq!(strip_markdown(input), expected); +} + +#[test] +fn headers_and_lists() { + let input = "# Title\n- Item 1\n- Item 2"; + let expected = "Title\nItem 1\nItem 2"; + assert_eq!(strip_markdown(input), expected); +} + +#[test] +fn inline_code() { + let input = "Use the `fn main()` function."; + let expected = "Use the fn main() function."; + assert_eq!(strip_markdown(input), expected); +} + +#[test] +fn test_pings() { + let input = "let's ping @foo"; + let expected = "let's ping foo"; + assert_eq!(strip_markdown(input), expected); +} + +#[test] +fn empty_and_whitespace() { + assert_eq!(strip_markdown(""), ""); + assert_eq!(strip_markdown(" "), ""); +} diff --git a/src/zulip.rs b/src/zulip.rs index dea4533a9..4f243cf73 100644 --- a/src/zulip.rs +++ b/src/zulip.rs @@ -1658,14 +1658,7 @@ fn format_date(date: Option>) -> String { /// Truncates the given text to the specified length, adding ellipsis if needed. /// Also removes various special Markdown symbols from it. fn truncate_and_normalize(text: &str, max_len: usize) -> String { - let normalized: String = text - .split_whitespace() - .collect::>() - .join(" ") - // Avoid things that could interact with the rest of the Markdown message. - // And avoid and @ pings (in case the text is outputted in a public Zulip stream). - .replace("`", "") - .replace("@", ""); + let normalized = parser::strip_markdown(text); if normalized.len() <= max_len { normalized