diff --git a/Cargo.lock b/Cargo.lock index 485c60b..888c801 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -103,6 +103,7 @@ name = "anomalyx-normalize" version = "0.1.0" dependencies = [ "anomalyx-core", + "calamine", "chrono", "csv", "etherparse", @@ -111,6 +112,7 @@ dependencies = [ "polars", "proptest", "roxmltree", + "rust_xlsxwriter", "serde", "serde_json", "serde_yaml", @@ -450,6 +452,23 @@ dependencies = [ "serde", ] +[[package]] +name = "calamine" +version = "0.35.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8822fe6253ca47aa5ad9a3be09f6fe7cd20c6a74e41b0aa42e8f4e3d523508df" +dependencies = [ + "atoi_simd", + "byteorder", + "codepage", + "encoding_rs", + "fast-float2", + "log", + "quick-xml", + "serde", + "zip", +] + [[package]] name = "camino" version = "1.2.2" @@ -583,6 +602,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" +[[package]] +name = "codepage" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48f68d061bc2828ae826206326e61251aca94c1e4a5305cf52d9138639c918b4" +dependencies = [ + "encoding_rs", +] + [[package]] name = "colorchoice" version = "1.0.5" @@ -884,6 +912,15 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a246d82be1c9d791c5dfde9a2bd045fc3cbba3fa2b11ad558f27d01712f00569" +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -2749,6 +2786,7 @@ version = "0.39.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cdcc8dd4e2f670d309a5f0e83fe36dfdc05af317008fea29144da1a2ac858e5e" dependencies = [ + "encoding_rs", "memchr", "serde", ] @@ -3175,6 +3213,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "rust_xlsxwriter" +version = "0.95.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f281b687352597d29efaad39701d1167d5c48aa76fb973e392bc13e9d44e7f36" +dependencies = [ + "zip", +] + [[package]] name = "rustc-hash" version = "2.1.2" @@ -4073,6 +4120,12 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "typed-path" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e" + [[package]] name = "typenum" version = "1.20.1" @@ -4866,6 +4919,20 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "zip" +version = "7.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c42e33efc22a0650c311c2ef19115ce232583abbe80850bc8b66509ebef02de0" +dependencies = [ + "crc32fast", + "flate2", + "indexmap", + "memchr", + "typed-path", + "zopfli", +] + [[package]] name = "zlib-rs" version = "0.6.3" @@ -4878,6 +4945,18 @@ version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" +[[package]] +name = "zopfli" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] + [[package]] name = "zstd" version = "0.13.3" diff --git a/crates/ax-normalize/Cargo.toml b/crates/ax-normalize/Cargo.toml index 988cbba..39950d5 100644 --- a/crates/ax-normalize/Cargo.toml +++ b/crates/ax-normalize/Cargo.toml @@ -51,12 +51,20 @@ evtx = { version = "0.11", optional = true } # feature, so the text-only build stays lean. pcap-parser = { version = "0.17", optional = true } etherparse = { version = "0.20", optional = true } +# Excel (.xlsx/.xls/.xlsb) and OpenDocument (.ods) spreadsheets. calamine reads +# all of them (pure Rust) and we lower the first sheet to a RecordSet. Behind the +# default-on `xlsx` feature (binary format), so the text-only build stays lean. +calamine = { version = "0.35", optional = true } [features] -default = ["polars", "evtx", "pcap"] +default = ["polars", "evtx", "pcap", "xlsx"] polars = ["dep:polars"] evtx = ["dep:evtx"] pcap = ["dep:pcap-parser", "dep:etherparse"] +xlsx = ["dep:calamine"] [dev-dependencies] proptest = { workspace = true } +# Writes a tiny .xlsx in-memory for the xlsx parser's roundtrip test (no +# committed binary fixture). +rust_xlsxwriter = "0.95" diff --git a/crates/ax-normalize/src/parser.rs b/crates/ax-normalize/src/parser.rs index e147e19..6476d68 100644 --- a/crates/ax-normalize/src/parser.rs +++ b/crates/ax-normalize/src/parser.rs @@ -218,6 +218,8 @@ mod tests { expected.push("evtx"); #[cfg(feature = "pcap")] expected.push("pcap"); + #[cfg(feature = "xlsx")] + expected.push("xlsx"); expected.extend([ "otlp", "cloudtrail", diff --git a/crates/ax-normalize/src/parsers/mod.rs b/crates/ax-normalize/src/parsers/mod.rs index 45b88a3..133350e 100644 --- a/crates/ax-normalize/src/parsers/mod.rs +++ b/crates/ax-normalize/src/parsers/mod.rs @@ -26,6 +26,8 @@ pub mod prometheus; pub mod syslog; pub mod toml; pub mod vpcflow; +#[cfg(feature = "xlsx")] +pub mod xlsx; pub mod xml; pub mod yaml; pub mod zeek; @@ -55,6 +57,8 @@ pub use prometheus::PrometheusParser; pub use syslog::SyslogParser; pub use toml::{IniParser, TomlParser}; pub use vpcflow::VpcFlowParser; +#[cfg(feature = "xlsx")] +pub use xlsx::XlsxParser; pub use xml::XmlParser; pub use yaml::YamlParser; pub use zeek::ZeekParser; @@ -74,6 +78,8 @@ pub fn default_registry() -> ParserRegistry { r.register(Box::new(EvtxParser)); #[cfg(feature = "pcap")] r.register(Box::new(PcapParser)); + #[cfg(feature = "xlsx")] + r.register(Box::new(XlsxParser)); // OTLP before NDJSON: a compact single-object OTLP doc must win the // `resourceSpans` signature before any JSON-line heuristic sees it. r.register(Box::new(OtlpParser)); diff --git a/crates/ax-normalize/src/parsers/xlsx.rs b/crates/ax-normalize/src/parsers/xlsx.rs new file mode 100644 index 0000000..2de0a97 --- /dev/null +++ b/crates/ax-normalize/src/parsers/xlsx.rs @@ -0,0 +1,232 @@ +//! Excel / OpenDocument spreadsheet parser — the universal business handoff. +//! +//! The first worksheet of a workbook (`.xlsx`/`.xls`/`.xlsb`/`.ods`) becomes a +//! RecordSet: the first row is the header (column names), each subsequent row is +//! a record, and every cell maps to the closed [`Value`] set — so all detectors +//! apply with no special-casing. A date/time cell keeps its Excel serial number +//! (numeric, deterministic); blanks and error cells are `Null` (honest absence). +//! +//! Reading is delegated to `calamine` (pure Rust, all four formats). Detected by +//! the ZIP magic plus an `xl/` or OpenDocument-spreadsheet marker; extensions +//! `.xlsx`/`.xls`/`.xlsb`/`.ods`. Behind the default-on `xlsx` feature. + +use crate::parser::{Confidence, FormatParser, MAGIC}; +use crate::table::TableBuilder; +use ax_core::{AxError, Column, Value}; +use calamine::{open_workbook_auto_from_rs, Data, Reader}; +use std::collections::BTreeMap; +use std::io::Cursor; + +#[derive(Debug, Default, Clone)] +pub struct XlsxParser; + +/// True if `needle` appears anywhere in `haystack`. +fn contains_seq(haystack: &[u8], needle: &[u8]) -> bool { + haystack.windows(needle.len()).any(|w| w == needle) +} + +/// Maps a calamine cell to the closed [`Value`] set. +fn data_to_value(cell: &Data) -> Value { + match cell { + Data::Int(i) => Value::Int(*i), + Data::Float(f) => { + if f.is_finite() { + Value::Float(*f) + } else { + Value::Null + } + } + Data::String(s) => Value::Str(s.clone()), + Data::Bool(b) => Value::Bool(*b), + // Keep the Excel serial number — numeric and deterministic. + Data::DateTime(dt) => Value::Float(dt.as_f64()), + Data::DateTimeIso(s) | Data::DurationIso(s) => Value::Str(s.clone()), + Data::Error(_) | Data::Empty => Value::Null, + } +} + +/// The column name for a header cell: a non-empty string cell verbatim, else a +/// positional `col{index}`. +fn header_name(cell: &Data, index: usize) -> String { + match cell { + Data::String(s) if !s.trim().is_empty() => s.trim().to_string(), + _ => format!("col{index}"), + } +} + +impl XlsxParser { + fn err(&self, msg: impl std::fmt::Display) -> AxError { + AxError::Parse { + format: self.id().to_string(), + message: msg.to_string(), + } + } +} + +impl FormatParser for XlsxParser { + fn id(&self) -> &'static str { + "xlsx" + } + fn extensions(&self) -> &'static [&'static str] { + &["xlsx", "xls", "xlsb", "ods"] + } + fn sniff(&self, bytes: &[u8]) -> Option { + if !bytes.starts_with(b"PK\x03\x04") { + return None; // not a ZIP (xls's OLE2 magic is handled by extension) + } + // A ZIP that is specifically a spreadsheet: xlsx/xlsb have an `xl/` part, + // ODS declares the OpenDocument-spreadsheet mimetype. (A docx/jar/plain + // zip has neither, so it is not claimed.) + (contains_seq(bytes, b"xl/") || contains_seq(bytes, b"opendocument.spreadsheet")) + .then_some(MAGIC) + } + fn parse(&self, _source: &str, bytes: &[u8]) -> Result, AxError> { + let mut workbook = + open_workbook_auto_from_rs(Cursor::new(bytes.to_vec())).map_err(|e| self.err(e))?; + let sheet = workbook + .sheet_names() + .first() + .cloned() + .ok_or_else(|| self.err("workbook has no sheets"))?; + let range = workbook.worksheet_range(&sheet).map_err(|e| self.err(e))?; + + let mut rows = range.rows(); + let Some(header) = rows.next() else { + return Ok(Vec::new()); // empty sheet → no columns + }; + let names: Vec = header + .iter() + .enumerate() + .map(|(i, cell)| header_name(cell, i)) + .collect(); + + let mut builder = TableBuilder::new(); + for row in rows { + let mut record: BTreeMap = BTreeMap::new(); + for (name, cell) in names.iter().zip(row) { + record.insert(name.clone(), data_to_value(cell)); + } + builder.push_row(record); + } + Ok(builder.finish()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use calamine::{CellErrorType, ExcelDateTime, ExcelDateTimeType}; + use rust_xlsxwriter::Workbook; + + /// Writes a tiny .xlsx in-memory: header + two records (string / number / bool). + fn build_xlsx() -> Vec { + let mut wb = Workbook::new(); + let ws = wb.add_worksheet(); + for (c, h) in ["name", "score", "active"].iter().enumerate() { + ws.write(0, c as u16, *h).unwrap(); + } + ws.write(1, 0, "alice").unwrap(); + ws.write(1, 1, 95).unwrap(); + ws.write(1, 2, true).unwrap(); + ws.write(2, 0, "bob").unwrap(); + ws.write(2, 1, 42.5).unwrap(); + ws.write(2, 2, false).unwrap(); + wb.save_to_buffer().unwrap() + } + + fn col<'a>(cols: &'a [Column], name: &str) -> &'a Column { + cols.iter() + .find(|c| c.name == name) + .unwrap_or_else(|| panic!("missing column {name}")) + } + + #[test] + fn roundtrip_first_sheet_to_records() { + let bytes = build_xlsx(); + let cols = XlsxParser.parse("book.xlsx", &bytes).unwrap(); + assert_eq!( + col(&cols, "name").cells, + vec![Value::Str("alice".into()), Value::Str("bob".into())] + ); + let score = col(&cols, "score"); + assert_eq!(score.numeric(), vec![95.0, 42.5]); + assert_eq!( + col(&cols, "active").cells, + vec![Value::Bool(true), Value::Bool(false)] + ); + } + + #[test] + fn data_to_value_units() { + assert_eq!(data_to_value(&Data::Int(7)), Value::Int(7)); + assert_eq!(data_to_value(&Data::Float(1.5)), Value::Float(1.5)); + assert_eq!(data_to_value(&Data::Float(f64::NAN)), Value::Null); // non-finite → null + assert_eq!( + data_to_value(&Data::String("x".into())), + Value::Str("x".into()) + ); + assert_eq!(data_to_value(&Data::Bool(true)), Value::Bool(true)); + assert_eq!(data_to_value(&Data::Empty), Value::Null); + assert_eq!( + data_to_value(&Data::Error(CellErrorType::Div0)), + Value::Null + ); + assert_eq!( + data_to_value(&Data::DateTimeIso("2021-01-01".into())), + Value::Str("2021-01-01".into()) + ); + // A date cell keeps its Excel serial number. + let dt = ExcelDateTime::new(44197.0, ExcelDateTimeType::DateTime, false); + assert_eq!(data_to_value(&Data::DateTime(dt)), Value::Float(44197.0)); + } + + #[test] + fn header_name_units() { + assert_eq!(header_name(&Data::String("score".into()), 1), "score"); + assert_eq!(header_name(&Data::String(" ".into()), 1), "col1"); // blank → positional + assert_eq!(header_name(&Data::Empty, 2), "col2"); + assert_eq!(header_name(&Data::Int(5), 0), "col0"); // non-string → positional + } + + #[test] + fn malformed_input_errors() { + assert!(matches!( + XlsxParser.parse("book.xlsx", b"not a spreadsheet"), + Err(AxError::Parse { .. }) + )); + // ZIP magic but not a valid workbook. + assert!(matches!( + XlsxParser.parse("book.xlsx", b"PK\x03\x04 garbage"), + Err(AxError::Parse { .. }) + )); + } + + #[test] + fn sniff_keys_on_zip_plus_spreadsheet_marker() { + assert_eq!(XlsxParser.sniff(&build_xlsx()), Some(MAGIC)); + // A ZIP that is not a spreadsheet (e.g. a .docx has `word/`, not `xl/`). + assert_eq!(XlsxParser.sniff(b"PK\x03\x04....word/document.xml"), None); + assert_eq!(XlsxParser.sniff(b"not a zip"), None); + assert_eq!(XlsxParser.sniff(b"PK"), None); // too short for the magic + } + + #[test] + fn contains_seq_units() { + assert!(contains_seq(b"hello xl/ world", b"xl/")); + assert!(!contains_seq(b"hello world", b"xl/")); + assert!(!contains_seq(b"ab", b"abc")); // needle longer than haystack + } + + #[test] + fn claims_spreadsheet_extensions() { + assert_eq!(XlsxParser.extensions(), &["xlsx", "xls", "xlsb", "ods"]); + } + + #[test] + fn resolves_by_extension_and_magic() { + let reg = crate::parser::ParserRegistry::default(); + assert_eq!(reg.resolve("book.xlsx", b"zz").unwrap().id(), "xlsx"); + assert_eq!(reg.resolve("sheet.ods", b"zz").unwrap().id(), "xlsx"); + assert_eq!(reg.resolve("-", &build_xlsx()).unwrap().id(), "xlsx"); + } +}