diff --git a/.cargo/mutants.toml b/.cargo/mutants.toml index 39e7596..112a069 100644 --- a/.cargo/mutants.toml +++ b/.cargo/mutants.toml @@ -84,6 +84,11 @@ exclude_re = [ # caught as a timeout by the `a=1 b=2` test.) 'replace \+ with \* in parse_cef_extension', + # `avro_to_value`'s explicit `AvroValue::Null => Value::Null` arm and its + # catch-all `_ => Value::Null` (for complex/unmodelled types) return the same + # value, so deleting the Null arm is behaviorally identical. Equivalent. + "delete match arm AvroValue::Null in avro_to_value", + # `naive_one_pass_variance` is a deliberately *inaccurate* reference baseline # (the textbook one-pass formula), kept only to demonstrate the precision gap # against our compensated reductions. Its sole contract — "less accurate than diff --git a/Cargo.lock b/Cargo.lock index f15a652..446cb21 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -15,6 +15,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ "cfg-if", + "const-random", "getrandom 0.3.4", "once_cell", "version_check", @@ -103,11 +104,15 @@ name = "anomalyx-normalize" version = "0.1.0" dependencies = [ "anomalyx-core", + "apache-avro", + "arrow", + "bytes", "calamine", "chrono", "csv", "etherparse", "evtx", + "orc-rust", "pcap-parser", "polars", "proptest", @@ -188,6 +193,30 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "apache-avro" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36fa98bc79671c7981272d91a8753a928ff6a1cd8e4f20a44c45bd5d313840bf" +dependencies = [ + "bigdecimal", + "bon", + "digest", + "log", + "miniz_oxide", + "num-bigint", + "quad-rand", + "rand 0.9.4", + "regex-lite", + "serde", + "serde_bytes", + "serde_json", + "strum", + "strum_macros", + "thiserror 2.0.18", + "uuid", +] + [[package]] name = "approx" version = "0.5.1" @@ -234,6 +263,226 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" +[[package]] +name = "arrow" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "378530e55cd479eda3c14eb345310799717e6f76d0c332041e8487022166b471" +dependencies = [ + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-csv", + "arrow-data", + "arrow-ipc", + "arrow-json", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "arrow-string", +] + +[[package]] +name = "arrow-arith" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0ab212d2c1886e802f51c5212d78ebbcbb0bec980fff9dadc1eb8d45cd0b738" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "num-traits", +] + +[[package]] +name = "arrow-array" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfd33d3e92f207444098c75b42de99d329562be0cf686b307b097cc52b4e999e" +dependencies = [ + "ahash", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "chrono-tz", + "half", + "hashbrown 0.17.1", + "num-complex", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-buffer" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c6cd424c2693bcdbc150d843dc9d4d137dd2de4782ce6df491ad11a3a0416c0" +dependencies = [ + "bytes", + "half", + "num-bigint", + "num-traits", +] + +[[package]] +name = "arrow-cast" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c5aefb56a2c02e9e2b30746241058b85f8983f0fcff2ba0c6d09006e1cded7f" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-ord", + "arrow-schema", + "arrow-select", + "atoi", + "base64", + "chrono", + "comfy-table", + "half", + "lexical-core", + "num-traits", + "ryu", +] + +[[package]] +name = "arrow-csv" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e94e8cf7e517657a52b91ea1263acf38c4ca62a84655d72458a3359b12ab97de" +dependencies = [ + "arrow-array", + "arrow-cast", + "arrow-schema", + "chrono", + "csv", + "csv-core", + "regex", +] + +[[package]] +name = "arrow-data" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c88210023a2bfee1896af366309a3028fc3bcbd6515fa29a7990ee1baa08ee0" +dependencies = [ + "arrow-buffer", + "arrow-schema", + "half", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-ipc" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "238438f0834483703d88896db6fe5a7138b2230debc31b34c0336c2996e3c64f" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "flatbuffers", + "lz4_flex 0.13.1", + "zstd", +] + +[[package]] +name = "arrow-json" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "205ca2119e6d679d5c133c6f30e68f027738d95ed948cf77677ea69c7800036b" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-ord", + "arrow-schema", + "arrow-select", + "chrono", + "half", + "indexmap", + "itoa", + "lexical-core", + "memchr", + "num-traits", + "ryu", + "serde_core", + "serde_json", + "simdutf8", +] + +[[package]] +name = "arrow-ord" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bffd8fd2579286a5d63bac898159873e5094a79009940bcb42bbfce4f19f1d0" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", +] + +[[package]] +name = "arrow-row" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bab5994731204603c73ba69267616c50f80780774c6bb0476f1f830625115e0c" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "half", +] + +[[package]] +name = "arrow-schema" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f633dbfdf39c039ada1bf9e34c694816eb71fbb7dc78f613993b7245e078a1ed" + +[[package]] +name = "arrow-select" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cd065c54172ac787cf3f2f8d4107e0d3fdc26edba76fdf4f4cc170258942222" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "num-traits", +] + +[[package]] +name = "arrow-string" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29dd7cda3ab9692f43a2e4acc444d760cc17b12bb6d8232ddf64e9bab7c06b42" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "memchr", + "num-traits", + "regex", + "regex-syntax", +] + [[package]] name = "async-channel" version = "2.5.0" @@ -279,6 +528,15 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + [[package]] name = "atoi_simd" version = "0.17.0" @@ -306,6 +564,20 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "bigdecimal" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d6867f1565b3aad85681f1015055b087fcfd840d6aeee6eee7f2da317603695" +dependencies = [ + "autocfg", + "libm", + "num-bigint", + "num-integer", + "num-traits", + "serde", +] + [[package]] name = "bincode" version = "2.0.1" @@ -379,6 +651,31 @@ dependencies = [ "generic-array", ] +[[package]] +name = "bon" +version = "3.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f47dbe92550676ee653353c310dfb9cf6ba17ee70396e1f7cf0a2020ad49b2fe" +dependencies = [ + "bon-macros", + "rustversion", +] + +[[package]] +name = "bon-macros" +version = "3.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "519bd3116aeeb42d5372c29d982d16d0170d3d4a5ed85fc7dd91642ffff3c67c" +dependencies = [ + "darling", + "ident_case", + "prettyplease", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.117", +] + [[package]] name = "boxcar" version = "0.2.14" @@ -618,6 +915,16 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "comfy-table" +version = "7.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "958c5d6ecf1f214b4c2bbbbf6ab9523a864bd136dcf71a7e8904799acfe1ad47" +dependencies = [ + "unicode-segmentation", + "unicode-width", +] + [[package]] name = "compact_str" version = "0.9.1" @@ -654,6 +961,26 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom 0.2.17", + "once_cell", + "tiny-keccak", +] + [[package]] name = "constant_time_eq" version = "0.4.2" @@ -783,6 +1110,40 @@ dependencies = [ "memchr", ] +[[package]] +name = "darling" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.117", +] + +[[package]] +name = "darling_macro" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" +dependencies = [ + "darling_core", + "quote", + "syn 2.0.117", +] + [[package]] name = "debug_unsafe" version = "0.1.4" @@ -1058,6 +1419,16 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "flatbuffers" +version = "25.12.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" +dependencies = [ + "bitflags 2.11.1", + "rustc_version", +] + [[package]] name = "flate2" version = "1.1.9" @@ -1573,6 +1944,12 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "idna" version = "1.1.0" @@ -1711,6 +2088,63 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" +[[package]] +name = "lexical-core" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d8d125a277f807e55a77304455eb7b1cb52f2b18c143b60e766c120bd64a594" +dependencies = [ + "lexical-parse-float", + "lexical-parse-integer", + "lexical-util", + "lexical-write-float", + "lexical-write-integer", +] + +[[package]] +name = "lexical-parse-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52a9f232fbd6f550bc0137dcb5f99ab674071ac2d690ac69704593cb4abbea56" +dependencies = [ + "lexical-parse-integer", + "lexical-util", +] + +[[package]] +name = "lexical-parse-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a7a039f8fb9c19c996cd7b2fcce303c1b2874fe1aca544edc85c4a5f8489b34" +dependencies = [ + "lexical-util", +] + +[[package]] +name = "lexical-util" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2604dd126bb14f13fb5d1bd6a66155079cb9fa655b37f875b3a742c705dbed17" + +[[package]] +name = "lexical-write-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50c438c87c013188d415fbabbb1dceb44249ab81664efbd31b14ae55dabb6361" +dependencies = [ + "lexical-util", + "lexical-write-integer", +] + +[[package]] +name = "lexical-write-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "409851a618475d2d5796377cad353802345cba92c867d9fbcde9cf4eac4e14df" +dependencies = [ + "lexical-util", +] + [[package]] name = "libc" version = "0.2.186" @@ -1786,6 +2220,34 @@ dependencies = [ "libc", ] +[[package]] +name = "lz4_flex" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "373f5eceeeab7925e0c1098212f2fbc4d416adec9d35051a6ab251e824c1854a" +dependencies = [ + "twox-hash", +] + +[[package]] +name = "lz4_flex" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e" +dependencies = [ + "twox-hash", +] + +[[package]] +name = "lzokay-native" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "792ba667add2798c6c3e988e630f4eb921b5cbc735044825b7111ef1582c8730" +dependencies = [ + "byteorder", + "thiserror 1.0.69", +] + [[package]] name = "matrixmultiply" version = "0.3.10" @@ -1887,6 +2349,20 @@ dependencies = [ "chrono", ] +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + [[package]] name = "num-bigint" version = "0.4.6" @@ -1895,6 +2371,7 @@ checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" dependencies = [ "num-integer", "num-traits", + "serde", ] [[package]] @@ -1943,6 +2420,17 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + [[package]] name = "num-rational" version = "0.4.2" @@ -2037,6 +2525,33 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" +[[package]] +name = "orc-rust" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32b9867c4e7343218682ba11aceae1310c80735ec2beaf6124a0f8f848dad197" +dependencies = [ + "arrow", + "async-trait", + "bytemuck", + "bytes", + "chrono", + "chrono-tz", + "fallible-streaming-iterator", + "flate2", + "futures", + "futures-util", + "log", + "lz4_flex 0.11.6", + "lzokay-native", + "num", + "prost", + "snafu", + "snap", + "tokio", + "zstd", +] + [[package]] name = "parking" version = "2.2.1" @@ -2760,6 +3275,29 @@ dependencies = [ "unarray", ] +[[package]] +name = "prost" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-derive" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "psm" version = "0.1.31" @@ -2801,6 +3339,12 @@ dependencies = [ "unicase", ] +[[package]] +name = "quad-rand" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a651516ddc9168ebd67b24afd085a718be02f8858fe406591b013d101ce2f40" + [[package]] name = "quick-error" version = "1.2.3" @@ -3115,6 +3659,12 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "regex-lite" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" + [[package]] name = "regex-syntax" version = "0.8.10" @@ -3280,6 +3830,15 @@ version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + [[package]] name = "rusticata-macros" version = "5.0.0" @@ -3469,6 +4028,16 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde_bytes" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" +dependencies = [ + "serde", + "serde_core", +] + [[package]] name = "serde_core" version = "1.0.228" @@ -3669,6 +4238,27 @@ version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +[[package]] +name = "snafu" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e84b3f4eacbf3a1ce05eac6763b4d629d60cbc94d632e4092c54ade71f1e1a2" +dependencies = [ + "snafu-derive", +] + +[[package]] +name = "snafu-derive" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "snap" version = "1.1.1" @@ -3822,6 +4412,12 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "strum" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" + [[package]] name = "strum_macros" version = "0.27.2" @@ -3987,6 +4583,15 @@ dependencies = [ "time-core", ] +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + [[package]] name = "tinystr" version = "0.8.3" @@ -4184,6 +4789,12 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "twox-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" + [[package]] name = "typed-path" version = "0.12.3" diff --git a/crates/ax-normalize/Cargo.toml b/crates/ax-normalize/Cargo.toml index 20eb517..388d771 100644 --- a/crates/ax-normalize/Cargo.toml +++ b/crates/ax-normalize/Cargo.toml @@ -62,14 +62,23 @@ rusqlite = { version = "0.40", features = [ "bundled", "serialize", ], optional = true } +# Avro and ORC, the data-lake siblings of Parquet. Avro via apache-avro (records +# flattened to columns); ORC via orc-rust into Arrow batches lowered to columns. +# Behind the default-on `datalake` feature (binary formats). +apache-avro = { version = "0.21", optional = true } +orc-rust = { version = "0.8", optional = true } +# Pinned to the arrow major that orc-rust depends on (so the RecordBatch types unify). +arrow = { version = "58", optional = true, default-features = false } +bytes = { version = "1", optional = true } [features] -default = ["polars", "evtx", "pcap", "xlsx", "sqlite"] +default = ["polars", "evtx", "pcap", "xlsx", "sqlite", "datalake"] polars = ["dep:polars"] evtx = ["dep:evtx"] pcap = ["dep:pcap-parser", "dep:etherparse"] xlsx = ["dep:calamine"] sqlite = ["dep:rusqlite"] +datalake = ["dep:apache-avro", "dep:orc-rust", "dep:arrow", "dep:bytes"] [dev-dependencies] proptest = { workspace = true } diff --git a/crates/ax-normalize/src/parser.rs b/crates/ax-normalize/src/parser.rs index ba7bae5..fdc9fbb 100644 --- a/crates/ax-normalize/src/parser.rs +++ b/crates/ax-normalize/src/parser.rs @@ -222,6 +222,11 @@ mod tests { expected.push("xlsx"); #[cfg(feature = "sqlite")] expected.push("sqlite"); + #[cfg(feature = "datalake")] + { + expected.push("avro"); + expected.push("orc"); + } expected.extend([ "otlp", "cloudtrail", diff --git a/crates/ax-normalize/src/parsers/avro.rs b/crates/ax-normalize/src/parsers/avro.rs new file mode 100644 index 0000000..1224185 --- /dev/null +++ b/crates/ax-normalize/src/parsers/avro.rs @@ -0,0 +1,356 @@ +//! Avro and ORC parsers — the data-lake siblings of Parquet. +//! +//! Both lower to the same engine-independent [`Column`]s as the Parquet/Arrow +//! parsers, so no library type escapes the contract. +//! +//! - **Avro** (`apache-avro`): each record in the object-container file is a row; +//! record fields become typed columns. Bytes/fixed → hex `Str`; dates/times → +//! their integer value; nested records/arrays/maps and decimals are recorded as +//! `Null` (honest absence — v1 lowers only flat scalar columns). +//! - **ORC** (`orc-rust` → Arrow): the file is read into Arrow batches; each cell +//! is rendered and type-inferred into the closed [`Value`] set. +//! +//! Detected by their binary magic (`Obj\x01` / `ORC`); extensions `.avro` / +//! `.orc`. Behind the default-on `datalake` feature. + +use crate::infer; +use crate::parser::{Confidence, FormatParser, MAGIC}; +use crate::table::TableBuilder; +use ax_core::{AxError, Column, Value}; +use std::collections::BTreeMap; + +/// A finite float becomes `Float`; NaN/±∞ become `Null` (honest absence). +fn finite_float(f: f64) -> Value { + if f.is_finite() { + Value::Float(f) + } else { + Value::Null + } +} + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} + +fn parse_err(id: &str, e: impl std::fmt::Display) -> AxError { + AxError::Parse { + format: id.to_string(), + message: e.to_string(), + } +} + +// ----------------------------------------------------------------- Avro ------- + +use apache_avro::types::Value as AvroValue; + +#[derive(Debug, Default, Clone)] +pub struct AvroParser; + +/// Maps an Avro value to the closed [`Value`] set. +fn avro_to_value(value: &AvroValue) -> Value { + match value { + AvroValue::Null => Value::Null, + AvroValue::Boolean(b) => Value::Bool(*b), + AvroValue::Int(i) => Value::Int(i64::from(*i)), + AvroValue::Long(i) => Value::Int(*i), + AvroValue::Float(f) => finite_float(f64::from(*f)), + AvroValue::Double(f) => finite_float(*f), + AvroValue::String(s) => Value::Str(s.clone()), + AvroValue::Enum(_, s) => Value::Str(s.clone()), + AvroValue::Bytes(b) | AvroValue::Fixed(_, b) => Value::Str(hex(b)), + AvroValue::Union(_, inner) => avro_to_value(inner), + AvroValue::Date(d) => Value::Int(i64::from(*d)), + AvroValue::TimeMillis(t) => Value::Int(i64::from(*t)), + AvroValue::TimeMicros(t) + | AvroValue::TimestampMillis(t) + | AvroValue::TimestampMicros(t) + | AvroValue::TimestampNanos(t) + | AvroValue::LocalTimestampMillis(t) + | AvroValue::LocalTimestampMicros(t) + | AvroValue::LocalTimestampNanos(t) => Value::Int(*t), + // Records, arrays, maps, decimals, uuids, durations are not flat scalars. + _ => Value::Null, + } +} + +impl FormatParser for AvroParser { + fn id(&self) -> &'static str { + "avro" + } + fn extensions(&self) -> &'static [&'static str] { + &["avro"] + } + fn sniff(&self, bytes: &[u8]) -> Option { + // Avro object-container file magic. + bytes.starts_with(b"Obj\x01").then_some(MAGIC) + } + fn parse(&self, _source: &str, bytes: &[u8]) -> Result, AxError> { + let reader = apache_avro::Reader::new(bytes).map_err(|e| parse_err(self.id(), e))?; + let mut builder = TableBuilder::new(); + for record in reader { + let value = record.map_err(|e| parse_err(self.id(), e))?; + let mut row: BTreeMap = BTreeMap::new(); + match value { + AvroValue::Record(fields) => { + for (name, field) in fields { + row.insert(name, avro_to_value(&field)); + } + } + other => { + row.insert("value".to_string(), avro_to_value(&other)); + } + } + builder.push_row(row); + } + Ok(builder.finish()) + } +} + +// ------------------------------------------------------------------ ORC ------- + +#[derive(Debug, Default, Clone)] +pub struct OrcParser; + +/// Renders one Arrow cell into the closed [`Value`] set (null-aware; the string +/// rendering is type-inferred so numbers/bools become typed columns). +fn orc_cell(array: &dyn arrow::array::Array, row: usize) -> Value { + if array.is_null(row) { + return Value::Null; + } + match arrow::util::display::array_value_to_string(array, row) { + Ok(s) => infer::infer_scalar(&s), + Err(_) => Value::Null, + } +} + +impl FormatParser for OrcParser { + fn id(&self) -> &'static str { + "orc" + } + fn extensions(&self) -> &'static [&'static str] { + &["orc"] + } + fn sniff(&self, bytes: &[u8]) -> Option { + // ORC files begin with the 3-byte magic "ORC". + bytes.starts_with(b"ORC").then_some(MAGIC) + } + fn parse(&self, _source: &str, bytes: &[u8]) -> Result, AxError> { + let cursor = bytes::Bytes::from(bytes.to_vec()); + let reader = orc_rust::ArrowReaderBuilder::try_new(cursor) + .map_err(|e| parse_err(self.id(), e))? + .build(); + let mut builder = TableBuilder::new(); + for batch in reader { + let batch = batch.map_err(|e| parse_err(self.id(), e))?; + let schema = batch.schema(); + for row in 0..batch.num_rows() { + let mut record: BTreeMap = BTreeMap::new(); + for (i, field) in schema.fields().iter().enumerate() { + record.insert( + field.name().clone(), + orc_cell(batch.column(i).as_ref(), row), + ); + } + builder.push_row(record); + } + } + Ok(builder.finish()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use ax_core::ColType; + + fn col<'a>(cols: &'a [Column], name: &str) -> &'a Column { + cols.iter() + .find(|c| c.name == name) + .unwrap_or_else(|| panic!("missing column {name}")) + } + + // ----------------------------------------------------------- Avro ------- + + /// Writes a tiny Avro object-container file in-memory (no committed fixture). + fn build_avro() -> Vec { + use apache_avro::{types::Record, Schema, Writer}; + let schema = Schema::parse_str( + r#"{"type":"record","name":"r","fields":[ + {"name":"id","type":"long"}, + {"name":"host","type":"string"}, + {"name":"score","type":"double"}, + {"name":"ok","type":"boolean"}]}"#, + ) + .unwrap(); + let mut writer = Writer::new(&schema, Vec::new()); + for (id, host, score, ok) in [(1i64, "a", 9.5f64, true), (2, "b", 3.25, false)] { + let mut rec = Record::new(writer.schema()).unwrap(); + rec.put("id", id); + rec.put("host", host); + rec.put("score", score); + rec.put("ok", ok); + writer.append(rec).unwrap(); + } + writer.into_inner().unwrap() + } + + #[test] + fn avro_records_become_typed_rows() { + let cols = AvroParser.parse("data.avro", &build_avro()).unwrap(); + assert_eq!(col(&cols, "id").ty, ColType::Int); + assert_eq!(col(&cols, "id").cells, vec![Value::Int(1), Value::Int(2)]); + assert_eq!( + col(&cols, "host").cells, + vec![Value::Str("a".into()), Value::Str("b".into())] + ); + assert_eq!(col(&cols, "score").numeric(), vec![9.5, 3.25]); + assert_eq!( + col(&cols, "ok").cells, + vec![Value::Bool(true), Value::Bool(false)] + ); + } + + #[test] + fn avro_to_value_units() { + assert_eq!(avro_to_value(&AvroValue::Null), Value::Null); + assert_eq!(avro_to_value(&AvroValue::Boolean(true)), Value::Bool(true)); + assert_eq!(avro_to_value(&AvroValue::Int(5)), Value::Int(5)); + assert_eq!(avro_to_value(&AvroValue::Long(9)), Value::Int(9)); + assert_eq!(avro_to_value(&AvroValue::Float(1.5)), Value::Float(1.5)); + assert_eq!(avro_to_value(&AvroValue::Double(2.5)), Value::Float(2.5)); + assert_eq!(avro_to_value(&AvroValue::Double(f64::NAN)), Value::Null); + assert_eq!( + avro_to_value(&AvroValue::String("x".into())), + Value::Str("x".into()) + ); + assert_eq!( + avro_to_value(&AvroValue::Enum(0, "GET".into())), + Value::Str("GET".into()) + ); + assert_eq!( + avro_to_value(&AvroValue::Bytes(vec![0x00, 0xab])), + Value::Str("00ab".into()) + ); + // A union unwraps to its held value (the common nullable shape). + assert_eq!( + avro_to_value(&AvroValue::Union(1, Box::new(AvroValue::Long(7)))), + Value::Int(7) + ); + assert_eq!(avro_to_value(&AvroValue::Date(19000)), Value::Int(19000)); + assert_eq!(avro_to_value(&AvroValue::TimeMillis(500)), Value::Int(500)); + assert_eq!( + avro_to_value(&AvroValue::TimestampMillis(1234)), + Value::Int(1234) + ); + // Complex/non-scalar values are recorded as absent. + assert_eq!(avro_to_value(&AvroValue::Array(vec![])), Value::Null); + assert_eq!(avro_to_value(&AvroValue::Record(vec![])), Value::Null); + } + + #[test] + fn avro_malformed_and_sniff() { + assert!(matches!( + AvroParser.parse("data.avro", b"not avro"), + Err(AxError::Parse { .. }) + )); + assert_eq!(AvroParser.sniff(&build_avro()), Some(MAGIC)); + assert_eq!(AvroParser.sniff(b"Obj\x01...."), Some(MAGIC)); + assert_eq!(AvroParser.sniff(b"ORC"), None); + assert_eq!(AvroParser.sniff(b"{}"), None); + assert_eq!(AvroParser.extensions(), &["avro"]); + } + + // ------------------------------------------------------------ ORC ------- + + /// Writes a tiny ORC file in-memory via the orc-rust Arrow writer. + fn build_orc() -> Vec { + use arrow::array::{ArrayRef, Float64Array, Int64Array, StringArray}; + use arrow::record_batch::RecordBatch; + use std::sync::Arc; + + let batch = RecordBatch::try_from_iter(vec![ + ("id", Arc::new(Int64Array::from(vec![1, 2, 3])) as ArrayRef), + ( + "host", + Arc::new(StringArray::from(vec!["a", "b", "c"])) as ArrayRef, + ), + ( + "score", + Arc::new(Float64Array::from(vec![9.5, 3.25, 7.5])) as ArrayRef, + ), + ]) + .unwrap(); + + let mut buf: Vec = Vec::new(); + { + let mut writer = orc_rust::ArrowWriterBuilder::new(&mut buf, batch.schema()) + .try_build() + .unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); + } + buf + } + + #[test] + fn orc_rows_are_type_inferred() { + let cols = OrcParser.parse("data.orc", &build_orc()).unwrap(); + assert_eq!(col(&cols, "id").ty, ColType::Int); + assert_eq!( + col(&cols, "id").cells, + vec![Value::Int(1), Value::Int(2), Value::Int(3)] + ); + assert_eq!( + col(&cols, "host").cells, + vec![ + Value::Str("a".into()), + Value::Str("b".into()), + Value::Str("c".into()) + ] + ); + assert_eq!(col(&cols, "score").numeric(), vec![9.5, 3.25, 7.5]); + } + + #[test] + fn orc_null_cell() { + use arrow::array::{ArrayRef, Int64Array}; + use arrow::record_batch::RecordBatch; + use std::sync::Arc; + let batch = RecordBatch::try_from_iter(vec![( + "v", + Arc::new(Int64Array::from(vec![Some(1), None, Some(3)])) as ArrayRef, + )]) + .unwrap(); + let mut buf = Vec::new(); + { + let mut w = orc_rust::ArrowWriterBuilder::new(&mut buf, batch.schema()) + .try_build() + .unwrap(); + w.write(&batch).unwrap(); + w.close().unwrap(); + } + let cols = OrcParser.parse("-", &buf).unwrap(); + assert_eq!(col(&cols, "v").cells[1], Value::Null); + } + + #[test] + fn orc_malformed_and_sniff() { + assert!(matches!( + OrcParser.parse("data.orc", b"not orc at all....."), + Err(AxError::Parse { .. }) + )); + assert_eq!(OrcParser.sniff(&build_orc()), Some(MAGIC)); + assert_eq!(OrcParser.sniff(b"ORC....."), Some(MAGIC)); + assert_eq!(OrcParser.sniff(b"Obj\x01"), None); + assert_eq!(OrcParser.extensions(), &["orc"]); + } + + #[test] + fn resolve_by_extension_and_magic() { + let reg = crate::parser::ParserRegistry::default(); + assert_eq!(reg.resolve("x.avro", b"zz").unwrap().id(), "avro"); + assert_eq!(reg.resolve("x.orc", b"zz").unwrap().id(), "orc"); + assert_eq!(reg.resolve("-", &build_avro()).unwrap().id(), "avro"); + assert_eq!(reg.resolve("-", &build_orc()).unwrap().id(), "orc"); + } +} diff --git a/crates/ax-normalize/src/parsers/mod.rs b/crates/ax-normalize/src/parsers/mod.rs index 11c16e4..532c318 100644 --- a/crates/ax-normalize/src/parsers/mod.rs +++ b/crates/ax-normalize/src/parsers/mod.rs @@ -6,6 +6,8 @@ use crate::parser::ParserRegistry; pub mod accesslog; pub mod auditd; +#[cfg(feature = "datalake")] +pub mod avro; pub mod cef; pub mod cloudtrail; pub mod delimited; @@ -39,6 +41,8 @@ pub mod columnar; pub use accesslog::AccessLogParser; pub use auditd::AuditdParser; +#[cfg(feature = "datalake")] +pub use avro::{AvroParser, OrcParser}; pub use cef::{CefParser, LeefParser}; pub use cloudtrail::CloudTrailParser; pub use delimited::{CsvParser, TsvParser}; @@ -86,6 +90,11 @@ pub fn default_registry() -> ParserRegistry { r.register(Box::new(XlsxParser)); #[cfg(feature = "sqlite")] r.register(Box::new(SqliteParser)); + #[cfg(feature = "datalake")] + { + r.register(Box::new(AvroParser)); + r.register(Box::new(OrcParser)); + } // OTLP before NDJSON: a compact single-object OTLP doc must win the // `resourceSpans` signature before any JSON-line heuristic sees it. r.register(Box::new(OtlpParser));