diff --git a/AGENTS.md b/AGENTS.md index 48174a7..cd67e3a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -55,3 +55,20 @@ These survive across refactors; reach for them when in doubt: - Dependencies and their roles: comments in `crates/mdvs/Cargo.toml`. - Skills (agent workflows): `.claude/skills/` — invoke via `Skill` for `commit`, `todo`, `rust`, `spec`, `code-editing`, etc. - TODOs (in-flight + done): `docs/spec/todos/index.md`. + + + +## mdvs knowledge base + +This project uses [mdvs](https://github.com/edochi/mdvs) to manage a markdown knowledge base with a schema defined in `mdvs.toml`. When working with files in this KB: + +- **Prefer `mdvs search ""` over `Grep` / `Glob` for semantic lookups.** mdvs runs semantic / hybrid / SQL-filtered search over the KB — usually a better fit than literal pattern matching for finding notes by meaning. Filter by frontmatter with `--where "field = 'value'"` and pick the mode with `--mode semantic|fulltext|hybrid`. +- **`mdvs check` validates frontmatter against the schema.** A `PostToolUse` hook may surface violations to you automatically as a markdown block under `additionalContext` in the hook output. Treat that block as **a warning, not a block** — the agent's edit already landed; the warning is for you to act on next. +- **When a violation reaches you, decide:** + 1. **Mistake** (a typo, the wrong type by accident, a dropped required field) → fix the file in the next turn. + 2. **Intentional deviation** (the KB is evolving, a category genuinely needs a new variant, a field is shifting type) → surface the deviation to the user and propose updating `mdvs.toml` to absorb the change. **Do not silently fix the file**; the user decides whether the schema or the file is wrong. + +The schema is meant to evolve with the KB. Enforcement follows the KB's shape, it does not freeze it. Whenever you propose a schema update, also explain *why* — which file(s) drove the change and what convention the user might want to standardise on. + +Full mdvs documentation: . diff --git a/Cargo.lock b/Cargo.lock index 3beb5ba..83d3dea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -118,6 +118,15 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "ar_archive_writer" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4087686b4b0a3427190bae57a1d9a478dbb2d40c5dc1bd6e2b6d797913bdd348" +dependencies = [ + "object", +] + [[package]] name = "arraydeque" version = "0.5.1" @@ -132,9 +141,9 @@ checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" [[package]] name = "arrayvec" -version = "0.7.6" +version = "0.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" +checksum = "f02882884d3e1bc524fb12c79f107f6ad0e1cfd498c536ffb494301740995dfe" [[package]] name = "arrow" @@ -565,9 +574,9 @@ dependencies = [ [[package]] name = "bitvec" -version = "1.0.1" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +checksum = "ddcec3d12c579d40898fe0a9a358a803c23e9c52ca3c425707f81c9436211837" dependencies = [ "funty", "radium", @@ -679,9 +688,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.11.1" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593" [[package]] name = "castaway" @@ -694,9 +703,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.64" +version = "1.2.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dad887fd958be91b5098c0248def011f4523ab786cd411be668777e55063501f" +checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96" dependencies = [ "find-msvc-tools", "jobserver", @@ -1779,6 +1788,38 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "defmt" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6e524506490a1953d237cb87b1cfc1e46f88c18f10a22dfe0f507dc6bfc7f7f" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0a27770e9c8f719a79d8b638281f4d828f77d8fd61e0bd94451b9b85e576a0b" +dependencies = [ + "defmt-parser", + "proc-macro-error2", + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "defmt-parser" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" +dependencies = [ + "thiserror", +] + [[package]] name = "deranged" version = "0.5.8" @@ -2335,15 +2376,13 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", "libc", "r-efi 6.0.0", - "wasip2", - "wasip3", ] [[package]] @@ -2573,7 +2612,7 @@ dependencies = [ "tokio", "tokio-rustls", "tower-service", - "webpki-roots 1.0.7", + "webpki-roots 1.0.8", ] [[package]] @@ -2760,12 +2799,6 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e4a2c462a4d927d512f5f882a033ddd62f33a05bb9f230d98f736ac3dc85938f" -[[package]] -name = "id-arena" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" - [[package]] name = "ident_case" version = "1.0.1" @@ -2809,6 +2842,25 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "include_dir" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "923d117408f1e49d914f1a379a309cffe4f18c05cf4e3d12e613a15fc81bd0dd" +dependencies = [ + "include_dir_macros", +] + +[[package]] +name = "include_dir_macros" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cab85a7ed0bd5f0e76d93846e0147172bed2e2d3f859bcc33a8d9699cad1a75" +dependencies = [ + "proc-macro2", + "quote", +] + [[package]] name = "indexmap" version = "1.9.3" @@ -2940,10 +2992,11 @@ dependencies = [ [[package]] name = "jiff" -version = "0.2.28" +version = "0.2.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4603d3033e49e2b0e31229fcab20a5d40089c607d975cd9c80551dc69eed9102" +checksum = "34f877a98676d2fb664698d74cc6a51ce6c484ce8c770f05d0108ec9090aeb46" dependencies = [ + "defmt", "jiff-static", "jiff-tzdb-platform", "log", @@ -2955,9 +3008,9 @@ dependencies = [ [[package]] name = "jiff-static" -version = "0.2.28" +version = "0.2.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "782d32378dddf207193ac91cefb848ad41abb58195c95168e1291227a0832b47" +checksum = "0666b5ab5ecaca213fc2a85b8c0083d9004e84ee2d5f9a7e0017aaf50986f25f" dependencies = [ "proc-macro2", "quote", @@ -3715,12 +3768,6 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" -[[package]] -name = "leb128fmt" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" - [[package]] name = "lexical-core" version = "1.0.6" @@ -3946,9 +3993,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.32" +version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" [[package]] name = "loom" @@ -4063,6 +4110,7 @@ dependencies = [ "globset", "gray_matter", "ignore", + "include_dir", "indextree", "jsonschema", "lance-index", @@ -4073,6 +4121,7 @@ dependencies = [ "regex", "serde", "serde_json", + "sqlparser", "tabled", "tempfile", "terminal_size", @@ -4096,9 +4145,9 @@ checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" [[package]] name = "memmap2" -version = "0.9.10" +version = "0.9.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3" +checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" dependencies = [ "libc", ] @@ -4380,6 +4429,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" +[[package]] +name = "object" +version = "0.37.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" +dependencies = [ + "memchr", +] + [[package]] name = "object_store" version = "0.12.5" @@ -4795,6 +4853,16 @@ dependencies = [ "prost", ] +[[package]] +name = "psm" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "645dbe486e346d9b5de3ef16ede18c26e6c70ad97418f4874b8b1889d6e761ea" +dependencies = [ + "ar_archive_writer", + "cc", +] + [[package]] name = "pulldown-cmark" version = "0.13.4" @@ -4816,9 +4884,9 @@ checksum = "007d8adb5ddab6f8e3f491ac63566a7d5002cc7ed73901f72057943fa71ae1ae" [[package]] name = "quinn" -version = "0.11.9" +version = "0.11.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" dependencies = [ "bytes", "cfg_aliases", @@ -4836,9 +4904,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.14" +version = "0.11.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +checksum = "4fcb935c5bec503c2f0e306bdd3e58bb9029dcb14fa8d9ac76e3a5256ac0763e" dependencies = [ "bytes", "getrandom 0.3.4", @@ -4871,9 +4939,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.45" +version = "1.0.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" dependencies = [ "proc-macro2", ] @@ -5000,6 +5068,26 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "recursive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0786a43debb760f491b1bc0269fe5e84155353c67482b9e60d0cfb596054b43e" +dependencies = [ + "recursive-proc-macro-impl", + "stacker", +] + +[[package]] +name = "recursive-proc-macro-impl" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" +dependencies = [ + "quote", + "syn 2.0.118", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -5129,7 +5217,7 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "webpki-roots 1.0.7", + "webpki-roots 1.0.8", ] [[package]] @@ -5235,9 +5323,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.40" +version = "0.23.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" +checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f" dependencies = [ "aws-lc-rs", "log", @@ -5699,6 +5787,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dbf5ea8d4d7c808e1af1cbabebca9a2abe603bcefc22294c5b95018d53200cb7" dependencies = [ "log", + "recursive", "sqlparser_derive", ] @@ -5719,6 +5808,19 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "stacker" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "640c8cdd92b6b12f5bcb1803ca3bbf5ab96e5e6b6b96b9ab77dabe9e880b3190" +dependencies = [ + "cc", + "cfg-if", + "libc", + "psm", + "windows-sys 0.60.2", +] + [[package]] name = "static_assertions" version = "1.1.0" @@ -5909,7 +6011,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.2", + "getrandom 0.4.3", "once_cell", "rustix", "windows-sys 0.61.2", @@ -5992,9 +6094,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.49" +version = "0.3.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "711a53c2d47bbd818258c498c8dbfe186a2526c631495cfe7e078567f86b8469" +checksum = "85c17d80feb7334b40c484e45ed1a5273dfd8bfda537c3be2e74a06a6686f327" dependencies = [ "deranged", "num-conv", @@ -6012,9 +6114,9 @@ checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" [[package]] name = "time-macros" -version = "0.2.29" +version = "0.2.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71c652a3727a9cbb9a02f707f530b618ce00d0ccd762009c8c23bd191df3c17d" +checksum = "dcef1a61bdb119096e153208ec5cbec23944ce8bca13be5c7f60c634f7403935" dependencies = [ "num-conv", "time-core", @@ -6436,12 +6538,6 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" -[[package]] -name = "unicode-xid" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" - [[package]] name = "unicode_categories" version = "0.1.1" @@ -6521,7 +6617,7 @@ version = "1.23.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "144d6b123cef80b301b8f72a9e2ca4370ddec21950d0a103dd22c437006d2db7" dependencies = [ - "getrandom 0.4.2", + "getrandom 0.4.3", "js-sys", "serde_core", "wasm-bindgen", @@ -6592,16 +6688,7 @@ version = "1.0.4+wasi-0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" dependencies = [ - "wit-bindgen 0.57.1", -] - -[[package]] -name = "wasip3" -version = "0.4.0+wasi-0.3.0-rc-2026-01-06" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" -dependencies = [ - "wit-bindgen 0.51.0", + "wit-bindgen", ] [[package]] @@ -6659,28 +6746,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "wasm-encoder" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" -dependencies = [ - "leb128fmt", - "wasmparser", -] - -[[package]] -name = "wasm-metadata" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" -dependencies = [ - "anyhow", - "indexmap 2.14.0", - "wasm-encoder", - "wasmparser", -] - [[package]] name = "wasm-streams" version = "0.4.2" @@ -6694,18 +6759,6 @@ dependencies = [ "web-sys", ] -[[package]] -name = "wasmparser" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" -dependencies = [ - "bitflags 2.13.0", - "hashbrown 0.15.5", - "indexmap 2.14.0", - "semver", -] - [[package]] name = "web-sys" version = "0.3.102" @@ -6728,9 +6781,9 @@ dependencies = [ [[package]] name = "webpki-root-certs" -version = "1.0.7" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f31141ce3fc3e300ae89b78c0dd67f9708061d1d2eda54b8209346fd6be9a92c" +checksum = "0d46a5a140e6f7afeccd8eae97eff335163939eac8b929834875168b29b3d267" dependencies = [ "rustls-pki-types", ] @@ -6741,14 +6794,14 @@ version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" dependencies = [ - "webpki-roots 1.0.7", + "webpki-roots 1.0.8", ] [[package]] name = "webpki-roots" -version = "1.0.7" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52f5ee44c96cf55f1b349600768e3ece3a8f26010c05265ab73f945bb1a2eb9d" +checksum = "bf85cb06032201fa7c6f829d7db5a7e5aa45bcc0655327713065f6f0576731bf" dependencies = [ "rustls-pki-types", ] @@ -7023,100 +7076,12 @@ dependencies = [ "memchr", ] -[[package]] -name = "wit-bindgen" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" -dependencies = [ - "wit-bindgen-rust-macro", -] - [[package]] name = "wit-bindgen" version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" -[[package]] -name = "wit-bindgen-core" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" -dependencies = [ - "anyhow", - "heck", - "wit-parser", -] - -[[package]] -name = "wit-bindgen-rust" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" -dependencies = [ - "anyhow", - "heck", - "indexmap 2.14.0", - "prettyplease", - "syn 2.0.118", - "wasm-metadata", - "wit-bindgen-core", - "wit-component", -] - -[[package]] -name = "wit-bindgen-rust-macro" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" -dependencies = [ - "anyhow", - "prettyplease", - "proc-macro2", - "quote", - "syn 2.0.118", - "wit-bindgen-core", - "wit-bindgen-rust", -] - -[[package]] -name = "wit-component" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" -dependencies = [ - "anyhow", - "bitflags 2.13.0", - "indexmap 2.14.0", - "log", - "serde", - "serde_derive", - "serde_json", - "wasm-encoder", - "wasm-metadata", - "wasmparser", - "wit-parser", -] - -[[package]] -name = "wit-parser" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" -dependencies = [ - "anyhow", - "id-arena", - "indexmap 2.14.0", - "log", - "semver", - "serde", - "serde_derive", - "serde_json", - "unicode-xid", - "wasmparser", -] - [[package]] name = "writeable" version = "0.6.3" diff --git a/README.md b/README.md index aa86c5c..57a7871 100644 --- a/README.md +++ b/README.md @@ -153,14 +153,16 @@ Searched "how to get in touch" — 3 hits └──────────────────────────┴─────────────────────────────────────────┘ ``` -`alice.md` doesn't contain "get in touch" — mdvs finds it by meaning, not keywords. Filter with SQL on frontmatter: +`alice.md` doesn't contain "get in touch" — mdvs finds it by meaning, not keywords. Filter with SQL on frontmatter or on path: ```bash mdvs search "rust" --where "draft = false" mdvs search "meeting notes" --where "date > '2026-05-01'" +mdvs search "incident" --where "filepath LIKE 'logs/%'" # restrict by directory +mdvs search "review" --where "filepath LIKE '%-postmortem.md'" # match by filename suffix ``` -The typed schema is what makes `--where` work. Without it, `tags = 'rust'` would be a fuzzy guess; with it, it's an equality check on a known-typed array column. +The typed schema is what makes frontmatter filters work — mdvs knows `tags` is `Array(String)`, so `--where "tags = 'rust'"` is auto-rewritten to `array_has(data.tags, 'rust')` (the search output prints a one-line "Note" showing the rewrite so it's never magic). `=`, `!=`, `IN`, and `NOT IN` all do element-containment against array fields. The always-present `filepath` column lets you filter by path with standard `LIKE` patterns; its last component is the filename. > **Try it on your own files:** > ```bash @@ -207,7 +209,7 @@ mdvs export-jsonschema --format json - **Rust** — mdvs is written in Rust; the CLI is a single static binary. - **[LanceDB](https://lancedb.com/)** — backs storage and search. Cosine vector search, BM25 full-text, and RRF hybrid all run natively against the Lance dataset. -- **[Model2Vec](https://minish.ai/)** — static embedding models; the default is `potion-base-8M` (~60 MB, CPU-only, no GPU). +- **[Model2Vec](https://minish.ai/)** — static embedding models; the default is `potion-multilingual-128M` (~480 MB, 101 languages, CPU-only, no GPU). Smaller models like `potion-base-8M` (~60 MB) are available via `--set-model`. - **[`jsonschema`](https://crates.io/crates/jsonschema)** — JSON Schema 2020-12 validator. mdvs translates your `mdvs.toml` into a canonical JSON Schema document and validates frontmatter values through per-field validators compiled from it. - **[`pulldown-cmark`](https://crates.io/crates/pulldown-cmark)** — markdown parsing; used to extract plain text from each chunk before embedding. - **[`text-splitter`](https://crates.io/crates/text-splitter)** — semantic-aware chunker that splits the markdown body along heading and paragraph boundaries. diff --git a/book/src/SUMMARY.md b/book/src/SUMMARY.md index c966466..ae0918d 100644 --- a/book/src/SUMMARY.md +++ b/book/src/SUMMARY.md @@ -24,6 +24,12 @@ - [export-jsonschema](./commands/export-jsonschema.md) - [Recipes](./recipes.md) + - [Agent harnesses](./recipes/agent-harnesses.md) + - [Antigravity](./recipes/agent-harnesses/antigravity.md) + - [Claude Code](./recipes/agent-harnesses/claude-code.md) + - [Codex](./recipes/agent-harnesses/codex.md) + - [Cursor](./recipes/agent-harnesses/cursor.md) + - [OpenCode](./recipes/agent-harnesses/opencode.md) - [Obsidian](./recipes/obsidian.md) - [Hugo](./recipes/hugo.md) - [CI](./recipes/ci.md) diff --git a/book/src/commands/build.md b/book/src/commands/build.md index c4bb19f..d078d1c 100644 --- a/book/src/commands/build.md +++ b/book/src/commands/build.md @@ -59,7 +59,7 @@ When nothing needs embedding, the model is never loaded. When the change set is ``` config changed since last build: - model: 'minishlab/potion-base-8M' → 'minishlab/potion-base-32M' + model: 'minishlab/potion-multilingual-128M' → 'minishlab/potion-base-8M' Use --force to rebuild with new config ``` @@ -72,10 +72,10 @@ Use --force to rebuild with new schema This catches edits to `[[fields.field]]` definitions, constraint changes, preprocessor changes, and path-scoping changes — anything that affects what gets stored in the `data` column of the index. -The `--set-model`, `--set-revision`, and `--set-chunk-size` flags update `mdvs.toml` and require `--force` (since they change the config and trigger a full re-embed). For example, to switch to a larger model: +The `--set-model`, `--set-revision`, and `--set-chunk-size` flags update `mdvs.toml` and require `--force` (since they change the config and trigger a full re-embed). For example, to switch to a smaller English-only model: ```bash -mdvs build --set-model minishlab/potion-base-32M --force +mdvs build --set-model minishlab/potion-base-8M --force ``` `--set-revision` pins the model to a specific HuggingFace commit SHA, ensuring reproducible embeddings even if the model is updated upstream: @@ -140,7 +140,7 @@ Scan: 43 files (4ms) Infer: 37 field(s) (0ms) Validate: 43 files — no violations (87ms) Classify: 43 to embed, 0 unchanged, 0 removed (0ms) -Load model: minishlab/potion-base-8M (24ms) +Load model: minishlab/potion-multilingual-128M (24ms) Embed: 43 files, 59 chunks (12ms) Write index: 43 files, 59 chunks (1ms) Built index — 43 files, 59 chunks (full rebuild) diff --git a/book/src/commands/info.md b/book/src/commands/info.md index 5dcefe5..284a20b 100644 --- a/book/src/commands/info.md +++ b/book/src/commands/info.md @@ -44,7 +44,7 @@ Config: Index: ┌──────────────────────────┬───────────────────────────────────────────────────┐ -│ model │ minishlab/potion-base-8M │ +│ model │ minishlab/potion-multilingual-128M │ ├──────────────────────────┼───────────────────────────────────────────────────┤ │ revision │ none │ ├──────────────────────────┼───────────────────────────────────────────────────┤ diff --git a/book/src/commands/search.md b/book/src/commands/search.md index e80680a..eaa23b8 100644 --- a/book/src/commands/search.md +++ b/book/src/commands/search.md @@ -16,7 +16,7 @@ mdvs search [path] [flags] | `path` | `.` | Directory containing `mdvs.toml` | | `--mode` | `hybrid` | Search mode: `semantic`, `fulltext`, or `hybrid` | | `--limit` / `-n` | `10` | Maximum number of results | -| `--where` | | SQL WHERE clause for filtering on frontmatter fields | +| `--where` | | SQL WHERE clause — filter on frontmatter fields or on the `filepath` column | | `--no-update` | | Skip auto-update | | `--no-build` | | Skip auto-build before searching | @@ -47,17 +47,17 @@ See [Search & Indexing](../concepts/search.md) for details on chunking, embeddin > | Model | Size | > |---|---| > | `potion-base-2M` | ~8 MB | -> | `potion-base-8M` (default) | ~30 MB | +> | `potion-base-8M` | ~30 MB | > | `potion-base-32M` | ~120 MB | -> | `potion-multilingual-128M` | ~480 MB | +> | `potion-multilingual-128M` (default) | ~480 MB | > > After the model is cached, a full build of 500+ files completes in under a second. ### `--where` -Filter results by frontmatter fields using SQL syntax. The filter and similarity ranking are combined in a single query, so files that don't match are excluded efficiently. +Filter results using SQL syntax. The filter and similarity ranking are combined in a single query, so files that don't match are excluded efficiently. `--where` operates on **any column** in the Lance index — frontmatter fields (auto-discovered from `mdvs.toml`) and the always-present `filepath` column. -Scalar comparisons: +Scalar frontmatter comparisons: ```bash mdvs search "experiment" --where "status = 'active'" @@ -65,13 +65,23 @@ mdvs search "experiment" --where "sample_count > 20" mdvs search "experiment" --where "status = 'active' AND priority = 1" ``` -Array fields (via LanceDB's SQL array functions): +Array fields — `=` / `!=` / `IN` / `NOT IN` are auto-rewritten to `array_has(...)`: ```bash -mdvs search "calibration" --where "array_has(tags, 'biosensor')" +mdvs search "calibration" --where "tags = 'biosensor'" # auto-rewritten +mdvs search "calibration" --where "tags IN ('biosensor', 'optics')" # OR-chain of array_has ``` -`--where` clauses that reference `Array(Float)` fields are rejected up front with a clear error. See the [Search Guide](../search-guide.md) for the full explanation and the workaround. +The translation note at the top of the result shows the rewrite. `--where` clauses that reference `Array(Float)` fields are rejected up front with a clear error — see the [Search Guide](../search-guide.md) for the workaround. + +Path filtering via the always-present `filepath` column (its last component is the filename): + +```bash +mdvs search "race condition" --where "filepath LIKE 'logs/%'" # everything under logs/ +mdvs search "review" --where "filepath LIKE '%-postmortem.md'" # filename suffix +mdvs search "alpha" --where "filepath = 'projects/alpha/overview.md'" # exact path +mdvs search "deploy" --where "filepath LIKE 'logs/%' AND status = 'published'" # combine +``` Field names with spaces need double-quoting: @@ -97,7 +107,7 @@ Searched "experiment" — 3 hits ┌──────────────────────────┬───────────────────────────────────────────────────┐ │ query │ experiment │ ├──────────────────────────┼───────────────────────────────────────────────────┤ -│ model │ minishlab/potion-base-8M │ +│ model │ minishlab/potion-multilingual-128M │ ├──────────────────────────┼───────────────────────────────────────────────────┤ │ limit │ 3 │ └──────────────────────────┴───────────────────────────────────────────────────┘ @@ -149,7 +159,7 @@ Searched "experiment" — 3 hits ┌──────────────────────────┬───────────────────────────────────────────────────┐ │ query │ experiment │ ├──────────────────────────┼───────────────────────────────────────────────────┤ -│ model │ minishlab/potion-base-8M │ +│ model │ minishlab/potion-multilingual-128M │ ├──────────────────────────┼───────────────────────────────────────────────────┤ │ limit │ 5 │ └──────────────────────────┴───────────────────────────────────────────────────┘ @@ -173,7 +183,7 @@ mdvs search "experiment" example_kb -v -n 3 Read config: example_kb/mdvs.toml (2ms) Scan: 43 files (2ms) ... -Load model: minishlab/potion-base-8M (22ms) +Load model: minishlab/potion-multilingual-128M (22ms) Embed query: "experiment" (0ms) Execute search: 3 hits (5ms) Searched "experiment" — 3 hits diff --git a/book/src/concepts/search.md b/book/src/concepts/search.md index b08af7d..fad4727 100644 --- a/book/src/concepts/search.md +++ b/book/src/concepts/search.md @@ -24,18 +24,18 @@ Chunks are embedded into dense vectors using a local [Model2Vec](https://minish. ```toml [embedding_model] provider = "model2vec" -name = "minishlab/potion-base-8M" +name = "minishlab/potion-multilingual-128M" ``` -The default is `potion-base-8M`, a good balance of size and quality. The full [POTION family](https://huggingface.co/collections/minishlab/potion-6721e0abd4ea41881417f062): +The default is `potion-multilingual-128M` — 101 languages, ~480 MB on disk. The full [POTION family](https://huggingface.co/collections/minishlab/potion-6721e0abd4ea41881417f062): | Model | Parameters | Notes | |---|---|---| | `minishlab/potion-base-2M` | 2M | Smallest, fastest | -| `minishlab/potion-base-8M` | 8M | Default — good balance | -| `minishlab/potion-base-32M` | 32M | Higher quality, slower | -| `minishlab/potion-retrieval-32M` | 32M | Optimized for retrieval tasks | -| `minishlab/potion-multilingual-128M` | 128M | 101 languages | +| `minishlab/potion-base-8M` | 8M | English-only, ~60 MB — good balance for English vaults | +| `minishlab/potion-base-32M` | 32M | English-only, higher quality, slower | +| `minishlab/potion-retrieval-32M` | 32M | English-only, optimized for retrieval tasks | +| `minishlab/potion-multilingual-128M` | 128M | Default — 101 languages | Any Model2Vec-compatible model on HuggingFace works — set the `name` to its model ID. You can pin a specific revision for reproducibility. diff --git a/book/src/configuration.md b/book/src/configuration.md index 3e0e7ce..51849cc 100644 --- a/book/src/configuration.md +++ b/book/src/configuration.md @@ -135,13 +135,13 @@ Specifies the embedding model for semantic search. See [Embedding](./concepts/se ```toml [embedding_model] provider = "model2vec" -name = "minishlab/potion-base-8M" +name = "minishlab/potion-multilingual-128M" ``` | Field | Type | Default | Description | |---|---|---|---| | `provider` | String | `"model2vec"` | Embedding provider (currently only `"model2vec"`) | -| `name` | String | `"minishlab/potion-base-8M"` | HuggingFace model ID | +| `name` | String | `"minishlab/potion-multilingual-128M"` | HuggingFace model ID | | `revision` | String | (none) | Pin to a specific HuggingFace commit SHA for reproducibility | The `provider` field can be omitted — it defaults to `"model2vec"`. The `revision` field only appears when explicitly set (e.g., via `build --set-revision`). @@ -448,7 +448,7 @@ frontmatter_format = "auto" [embedding_model] provider = "model2vec" -name = "minishlab/potion-base-8M" +name = "minishlab/potion-multilingual-128M" [chunking] max_chunk_size = 1024 diff --git a/book/src/getting-started.md b/book/src/getting-started.md index a1e7e84..014e976 100644 --- a/book/src/getting-started.md +++ b/book/src/getting-started.md @@ -203,7 +203,7 @@ Searched "calibration" — 10 hits ┌──────────────────────────┬───────────────────────────────────────────────────┐ │ query │ calibration │ ├──────────────────────────┼───────────────────────────────────────────────────┤ -│ model │ minishlab/potion-base-8M │ +│ model │ minishlab/potion-multilingual-128M │ ├──────────────────────────┼───────────────────────────────────────────────────┤ │ limit │ 10 │ └──────────────────────────┴───────────────────────────────────────────────────┘ @@ -243,7 +243,7 @@ Searched "quantum" — 3 hits ┌──────────────────────────┬───────────────────────────────────────────────────┐ │ query │ quantum │ ├──────────────────────────┼───────────────────────────────────────────────────┤ -│ model │ minishlab/potion-base-8M │ +│ model │ minishlab/potion-multilingual-128M │ ├──────────────────────────┼───────────────────────────────────────────────────┤ │ limit │ 10 │ └──────────────────────────┴───────────────────────────────────────────────────┘ diff --git a/book/src/introduction.md b/book/src/introduction.md index 37e8f53..f31d2f1 100644 --- a/book/src/introduction.md +++ b/book/src/introduction.md @@ -4,7 +4,7 @@ mdvs treats your markdown directory like a database. It scans your files, infers Not a document database. A database *for* documents. -## The problem +## The challenge Markdown directories grow organically. You start with a few notes, add frontmatter when it's useful, and eventually have hundreds of files with inconsistent metadata. Tags are misspelled. Required fields are missing. You can't find anything without `grep`. @@ -18,7 +18,7 @@ Frontmatter is the YAML block between `---` fences at the top of a markdown file --- title: "Experiment A-017: SPR-A1 baseline calibration" # String status: completed # String -author: Giulia Ferretti # String +author: Federica Bianchi # String draft: false # Boolean priority: 2 # Integer drift_rate: 0.023 # Float diff --git a/book/src/recipes.md b/book/src/recipes.md index cba58ef..ac95e86 100644 --- a/book/src/recipes.md +++ b/book/src/recipes.md @@ -1,7 +1,8 @@ # Recipes -Walkthroughs for pointing mdvs at common markdown ecosystems. +Walkthroughs for pointing mdvs at common markdown ecosystems and the agents that work in them. +- **[Agent harnesses](./recipes/agent-harnesses.md)** — Wiring mdvs into Claude Code, Codex, Cursor, OpenCode, Antigravity. Skill file, project-rules snippet, validate-on-write hook, search-nudge hook. Per-platform pages for copy-paste install; overview page for the architecture and how to extend to a new harness. - **[Obsidian](./recipes/obsidian.md)** — YAML-frontmatter vaults, `.mdvsignore` patterns, Dataview caveats, common validation setups - **[Hugo](./recipes/hugo.md)** — Mixed-format sites (YAML / TOML / JSON), native TOML date queries, forced-format mode for opinionated repos - **[CI](./recipes/ci.md)** — Running `mdvs check` in a pipeline as a frontmatter linter diff --git a/book/src/recipes/agent-harnesses.md b/book/src/recipes/agent-harnesses.md new file mode 100644 index 0000000..7ca416c --- /dev/null +++ b/book/src/recipes/agent-harnesses.md @@ -0,0 +1,84 @@ +# Agent harnesses + +mdvs ships agent integration in three pieces: + +1. A **skill** (the [Agent Skills standard](https://agentskills.io)) — works in any harness that loads `.md` skills. +2. A **project-rules snippet** — works in any harness that reads `AGENTS.md` / `CLAUDE.md` / `.cursor/rules`. +3. A **PostToolUse hook** that calls `mdvs hook handle` — only verified end-to-end on Claude Code today. + +Per-harness install steps in the left nav. + +## How violations reach the agent (Claude Code) + +When the agent edits a markdown file in your vault: + +1. The harness's PostToolUse hook fires the configured `mdvs hook handle` command. +2. mdvs reads the tool-call payload, walks up to find `mdvs.toml`. If the edit happened outside any vault, the hook stays silent. +3. mdvs runs `check` on the vault. If the file is clean, the hook stays silent (no noise on the happy path). +4. If there are violations, mdvs writes a Claude-Code-shaped envelope JSON to stdout. The harness reads it and surfaces the markdown body to the agent through `additionalContext` and the pretty render to the user through `systemMessage`. +5. The agent sees the violation and reacts on its next turn — per the [schema-evolution loop](https://github.com/edochi/mdvs/blob/main/crates/mdvs/scaffolding/skill/SKILL.md): if it's a mistake, fix the file; if it's intentional (KB evolving), surface the deviation to the user and propose updating `mdvs.toml`. + +A separate **search-nudge** hook fires after every Bash command that runs `grep` / `rg` / `find` / `ag` / `ack` / `fd` / `git grep`. If the agent's cwd is inside an mdvs vault, the hook surfaces a one-line tip suggesting `mdvs search`. Like validate, it's non-blocking — the agent decides whether to switch tools. + +## Per-platform support + +| Platform | Skill | Snippet | Hooks | +|---|---|---|---| +| [Claude Code](agent-harnesses/claude-code.md) | ✓ | ✓ | ✓ | +| [Codex](agent-harnesses/codex.md) | ✓ | ✓ | see [Codex hooks docs](https://developers.openai.com/codex/hooks) | +| [Cursor](agent-harnesses/cursor.md) | ✓ | ✓ | see [Cursor hooks docs](https://cursor.com/docs/hooks) | +| [OpenCode](agent-harnesses/opencode.md) | ✓ | ✓ | see [OpenCode docs](https://opencode.ai/docs/) | +| [Antigravity](agent-harnesses/antigravity.md) | ✓ | ✓ | see [Gemini CLI hooks docs](https://github.com/google-gemini/gemini-cli/tree/main/docs/hooks) | + +## Pre-commit hook + +A **pre-commit hook** is a script git runs locally before each `git commit` — if it exits non-zero, the commit is blocked. The community [`pre-commit`](https://pre-commit.com/) tool manages hooks declaratively per-repo via a YAML config; mdvs plugs into it as a one-line entry. + +Running `mdvs check` as a pre-commit hook catches frontmatter violations before they reach the repo, **regardless of how the file was edited** — agent, IDE, or by hand. It's the simplest harness-independent safety net, and the recommended fallback for harnesses where the post-edit hook isn't wired up. + +### Install + +To install the `pre-commit` tool on your machine check the docs at this [link](https://pre-commit.com/#install). + +It's also possible to install `pre-commit` using [`uv`](https://docs.astral.sh/uv/): + +```bash +uv tool install pre-commit +``` + +### Configure + +In your mdvs vault, create `.pre-commit-config.yaml`: + +```yaml +repos: + - repo: local + hooks: + - id: mdvs-check + name: mdvs check + entry: mdvs check --no-update + language: system + pass_filenames: false +``` + +Activate the hook in this repo (writes `.git/hooks/pre-commit`): + +```bash +pre-commit install +``` + +That's it. The next `git commit` runs `mdvs check`; if there are violations the commit aborts and the violation report is printed. To run the check manually without committing: + +```bash +pre-commit run --all-files +``` + +### Notes + +- **Works with any install method.** `language: system` just runs the `mdvs` already on your PATH — it doesn't matter whether you installed via `cargo install mdvs`, the release shell installer, Homebrew, or a manually-placed binary. The only requirement is that `mdvs` is invocable from git's environment. +- **PATH gotcha for GUI git clients.** git pre-commit hooks fire under git's environment, which isn't always the same as your interactive shell's PATH. If `mdvs` lives in `~/.cargo/bin/` and you commit from a GUI client that doesn't inherit your shell PATH, the hook fails with `mdvs: command not found`. Either commit from the terminal, or use the absolute path in `entry:` (`entry: /Users/you/.cargo/bin/mdvs check --no-update`). +- **Version-pinned alternative.** To have `pre-commit` fetch `mdvs` into its own isolated environment (slower per-repo install, but reproducible across machines and CI), swap to `language: rust` and `additional_dependencies: ["mdvs"]`. +- `--no-update` tells `mdvs check` not to auto-update `mdvs.toml` from inferred new fields. The hook validates against the committed schema; schema evolution stays an explicit user action. +- `pass_filenames: false` because `mdvs check` runs against the whole vault, not file-by-file. The same validation pass covers every staged change in one shot. + +For CI-side validation (catches violations even if a contributor skipped the local hook), see the [CI recipe](./ci.md). diff --git a/book/src/recipes/agent-harnesses/antigravity.md b/book/src/recipes/agent-harnesses/antigravity.md new file mode 100644 index 0000000..823bbeb --- /dev/null +++ b/book/src/recipes/agent-harnesses/antigravity.md @@ -0,0 +1,32 @@ +# Antigravity + +## Install + +```bash +mkdir -p .agents/skills/mdvs +mdvs scaffold skill > .agents/skills/mdvs/SKILL.md +mdvs scaffold snippet --platform antigravity >> AGENTS.md +``` + +Antigravity reads skills from `.agents/skills//SKILL.md` (the cross-harness Agent Skills convention — same path Codex uses) and reads project rules from `AGENTS.md` at the workspace root. + +## What you get + +- **Skill**: agent learns when to call which mdvs command, how to interpret violations, and the schema-evolution loop. Loaded by Antigravity on session start. +- **Snippet**: always-on project-rules block telling the agent to prefer `mdvs search` over `Grep` for KB lookups. + +## Hooks + +mdvs doesn't ship a verified Antigravity hook config. Antigravity inherits parts of its configuration sources from Gemini CLI, so the hooks system should be compatible with what's documented at the [Gemini CLI hooks reference](https://github.com/google-gemini/gemini-cli/tree/main/docs/hooks). + +As a harness-independent fallback, the [pre-commit hook](../agent-harnesses.md#pre-commit-hook) runs `mdvs check` on every commit. + +## Per-platform notes + +- **Skill install path**: `.agents/skills/mdvs/SKILL.md`. Project-scoped path; the agent picks it up when working in the directory. +- **AGENTS.md**: documented as the post-rebrand convention. Legacy Gemini CLI sessions also recognized `GEMINI.md`; both still appear in flux. + +## Sources + +- [Authoring Google Antigravity Skills (Codelab)](https://codelabs.developers.google.com/getting-started-with-antigravity-skills) +- [Gemini CLI configuration (inherited reference)](https://github.com/google-gemini/gemini-cli/tree/main/docs/hooks) diff --git a/book/src/recipes/agent-harnesses/claude-code.md b/book/src/recipes/agent-harnesses/claude-code.md new file mode 100644 index 0000000..0eadb1d --- /dev/null +++ b/book/src/recipes/agent-harnesses/claude-code.md @@ -0,0 +1,33 @@ +# Claude Code + +## Install + +Three commands. Run each in your project root: + +```bash +mkdir -p .claude/skills/mdvs +mdvs scaffold skill > .claude/skills/mdvs/SKILL.md +mdvs scaffold snippet --platform claude-code >> CLAUDE.md +mdvs scaffold hook --platform claude-code >> .claude/settings.json # merge into existing hooks +``` + +The last command emits a JSON snippet — if `.claude/settings.json` already exists with other settings, **merge by hand** instead of appending blindly: the `hooks.PostToolUse` array should be unioned with anything you already have. mdvs's emitted snippet self-documents the merge target in a `_comment` field at the top. + +## What you get + +- **Skill**: agent learns when to call which mdvs command, how to interpret violations, and the schema-evolution loop. Loaded on session start; activated by description-match or directly via `/mdvs`. +- **Snippet**: always-on `CLAUDE.md` block telling the agent to prefer `mdvs search` over `Grep` for KB lookups. +- **Validate hook**: after every `Edit` / `Write` / `MultiEdit` on a markdown file inside an mdvs vault, `mdvs hook handle` runs `check` and surfaces violations through `additionalContext` (agent-visible) and `systemMessage` (user-visible, capped at 15 lines). Hook always exits 0 — never blocks. +- **Search-nudge hook**: after every `Bash` command that runs `grep` / `rg` / `find` / etc., if the agent's cwd is in an mdvs vault, surfaces a one-line tip pointing at `mdvs search`. + +## Per-platform notes + +- **Skill path**: `.claude/skills/mdvs/SKILL.md` (Claude Code reads only from `.claude/skills/`, not the cross-harness `.agents/skills/`). +- **Project rules**: `CLAUDE.md` at workspace root. +- **Hook envelope**: Claude Code's `hookSpecificOutput.additionalContext` + `systemMessage` shape. PascalCase event name (`PostToolUse`). +- **mdvs on PATH**: the hook command is `mdvs hook handle --platform claude-code --kind `. + +## Sources + +- [Claude Code skills](https://code.claude.com/docs/en/skills) +- [Claude Code hooks](https://code.claude.com/docs/en/hooks) diff --git a/book/src/recipes/agent-harnesses/codex.md b/book/src/recipes/agent-harnesses/codex.md new file mode 100644 index 0000000..9158596 --- /dev/null +++ b/book/src/recipes/agent-harnesses/codex.md @@ -0,0 +1,32 @@ +# Codex + +## Install + +```bash +mkdir -p .agents/skills/mdvs +mdvs scaffold skill > .agents/skills/mdvs/SKILL.md +mdvs scaffold snippet --platform codex >> AGENTS.md +``` + +## What you get + +- **Skill**: agent learns when to call which mdvs command, how to interpret violations, and the schema-evolution loop. Loaded from `.agents/skills/mdvs/SKILL.md` (the cross-harness Agent Skills convention). +- **Snippet**: always-on `AGENTS.md` block telling the agent to prefer `mdvs search` over `Grep`. + +## Hooks + +mdvs doesn't ship a verified Codex hook config. To wire `mdvs hook handle` into Codex's PostToolUse mechanism, follow the [Codex hooks docs](https://developers.openai.com/codex/hooks). + +As a harness-independent fallback, the [pre-commit hook](../agent-harnesses.md#pre-commit-hook) runs `mdvs check` on every commit. + +## Per-platform notes + +- **Skill path**: `.agents/skills/mdvs/SKILL.md` (Codex's canonical path per [their skills docs](https://developers.openai.com/codex/skills/) — same path Cursor and Antigravity also honor). +- **Project rules**: `AGENTS.md` at workspace root. `AGENTS.override.md` takes precedence if present. +- **mdvs on PATH**: `mdvs` must be available to any subprocess Codex runs. + +## Sources + +- [Codex skills](https://developers.openai.com/codex/skills/) +- [Codex hooks](https://developers.openai.com/codex/hooks) +- [Codex AGENTS.md guide](https://developers.openai.com/codex/guides/agents-md) diff --git a/book/src/recipes/agent-harnesses/cursor.md b/book/src/recipes/agent-harnesses/cursor.md new file mode 100644 index 0000000..963878e --- /dev/null +++ b/book/src/recipes/agent-harnesses/cursor.md @@ -0,0 +1,31 @@ +# Cursor + +## Install + +```bash +mkdir -p .cursor/skills/mdvs .cursor/rules +mdvs scaffold skill > .cursor/skills/mdvs/SKILL.md +mdvs scaffold snippet --platform cursor > .cursor/rules/mdvs.mdc +``` + +## What you get + +- **Skill**: agent learns when to call which mdvs command, how to interpret violations, and the schema-evolution loop. Cursor reads skills from `.cursor/skills/`, `.agents/skills/`, `.claude/skills/`, and `.codex/skills/` — `mdvs scaffold skill --platform cursor` writes to `.cursor/skills/` as the native path. +- **Snippet**: `.cursor/rules/mdvs.mdc` with `alwaysApply: true`. Cursor includes it in every conversation automatically. + +## Hooks + +mdvs doesn't ship a verified Cursor hook config. To wire `mdvs hook handle` into Cursor's PostToolUse mechanism, follow the [Cursor hooks docs](https://cursor.com/docs/hooks). + +As a harness-independent fallback, the [pre-commit hook](../agent-harnesses.md#pre-commit-hook) runs `mdvs check` on every commit. + +## Per-platform notes + +- **Project rules**: Cursor also honors `AGENTS.md` at workspace root. If you'd rather paste the universal snippet there: `mdvs scaffold snippet >> AGENTS.md` (without `--platform`). +- **mdvs on PATH**: `mdvs` must be available to any subprocess Cursor runs. On macOS, Cursor launched from Spotlight may not see your shell PATH — symlink `mdvs` into `/usr/local/bin/` or install via `cargo install --path crates/mdvs`. + +## Sources + +- [Cursor skills](https://cursor.com/docs/context/skills) +- [Cursor rules](https://cursor.com/docs/rules) +- [Cursor hooks](https://cursor.com/docs/hooks) diff --git a/book/src/recipes/agent-harnesses/opencode.md b/book/src/recipes/agent-harnesses/opencode.md new file mode 100644 index 0000000..3dbbd78 --- /dev/null +++ b/book/src/recipes/agent-harnesses/opencode.md @@ -0,0 +1,31 @@ +# OpenCode + +## Install + +```bash +mkdir -p .opencode/skills/mdvs +mdvs scaffold skill > .opencode/skills/mdvs/SKILL.md +mdvs scaffold snippet --platform opencode >> AGENTS.md +``` + +## What you get + +- **Skill**: agent learns when to call which mdvs command, how to interpret violations, and the schema-evolution loop. Loaded by OpenCode on session start. +- **Snippet**: always-on `AGENTS.md` block telling the agent to prefer `mdvs search` over `Grep`. + +## Hooks + +OpenCode handles tool events through a TypeScript plugin API rather than shell-command hooks. At the moment, mdvs doesn't ship a verified plugin or hook config for OpenCode, to wire `mdvs hook handle` into OpenCode's plugin events, follow the [OpenCode docs](https://opencode.ai/docs/). + +As a harness-independent fallback, the [pre-commit hook](../agent-harnesses.md#pre-commit-hook) runs `mdvs check` on every commit. + +## Per-platform notes + +- **Skill path**: `.opencode/skills/mdvs/SKILL.md` (native). OpenCode also reads `.claude/skills/` and `.agents/skills/` — you can symlink across if you want a single source of truth shared with another harness. +- **Project rules**: `AGENTS.md` at workspace root. OpenCode also reads `CLAUDE.md` as a Claude Code-compat fallback. + +## Sources + +- [OpenCode skills](https://opencode.ai/docs/skills/) +- [OpenCode agents](https://opencode.ai/docs/agents/) +- [OpenCode rules](https://opencode.ai/docs/rules/) diff --git a/book/src/search-guide.md b/book/src/search-guide.md index 2fa8b7c..a580169 100644 --- a/book/src/search-guide.md +++ b/book/src/search-guide.md @@ -1,6 +1,6 @@ # Search Guide -The `--where` flag on [search](./commands/search.md) lets you filter results by frontmatter fields using SQL syntax. The filter is combined with similarity ranking in a single query — files that don't match are excluded before results are returned. +The `--where` flag on [search](./commands/search.md) lets you filter results using SQL syntax. The filter is combined with similarity ranking in a single query — files that don't match are excluded before results are returned. `--where` operates on **any column** in the Lance index: frontmatter fields (auto-discovered from `mdvs.toml`) and the always-present `filepath` column (see [Filtering by file path](#filtering-by-file-path)). Under the hood, mdvs hands the clause to [LanceDB](https://lancedb.com/)'s SQL filter, which is built on top of DataFusion — so any expression valid in DataFusion's SQL dialect works in `--where`. @@ -33,7 +33,7 @@ Searched "experiment" — 2 hits ┌──────────────────────────┬───────────────────────────────────────────────────┐ │ query │ experiment │ ├──────────────────────────┼───────────────────────────────────────────────────┤ -│ model │ minishlab/potion-base-8M │ +│ model │ minishlab/potion-multilingual-128M │ ├──────────────────────────┼───────────────────────────────────────────────────┤ │ limit │ 10 │ └──────────────────────────┴───────────────────────────────────────────────────┘ @@ -167,7 +167,7 @@ Searched "calibration" — 4 hits ┌──────────────────────────┬───────────────────────────────────────────────────┐ │ query │ calibration │ ├──────────────────────────┼───────────────────────────────────────────────────┤ -│ model │ minishlab/potion-base-8M │ +│ model │ minishlab/potion-multilingual-128M │ ├──────────────────────────┼───────────────────────────────────────────────────┤ │ limit │ 10 │ └──────────────────────────┴───────────────────────────────────────────────────┘ @@ -215,7 +215,7 @@ Searched "experiment" — 8 hits ┌──────────────────────────┬───────────────────────────────────────────────────┐ │ query │ experiment │ ├──────────────────────────┼───────────────────────────────────────────────────┤ -│ model │ minishlab/potion-base-8M │ +│ model │ minishlab/potion-multilingual-128M │ ├──────────────────────────┼───────────────────────────────────────────────────┤ │ limit │ 10 │ └──────────────────────────┴───────────────────────────────────────────────────┘ @@ -229,17 +229,27 @@ Searched "experiment" — 8 hits ... ``` -File paths are stored as relative paths (e.g., `projects/alpha/notes/experiment-1.md`), so use `LIKE` with `%` for path prefix matching: +File paths are stored as relative paths (e.g., `projects/alpha/notes/experiment-1.md`). The **last component is the filename**, so you can match by directory, by filename, or by both: ```bash -# All blog posts +# All blog posts (directory prefix) --where "filepath LIKE 'blog/%'" -# Only published blog posts +# Only published blog posts (deeper directory prefix) --where "filepath LIKE 'blog/published/%'" -# Files in any meetings directory +# Files in any meetings directory (mid-path match) --where "filepath LIKE '%/meetings/%'" + +# Match by filename suffix (last component) +--where "filepath LIKE '%-postmortem.md'" +--where "filepath LIKE '%/README.md'" + +# Exact file +--where "filepath = 'projects/alpha/overview.md'" + +# Combine with frontmatter fields +--where "filepath LIKE 'blog/%' AND status = 'published'" ``` ## Nested objects @@ -256,7 +266,7 @@ Searched "sensor" — 2 hits ┌──────────────────────────┬───────────────────────────────────────────────────┐ │ query │ sensor │ ├──────────────────────────┼───────────────────────────────────────────────────┤ -│ model │ minishlab/potion-base-8M │ +│ model │ minishlab/potion-multilingual-128M │ ├──────────────────────────┼───────────────────────────────────────────────────┤ │ limit │ 10 │ └──────────────────────────┴───────────────────────────────────────────────────┘ diff --git a/crates/mdvs/Cargo.toml b/crates/mdvs/Cargo.toml index 3358cbc..da0f760 100644 --- a/crates/mdvs/Cargo.toml +++ b/crates/mdvs/Cargo.toml @@ -9,7 +9,7 @@ homepage = "https://edochi.github.io/mdvs/" readme = "../../README.md" keywords = ["markdown", "search", "embeddings", "semantic", "vector"] categories = ["command-line-utilities", "text-processing"] -include = ["src/", "skills/", "CHANGELOG.md", "Cargo.toml", "LICENSE"] +include = ["src/", "scaffolding/", "CHANGELOG.md", "Cargo.toml", "LICENSE"] [features] # Enables a deterministic mock embedder selectable via @@ -18,6 +18,27 @@ include = ["src/", "skills/", "CHANGELOG.md", "Cargo.toml", "LICENSE"] # hermetic fast-lane test suite (TODO-0184). testing-mocks = [] +# Gates the examples in `examples/` (dev-only probes that touch +# LanceDB / pipeline internals). Off by default so `cargo build +# --all-targets` and `cargo clippy --all-targets` skip them. +# Run a probe explicitly with: +# cargo run --features dev-examples --example probe_lance_incremental +dev-examples = [] + +# --- Example targets ------------------------------------------------- +# Both examples require `dev-examples` so they're never built unless +# asked. Cargo's auto-discovery still picks them up by filename; the +# `required-features` line is what excludes them from default / all- +# targets builds. + +[[example]] +name = "probe_lance_incremental" +required-features = ["dev-examples"] + +[[example]] +name = "profile_pipeline" +required-features = ["dev-examples"] + [dependencies] # CLI clap = { version = "4", features = ["derive"] } @@ -87,6 +108,19 @@ xxhash-rust = { version = "0.8.15", features = ["xxh3"] } # runtime. See TODO-0180. lazy-regex = "3" +# SQL parser for the `--where` translator. AST-based rewriting (column +# qualification, array-field equality → array_has, etc.) instead of regex +# pattern-matching, which compounds in fragility with each new rule. Pin +# matches the version Lance/DataFusion already pull in transitively, so +# rewrite output stays parseable by the downstream filter engine. +sqlparser = "0.61" + +# Embeds the `scaffolding/` directory into the binary at build time. Lets +# `scaffold::Platform::load` and `Platform::list` read bundled +# `platforms//platform.toml` files without any disk access at run +# time. +include_dir = "0.7" + [dev-dependencies] tempfile = "3" diff --git a/crates/mdvs/scaffolding/platforms/antigravity/platform.toml b/crates/mdvs/scaffolding/platforms/antigravity/platform.toml new file mode 100644 index 0000000..5db9121 --- /dev/null +++ b/crates/mdvs/scaffolding/platforms/antigravity/platform.toml @@ -0,0 +1,34 @@ +# mdvs platform config — Antigravity CLI (Google, formerly Gemini CLI). +# +# Read by: +# - `mdvs scaffold skill --platform antigravity` +# - `mdvs scaffold snippet --platform antigravity` +# - `mdvs scaffold hook --platform antigravity` → refuses with a pointer +# +# Sources: +# skills: https://codelabs.developers.google.com/getting-started-with-antigravity-skills +# inherited config: https://github.com/google-gemini/gemini-cli/tree/main/docs/hooks + +[meta] +name = "antigravity" +display_name = "Antigravity CLI" +documentation_url = "https://edochi.github.io/mdvs/recipes/agent-harnesses/antigravity.html" + +[skill] +# Antigravity reads from .agents/skills/ at project scope (Google's +# authoring codelab is explicit about this — same as Codex's path). +install_path = ".agents/skills/mdvs/SKILL.md" + +[snippet] +# Post-rebrand, Antigravity uses AGENTS.md as the workspace-level rules +# file. Pre-rebrand Gemini CLI used GEMINI.md; both formats are in flight +# during the transition. +target_file = "AGENTS.md" +body = "agents-md" + +# No [hooks] section — Antigravity's hook surface is undocumented post- +# rebrand. The inherited Gemini CLI hooks reference uses different event +# names (BeforeTool / AfterTool instead of Pre/PostToolUse), but Antigravity +# may have diverged. `mdvs scaffold hook --platform antigravity` refuses +# until upstream documentation solidifies. When that happens we can add a +# [hooks] table here without any Rust changes. diff --git a/crates/mdvs/scaffolding/platforms/claude-code/platform.toml b/crates/mdvs/scaffolding/platforms/claude-code/platform.toml new file mode 100644 index 0000000..f5d1499 --- /dev/null +++ b/crates/mdvs/scaffolding/platforms/claude-code/platform.toml @@ -0,0 +1,63 @@ +# mdvs platform config — Claude Code. +# +# Read by: +# - `mdvs scaffold skill --platform claude-code` (install_path → help text) +# - `mdvs scaffold snippet --platform claude-code` (target_file + body) +# - `mdvs scaffold hook --platform claude-code` (substitutes hooks.config) +# - `mdvs hook handle --platform claude-code --kind ` +# (substitutes hooks.envelope) +# +# Sources: +# skills: https://code.claude.com/docs/en/skills +# hooks: https://code.claude.com/docs/en/hooks + +[meta] +name = "claude-code" +display_name = "Claude Code" +documentation_url = "https://code.claude.com/docs/en/hooks" + +[skill] +# Claude Code reads skills only from .claude/skills/ (does not honor the +# cross-harness .agents/skills/ convention). +install_path = ".claude/skills/mdvs/SKILL.md" + +[snippet] +# Claude Code reads project rules from CLAUDE.md at workspace root. +target_file = "CLAUDE.md" +body = "agents-md" + +[hooks] +config_path = ".claude/settings.json" +config_format = "json" + +[hooks.envelope] +# Emitted by `mdvs hook handle` after a fired hook. Placeholders: +# <> — agent-context message (always populated) +# <> — user-visible message; populated for validate, pruned +# for search-nudge (search-nudge stays out of the user UI). +template = """ +{ + "hookSpecificOutput": { + "hookEventName": "PostToolUse", + "additionalContext": "<>" + }, + "systemMessage": "<>" +} +""" + +[hooks.config] +# Emitted by `mdvs scaffold hook`. Placeholders: +# <> — full `mdvs hook handle … --kind validate` line +# <> — full `mdvs hook handle … --kind search-nudge` line +template = """ +{ + "hooks": { + "PostToolUse": [ + { "matcher": "Edit|Write|MultiEdit", + "hooks": [{ "type": "command", "command": "<>" }] }, + { "matcher": "Bash", + "hooks": [{ "type": "command", "command": "<>" }] } + ] + } +} +""" diff --git a/crates/mdvs/scaffolding/platforms/codex/platform.toml b/crates/mdvs/scaffolding/platforms/codex/platform.toml new file mode 100644 index 0000000..55446b2 --- /dev/null +++ b/crates/mdvs/scaffolding/platforms/codex/platform.toml @@ -0,0 +1,25 @@ +# mdvs platform config — Codex (OpenAI Codex CLI). +# +# Sources: +# skills: https://developers.openai.com/codex/skills/ +# hooks: https://developers.openai.com/codex/hooks +# AGENTS.md: https://developers.openai.com/codex/guides/agents-md +# +# No [hooks] section — mdvs doesn't ship a verified hook config for Codex. +# `mdvs scaffold hook --platform codex` refuses with a pointer at the per- +# platform mdbook page. Skill and snippet install normally. + +[meta] +name = "codex" +display_name = "Codex" +documentation_url = "https://edochi.github.io/mdvs/recipes/agent-harnesses/codex.html" + +[skill] +# Codex reads from the cross-harness .agents/skills/ path. +install_path = ".agents/skills/mdvs/SKILL.md" + +[snippet] +# Codex reads project rules from AGENTS.md (or AGENTS.override.md, which +# takes precedence). +target_file = "AGENTS.md" +body = "agents-md" diff --git a/crates/mdvs/scaffolding/platforms/cursor/platform.toml b/crates/mdvs/scaffolding/platforms/cursor/platform.toml new file mode 100644 index 0000000..d919e54 --- /dev/null +++ b/crates/mdvs/scaffolding/platforms/cursor/platform.toml @@ -0,0 +1,24 @@ +# mdvs platform config — Cursor. +# +# Sources: +# skills: https://cursor.com/docs/context/skills +# rules: https://cursor.com/docs/rules +# hooks: https://cursor.com/docs/hooks +# +# No [hooks] section — mdvs doesn't ship a verified hook config for Cursor. +# `mdvs scaffold hook --platform cursor` refuses with a pointer at the per- +# platform mdbook page. Skill and snippet install normally. + +[meta] +name = "cursor" +display_name = "Cursor" +documentation_url = "https://edochi.github.io/mdvs/recipes/agent-harnesses/cursor.html" + +[skill] +install_path = ".cursor/skills/mdvs/SKILL.md" + +[snippet] +# Cursor's idiomatic always-on rules slot is .cursor/rules/.mdc with +# `alwaysApply: true` in frontmatter. We use the .mdc body by default. +target_file = ".cursor/rules/mdvs.mdc" +body = "cursor-rules" diff --git a/crates/mdvs/scaffolding/platforms/opencode/platform.toml b/crates/mdvs/scaffolding/platforms/opencode/platform.toml new file mode 100644 index 0000000..f970f7a --- /dev/null +++ b/crates/mdvs/scaffolding/platforms/opencode/platform.toml @@ -0,0 +1,36 @@ +# mdvs platform config — OpenCode. +# +# Read by: +# - `mdvs scaffold skill --platform opencode` +# - `mdvs scaffold snippet --platform opencode` +# - `mdvs scaffold hook --platform opencode` → refuses with a pointer +# +# Sources: +# skills: https://opencode.ai/docs/skills/ +# agents: https://opencode.ai/docs/agents/ +# rules: https://opencode.ai/docs/rules/ + +[meta] +name = "opencode" +display_name = "OpenCode" +documentation_url = "https://edochi.github.io/mdvs/recipes/agent-harnesses/opencode.html" + +[skill] +# OpenCode reads from .opencode/skills/ natively, plus .claude/skills/ and +# .agents/skills/ for cross-harness compatibility. We use the native path +# as the default install location. +install_path = ".opencode/skills/mdvs/SKILL.md" + +[snippet] +# OpenCode reads project rules from AGENTS.md (also CLAUDE.md as a +# Claude-Code-compat fallback). +target_file = "AGENTS.md" +body = "agents-md" + +# No [hooks] section — OpenCode's hook surface is a TypeScript plugin API +# (tool.execute.before / tool.execute.after), not a shell-command config +# file. `mdvs scaffold hook --platform opencode` refuses with a pointer to +# the recipe page section that describes the community OpenCode-Hooks +# shell-command package (https://github.com/KristjanPikhof/OpenCode-Hooks). +# When/if OpenCode adds a first-class shell-hook config we can fill in a +# [hooks] table here and the rest of the wiring takes care of itself. diff --git a/crates/mdvs/scaffolding/skill/SKILL.md b/crates/mdvs/scaffolding/skill/SKILL.md new file mode 100644 index 0000000..969e27c --- /dev/null +++ b/crates/mdvs/scaffolding/skill/SKILL.md @@ -0,0 +1,463 @@ +--- +name: mdvs +description: >- + Use `mdvs search` for any content lookup in a markdown directory — + semantic / hybrid / SQL-filtered, beats Grep / Glob for finding by + meaning. `mdvs init` infers a schema from existing markdown, `mdvs + check` validates frontmatter, `mdvs update` evolves the schema as + the KB grows. Activate whenever the project contains markdown with + frontmatter, whether or not `mdvs.toml` exists yet. +--- + +# mdvs — Markdown Validation & Search + +A CLI that treats a markdown directory as a database: schema inference, typed frontmatter validation, and semantic / full-text / hybrid search with SQL filters. Single binary, no external services. Full documentation at . + +## Usage + +``` +mdvs init [path] # infer schema, write mdvs.toml +mdvs init [path] --dry-run # preview inference +mdvs init [path] --force # overwrite existing config +mdvs init [path] --from-jsonschema # import schema from JSON Schema 2020-12 +mdvs init [path] --ignore-bare-files # exclude files with no frontmatter + +mdvs check [path] # validate frontmatter against mdvs.toml +mdvs check [path] --jsonschema # override [fields] for this run +mdvs check [path] --no-update # skip auto-update before validating + +mdvs update [path] # detect and add new fields +mdvs update reinfer [path] # re-infer type and constraints +mdvs update reinfer [path] --dry-run # preview reinfer +mdvs update reinfer [path] --with= + +mdvs build [path] # validate + chunk + embed → Lance index +mdvs build [path] --force # full rebuild (ignore incremental cache) + +mdvs search "" [path] # hybrid search (default) +mdvs search "" [path] --mode +mdvs search "" [path] --where "" # frontmatter filter +mdvs search "" [path] --limit # default 10 +mdvs search "" [path] -v # show matching chunk text +mdvs search "" [path] --no-build # fail if no index exists +mdvs search "" [path] --no-update # skip auto-update before build + +mdvs info [path] # config + index status +mdvs info [path] -v # full field detail + +mdvs clean [path] # delete .mdvs/ + +mdvs export-jsonschema [path] # emit canonical JSON Schema +mdvs export-jsonschema [path] --format +mdvs export-jsonschema [path] --output-file + +mdvs scaffold skill [--platform ] # this skill file +mdvs scaffold snippet [--platform ] # AGENTS.md / CLAUDE.md snippet +mdvs scaffold hook --platform # PostToolUse hook config (JSON snippet) +``` + +All commands take `--output `. **For agent context, pass `--output markdown` explicitly** or set `default_output_format = "markdown"` in `mdvs.toml`. `` defaults to `.` for every command. + +## What mdvs is for + +mdvs treats a markdown directory as a database: it infers a typed schema from frontmatter, validates that schema on every edit, and searches the content semantically. The schema (`mdvs.toml`) is the source of truth; everything else (search index, validation reports) is derived. The schema is meant to **evolve with the KB** as conventions emerge — mdvs makes deviations visible without freezing them. + +## What you must do when invoked + +### Step 1 — Detect mdvs context + +Look for `mdvs.toml` in the current working directory or any ancestor. + +- **If found**, that directory (or its parent containing the file) is an mdvs vault. Treat any markdown file under it as living under that schema. +- **If not found** but the project has markdown files with frontmatter, mdvs is still relevant — propose bootstrapping with `mdvs init ` before reaching for Grep on the markdown. + +### Step 2 — For any content lookup, use `mdvs search` first + +When the user (or your own task) needs to find something in markdown content — a note about a topic, a project status, a person, an experiment, anything — **default to `mdvs search ""`, not Grep / Glob**. mdvs is built exactly for this: + +- `--mode hybrid` (default) — semantic + BM25 reranked; best general-purpose mode +- `--mode semantic` — vector only; best for "find notes about X" where X is a concept, not a phrase +- `--mode fulltext` — BM25 only; for known literal phrases +- `--where "field = 'value'"` — filter by frontmatter (SQL syntax) +- `--limit N` — cap result count (default 10) +- `-v` — show the best matching chunk text per result + +Only fall back to Grep when (a) you need a literal substring match and already know which file to look in, or (b) `mdvs search` returns no results and you suspect missing content rather than missing relevance. + +### Step 3 — For frontmatter changes, check then evolve + +After any edit that touches frontmatter (yours or the user's): + +1. Run `mdvs check` to validate against the schema. +2. If **violations** appear: see Step 4 (the schema-evolution loop). +3. If **new fields** appear (present in files but not in `mdvs.toml`): they show up as informational, not violations. Run `mdvs update` to add them, or `mdvs update reinfer ` to refresh a specific field's constraints. + +### Step 4 — When a hook surfaces a violation, follow the schema-evolution loop + +If a markdown block lands in your context from a `PostToolUse` hook listing `MissingRequired` / `WrongType` / `Disallowed` / `InvalidCategory` / `OutOfRange` violations, **that's mdvs talking to you via the validation hook**. Claude Code surfaces it under `additionalContext`; other harnesses use their own channel (the wiring is harness-specific). The hook is non-blocking by design: the edit already landed; the warning is for you to act on next. + +Your job: + +1. Read the violation block: which file, which field, which rule, expected vs actual. +2. Decide: **mistake** (typo, wrong type by accident, dropped required field) or **intentional** (KB is evolving, category needs a new variant, field shifting type)? +3. **Mistake** → fix the file in the next turn. Acknowledge briefly so the user knows the loop is working. +4. **Intentional** → surface the deviation to the user and propose updating `mdvs.toml` (`mdvs update`, `mdvs update reinfer --with=`, or a manual edit). **Do not silently fix the file.** The user decides whether the schema or the file is the source of truth. + +A worked example of the intentional path is in the [Examples](#example--responding-to-a-hook-delivered-violation) section. + +## Rules + +- **Prefer `mdvs search` over Grep / Glob in any markdown corpus.** Inside an mdvs vault, the default content-search tool is mdvs. +- **The validation hook is a warning, not a block.** Treat violations as a prompt for discussion, not as edits to be reverted. +- **Never silently fix an intentional deviation.** Propose a schema update; let the user decide. +- **The schema is meant to evolve.** A schema that never changes is a schema that's wrong. Enforcement follows the KB's shape; it does not freeze it. +- **`build` always runs `check` first.** Validation gates the index build. +- **`mdvs.toml` is the only source of truth.** `.mdvs/` is derived — gitignored, recreatable with `mdvs build`. +- **There is no lock file.** Schema changes flow through `mdvs.toml` only. +- **Frontmatter formats are auto-detected** per file (YAML / TOML / JSON). A single vault can mix all three. +- **Use `--output markdown` for any output you intend to read.** It's the format LLMs parse most fluently, and the format the validation hook surfaces back through the harness's model-context channel. + +## Two layers + +mdvs has two independent layers: + +1. **Validation** (`init`, `check`, `update`) — works immediately, no model download, no build step. Reads markdown and validates frontmatter against `mdvs.toml`. +2. **Search** (`build`, `search`) — downloads an embedding model, chunks markdown content, builds a local LanceDB index in `.mdvs/`. + +Validation stands alone. You never need to build an index just to validate. + +## Key files + +- **`mdvs.toml`** — schema config, committed to version control. Source of truth for field types, allowed/required paths, constraints. +- **`.mdvs/`** — build artifacts (the Lance dataset under `index.lance/` plus a cached model). To be gitignored. Recreatable with `mdvs build`. Never edit directly. + +## Command reference + +### `mdvs init` + +Scans markdown files, infers a typed schema from frontmatter, writes `mdvs.toml`. + +- `--force` — overwrite an existing `mdvs.toml` (deletes `.mdvs/` too) +- `--dry-run` — show what would be inferred without writing +- `--ignore-bare-files` — exclude files that have no frontmatter +- `--from-jsonschema PATH` — import schema from an external JSON Schema 2020-12 document. Round-trips with `mdvs export-jsonschema`. + +Use `init --force` to start over. Use `update` to incrementally add new fields. + +### `mdvs check` + +Validates all frontmatter against `mdvs.toml`. Reports violation kinds: + +- **`MissingRequired`** — required field is absent from a file +- **`WrongType`** — value doesn't match declared type +- **`Disallowed`** — field appears in a path not covered by its `allowed` globs +- **`InvalidCategory`** — value is not in the declared category list +- **`OutOfRange`** — numeric value outside declared `min`/`max` + +New fields (in files but not in `mdvs.toml`) are reported separately as informational — no non-zero exit. Run `update` to add them. + +- `--jsonschema PATH` — override `[fields]` in `mdvs.toml` for this run + +Violation output is deterministic: sorted by `(field, kind, rule)` and `path` within. + +### `mdvs update` + +Re-scans files; adds newly discovered fields to `mdvs.toml`. Doesn't remove or change existing fields by default. + +- `mdvs update` — detect and add new fields +- `mdvs update reinfer ` — re-infer type and constraints +- `mdvs update reinfer --dry-run` — preview +- `mdvs update reinfer --with=categorical` — force categorical +- `mdvs update reinfer --with=range` — infer min/max +- `mdvs update reinfer --with=none` — strip all constraints + +Use `reinfer` when a field's type has changed or you want to refresh its constraints. `--with` requires a named field. + +### `mdvs build` + +Validates, then chunks markdown, generates embeddings, writes the Lance dataset to `.mdvs/`. + +- `--force` — full rebuild (ignore incremental cache) +- Incremental by default — only re-embeds new or edited files +- Aborts if `check` finds violations + +First build downloads the default embedding model `minishlab/potion-multilingual-128M` (~480 MB, 101 languages). Subsequent builds reuse it. + +### `mdvs search` + +Searches the indexed notes — semantic (vector), full-text (BM25), or hybrid (RRF reranker). Auto-builds the index if needed. + +```bash +mdvs search "" [path] [--mode ] [--where ""] [--limit N] [-v] +``` + +`--where` operates on **any column** in the Lance index: frontmatter fields (auto-discovered from `mdvs.toml`, referenced by bare name) and the always-present `filepath` column. Field names with spaces need double-quote escaping: `--where "\"lab section\" = 'Photonics'"`. Filtering on `Array(Float)` fields is rejected up front (Lance can't safely decode them); store as parallel scalar arrays. + +#### Scalar frontmatter — equality, inequality, comparison + +```bash +--where "author = 'Federica Bianchi'" # string equality +--where "year >= 2020" # numeric comparison +--where "year != 2025" # inequality +--where "year BETWEEN 2018 AND 2024" # closed range +--where "year IN (2021, 2022, 2023)" # discrete set +--where "year NOT IN (2020, 2024)" # exclusion +``` + +#### Strings — pattern matching with LIKE + +`%` matches any sequence, `_` matches one character. + +```bash +--where "title LIKE 'Async%'" # starts with "Async" +--where "title LIKE '%network%'" # contains "network" +--where "title NOT LIKE '%Tutorial%'" # excludes the word +--where "lower(title) LIKE '%rust%'" # case-insensitive (via the lower() function) +``` + +#### Null filters + +```bash +--where "author IS NOT NULL" # field must be set +--where "url IS NULL" # field must be absent +``` + +#### Array fields — auto-rewritten to `array_has(...)` + +The `=` / `!=` / `IN` / `NOT IN` operators against an array field auto-rewrite to `array_has(...)` so element-containment "just works". The search output shows the rewrite as a one-line `Note` at the top. + +```bash +--where "tags = 'rust'" # has 'rust' as one of its tags +--where "tags != 'archived'" # does NOT have 'archived' as a tag +--where "tags IN ('rust', 'python', 'go')" # has at least one of these +--where "tags NOT IN ('archived', 'draft')" # has NONE of these +--where "tags = 'rust' AND tags = 'async'" # has BOTH (two array_has, AND'd) +--where "tags = 'rust' OR tags = 'python'" # equivalent to IN, longer form +--where "array_has(tags, 'rust')" # the explicit form — bypasses the rewrite +``` + +#### Date fields + +Date literals use the `date '...'` keyword form. RFC 3339 datetimes use `timestamp '...'`. + +```bash +--where "published > date '2024-01-01'" +--where "created BETWEEN date '2024-01-01' AND date '2024-12-31'" +--where "published >= date '2024-01-01' AND published < date '2025-01-01'" +``` + +#### Path filtering — the always-present `filepath` column + +The `filepath` column stores the path relative to the project root — the **last component is the filename** (e.g. `articles/long-essay-2024.md` → filename is `long-essay-2024.md`). + +```bash +--where "filepath LIKE 'articles/%'" # everything under articles/ +--where "filepath LIKE '%/notes/%'" # any directory called notes/ +--where "filepath LIKE '%-postmortem.md'" # filename suffix +--where "filepath = 'articles/foo.md'" # exact path +``` + +#### Combining filters — AND, OR, NOT, parentheses + +```bash +--where "year > 2020 AND tags = 'rust'" +--where "year > 2020 AND (tags = 'rust' OR tags = 'python')" +--where "(author = 'Federica Bianchi' OR author = 'Lorenzo Conti') AND year > 2020" +--where "tags = 'rust' AND filepath LIKE 'technologies/%'" # array + path +--where "concepts != 'CRDT' AND filepath LIKE 'technologies/%'" +``` + +#### Functions + +Most DataFusion scalar functions work — handy when the raw value doesn't quite match. + +```bash +--where "length(title) > 50" # long titles +--where "lower(author) LIKE '%bianchi%'" # case-insensitive contains +--where "year + 1 > 2025" # arithmetic +``` + +Other internal columns exist (`start_line`, `end_line`, `built_at`, `chunk_text`) but are rarely useful for filtering — semantic / fulltext search handles those concerns better. + +### `mdvs info` + +Shows current config and index status: scan settings, field definitions, build metadata (model, chunk size, file counts). `-v` for full field detail. + +### `mdvs clean` + +Deletes `.mdvs/`. Doesn't touch `mdvs.toml`. + +### `mdvs export-jsonschema` + +Translates `[fields]` into a canonical JSON Schema 2020-12 document. Useful for sharing with other tools, or round-tripping through `mdvs init --from-jsonschema`. + +- `--format json|toml` — output format (default `json`) +- `--output-file FILE` — write to file instead of stdout + +### `mdvs scaffold` + +Emits the artifacts that integrate mdvs with an agent harness (Claude Code, Codex, OpenCode, Cursor, Antigravity). Each subcommand prints to stdout; pipe it into the right location for your harness. + +- `mdvs scaffold skill [--platform ]` — this skill file. Default destination: `.agents/skills/mdvs/SKILL.md` for Codex / OpenCode / Cursor / Antigravity; `.claude/skills/mdvs/SKILL.md` for Claude Code. +- `mdvs scaffold snippet [--platform ]` — the project-rules snippet for `AGENTS.md` / `CLAUDE.md` / `.cursor/rules/mdvs.mdc`. +- `mdvs scaffold hook --platform claude-code` — the `PostToolUse` hook config (a JSON snippet to merge into `.claude/settings.json`). The emitted snippet's `command:` fields call `mdvs hook handle` directly — no shell scripts, no `jq` dependency. **Currently the only verified hook integration.** The other platforms refuse with a pointer to their per-platform mdbook page; wiring `mdvs hook handle` into Codex / Cursor / OpenCode / Antigravity is possible by following each harness's own hooks documentation. + +## Agent-harness integration + +mdvs ships as a CLI; integrating it with an agent harness is wiring rather than installing. Three artifacts cover the three integration points: + +| Artifact | Purpose | Coverage | +|---|---|---| +| **Skill file** (`SKILL.md`, this file) | Activated by harnesses implementing the [Agent Skills open standard](https://agentskills.io). Loaded on demand; agent reads procedure + reference. | Works in any harness that loads `.md` skills (Claude Code, Codex, Cursor, OpenCode, Antigravity, …). | +| **Project-rules snippet** | Always-on text in `AGENTS.md` / `CLAUDE.md` / `.cursor/rules/mdvs.mdc`. Short — names the KB, the search-vs-Grep preference, and the warning-loop rule. | Works in any harness that reads `AGENTS.md` / `CLAUDE.md` / `.cursor/rules`. | +| **`PostToolUse` hook** | Calls `mdvs hook handle` after every Edit / Write on a markdown file inside the vault. mdvs walks up to find `mdvs.toml`, runs `check`, and surfaces violations to you as **non-blocking** model-context. | **Shipped for Claude Code only.** Other harnesses: follow their documented hook system to wire `mdvs hook handle` in — see . | + +## Output format + +Three formats; default is `pretty`. + +- `--output pretty` — box-drawing tables for terminal display. Adapts to width. +- `--output markdown` — GFM tables and `##` headers. **Best for agent consumption** and the format the validation hook surfaces back through the harness's model-context channel. +- `--output json` — structured JSON for programmatic extraction (pipe through `jq` or any JSON tool). + +Priority chain when `--output` is omitted: CLI flag > `default_output_format` in `mdvs.toml` > hard default (`pretty`). Same command always produces the same output regardless of TTY state. `-v` (verbose) adds per-step pipeline output with timings. + +## Exit codes + +- **0** — success (no violations) +- **1** — violations found (`check` and `build`) +- **2** — error (bad config, missing files, model mismatch) + +Hook scripts use `|| true` to mask the exit-1 from `check` — the hook is intentionally non-blocking. Don't change that contract. + +## Things to know + +- Field types inferred automatically: `String`, `Integer`, `Float`, `Boolean`, `Date` (`YYYY-MM-DD`), `DateTime` (RFC 3339 with mandatory timezone), and `Array()` for any scalar type. The on-disk grammar is `Scalar | Array(Scalar)` only. Mixed scalar types widen (`Integer + String → String`). +- **Nested frontmatter uses dotted-name leaves.** `calibration.baseline.wavelength: 850.0` becomes a `[[fields.field]]` named `"calibration.baseline.wavelength"` of type `Float`. Top-level `Object` is rejected; nested `Array(Object{...})` is also rejected — represent arrays of structured items as parallel scalar arrays (e.g. `measurement_timestamps: Array(String)` + `measurement_values: Array(Float)`). SQL filters use dot notation: `--where "calibration.baseline.wavelength > 800"`. +- **Preprocessors opt into widening.** Each field carries a `preprocess` array. Built-ins: `coerce_to_string` (accepts non-string scalars on a `String` field) and `widen_int_to_float` (accepts integers on a `Float` field). Inference auto-populates these when widening was observed. `preprocess = []` means strict. +- Categorical detection is automatic for low-cardinality repeated values. Out-of-category values → `InvalidCategory`. +- Constraint kinds: `categories` (closed-set enum, mutually exclusive with everything else), `min`/`max` (numeric range), `min_length`/`max_length` (string and array length), `pattern` (regex on strings). Range / length / pattern are not auto-inferred; add manually or via `update reinfer --with=range`. +- `init --force` rewrites the config from scratch. `update` preserves existing config and only adds new fields. `update reinfer` re-infers specific fields. +- Model identity is tracked: changing the model in `mdvs.toml` requires `build --force` to confirm a full re-embed. +- `check` auto-runs `update` first by default (unless `--no-update`, or `--jsonschema` is given, or `[check].auto_update = false` is set in `mdvs.toml`). +- `search` auto-runs `update` and `build` if needed (unless `--no-update` / `--no-build`). +- **`.mdvsignore` and `.gitignore` are both honored** when scanning. mdvs reuses the [`ignore`](https://crates.io/crates/ignore) crate (same matcher git uses), so the per-directory `.gitignore` rules you already have apply automatically. For mdvs-only exclusions (e.g. "don't index this draft directory, but keep it tracked in git"), drop a `.mdvsignore` next to the files using the same syntax as `.gitignore`. Both apply to `init` / `update` / `check` / `build` / `search`. To stop honoring `.gitignore` (for example, to index files that are deliberately untracked but should still be searchable), set `[scan].skip_gitignore = true` in `mdvs.toml`. Hidden files and dotfiles are NOT skipped — only `.md` / `.markdown` extensions are scanned to begin with. + +## Examples + +### Setting up a new vault from scratch + +```bash +mdvs init # scan current directory, infer schema, write mdvs.toml +mdvs init ~/notes # or point at another directory +mdvs init --dry-run # preview without writing +``` + +### User added a new field to some files + +```bash +mdvs check # → reports "category" as a new field (informational) +mdvs update # → detects "category", adds it to mdvs.toml +mdvs check # → clean +``` + +### Fixing violations after `check` + +```bash +mdvs check +# → +# MissingRequired: "title" missing in blog/drafts/untitled.md +# WrongType: "priority" expected Integer, got String in projects/alpha.md +# InvalidCategory: "status" got "wip", expected one of [draft, published, archived] +# OutOfRange: "rating" got 11, expected min=1, max=5 +``` + +Resolution per kind: +- **MissingRequired** — add the field, or remove the path from `required` in `mdvs.toml` +- **WrongType** — fix the value, or `mdvs update reinfer ` if the type should change +- **InvalidCategory** — fix the value, or `mdvs update reinfer --with=categorical` to update the list +- **OutOfRange** — fix the value, or `mdvs update reinfer --with=range` to widen bounds +- **Disallowed** — remove the field from that file, or widen `allowed` globs in `mdvs.toml` + +### Example — responding to a hook-delivered violation + +You wrote `status: in_review` to `kb/projects/alpha/sprint-12.md`. The next thing in your context is a hook-surfaced violation block (the exact channel name varies per harness; the content is the same): + +``` +## Violations + +### InvalidCategory + +- `kb/projects/alpha/sprint-12.md`: field `status` — expected one of `draft | published | archived`, got `in_review` +``` + +Your response: + +> Just wrote `status: in_review` on `sprint-12.md`. mdvs's schema only allows `draft | published | archived` for `status`. Looking back, I've seen the `in_review` state come up in two other recent edits — this might be a real new lifecycle stage rather than a typo. Want me to add `in_review` to the schema (`mdvs update reinfer status --with=categorical` after updating the file with the new value), or should I change the value on `sprint-12.md` to one of the existing categories? + +Note what you did NOT do: silently `mdvs update reinfer status --with=categorical` to make the warning go away. The user decides whether the schema or the file is the source of truth. + +### Wiring mdvs into Claude Code (end-to-end) + +```bash +mdvs scaffold skill > .claude/skills/mdvs/SKILL.md # the skill file +mdvs scaffold snippet >> CLAUDE.md # the always-on rules block +mdvs scaffold hook --platform claude-code # PostToolUse hook config — read stderr for the destination +``` + +`mdvs scaffold hook` prints a JSON snippet to merge into `.claude/settings.json`. No shell scripts; the `command:` fields call `mdvs hook handle` directly. + +### Wiring mdvs into other harnesses (skill + snippet) + +For Codex, Cursor, OpenCode, or Antigravity, install the skill and snippet — the hook half isn't shipped: + +```bash +mdvs scaffold skill --platform > /mdvs/SKILL.md +mdvs scaffold snippet --platform >> +``` + +`mdvs scaffold hook --platform ` for these harnesses refuses with a pointer at , which describes how to wire `mdvs hook handle` into each harness's own hook config using that harness's documentation. + +### Searching with filters + +Full reference for `--where` is in the [search reference section](#mdvs-search) above. A few realistic shapes: + +```bash +# Recent articles by a specific author +mdvs search "actor model" --where "author = 'Federica Bianchi' AND year >= 2022" + +# Notes tagged with both 'rust' and 'async' (two array_has, AND'd) +mdvs search "cancellation" --where "tags = 'rust' AND tags = 'async'" + +# Anything published in 2024 +mdvs search "consensus" --where "published BETWEEN date '2024-01-01' AND date '2024-12-31'" + +# Restrict to a subtree, exclude archived +mdvs search "session model" --where "filepath LIKE 'projects/%' AND tags != 'archived'" + +# Either of two authors, recent +mdvs search "types" --where "(author = 'Lorenzo Conti' OR author = 'Federica Bianchi') AND year > 2020" + +# Verbose mode — shows the matching chunk text under each hit +mdvs search "calibration" -v +``` + +### Edge cases + +- **Files without frontmatter (bare files):** `init` includes them by default. Use `--ignore-bare-files` or set `include_bare_files = false` in `[scan]` to exclude. +- **Null values:** `nullable = true` accepts null. Null skips type and category checks. A `required` + `nullable` field passes with `key: null` — fails only if the key is entirely absent. +- **Mixed-type fields:** widen to `String`. `1` becomes `"1"`. Intentional, not data loss. +- **Special characters in field names:** TOML handles quoting in `mdvs.toml`. In `--where`, wrap with double quotes: `--where "\"author's note\" IS NOT NULL"`. +- **Hook fires on non-vault edits:** if no `mdvs.toml` is reachable upward from the edited file, the hook exits silently. No false positives. +- **Hook stays silent on a bad edit:** check (in order) — harness matcher pattern, file extension (`.md`), `mdvs` on PATH for the harness's hook subprocess, symlinks outside the vault. + +## Common errors + +| Error | Cause | Fix | +|---|---|---| +| `mdvs.toml already exists` | Running `init` twice | `init --force` or `update` | +| `no markdown files found` | Wrong path or glob | Check path + `[scan].glob` in config | +| `model mismatch` | Config model differs from index | `build --force` to re-embed | +| `field 'X' is not in mdvs.toml` | `reinfer` on unknown field | Check spelling, or `update` first to add it | +| Violations on `check` | Frontmatter doesn't match schema | Read the list, fix files or evolve the schema (Step 4) | +| Hook stays silent on a bad edit | See Edge cases above | Check matcher, extension, PATH, vault location | diff --git a/crates/mdvs/scaffolding/snippet/agents-md.md b/crates/mdvs/scaffolding/snippet/agents-md.md new file mode 100644 index 0000000..b267524 --- /dev/null +++ b/crates/mdvs/scaffolding/snippet/agents-md.md @@ -0,0 +1,16 @@ + + +## mdvs knowledge base + +This project uses [mdvs](https://github.com/edochi/mdvs) to manage a markdown knowledge base with a schema defined in `mdvs.toml`. When working with files in this KB: + +- **Prefer `mdvs search ""` over `Grep` / `Glob` for semantic lookups.** mdvs runs semantic / hybrid / SQL-filtered search over the KB — usually a better fit than literal pattern matching for finding notes by meaning. Filter by frontmatter with `--where "field = 'value'"` and pick the mode with `--mode semantic|fulltext|hybrid`. +- **`mdvs check` validates frontmatter against the schema.** A `PostToolUse` hook may surface violations to you automatically as a markdown block under `additionalContext` in the hook output. Treat that block as **a warning, not a block** — the agent's edit already landed; the warning is for you to act on next. +- **When a violation reaches you, decide:** + 1. **Mistake** (a typo, the wrong type by accident, a dropped required field) → fix the file in the next turn. + 2. **Intentional deviation** (the KB is evolving, a category genuinely needs a new variant, a field is shifting type) → surface the deviation to the user and propose updating `mdvs.toml` to absorb the change. **Do not silently fix the file**; the user decides whether the schema or the file is wrong. + +The schema is meant to evolve with the KB. Enforcement follows the KB's shape, it does not freeze it. Whenever you propose a schema update, also explain *why* — which file(s) drove the change and what convention the user might want to standardise on. + +Full mdvs documentation: . diff --git a/crates/mdvs/scaffolding/snippet/cursor-rules.mdc b/crates/mdvs/scaffolding/snippet/cursor-rules.mdc new file mode 100644 index 0000000..7f0bb04 --- /dev/null +++ b/crates/mdvs/scaffolding/snippet/cursor-rules.mdc @@ -0,0 +1,21 @@ +--- +description: mdvs knowledge base — search + validation contract for agents working in this project +alwaysApply: true +--- + + + +## mdvs knowledge base + +This project uses [mdvs](https://github.com/edochi/mdvs) to manage a markdown knowledge base with a schema defined in `mdvs.toml`. When working with files in this KB: + +- **Prefer `mdvs search ""` over `Grep` / `Glob` for content lookups.** mdvs runs semantic / hybrid / SQL-filtered search over the KB — usually a better fit than literal pattern matching for finding notes by meaning. Filter by frontmatter with `--where "field = 'value'"` and pick the mode with `--mode semantic|fulltext|hybrid`. +- **`mdvs check` validates frontmatter against the schema.** A `postToolUse` hook may surface violations to you automatically as a markdown block under `additionalContext` in the hook output. Treat that block as **a warning, not a block** — the agent's edit already landed; the warning is for you to act on next. +- **When a violation reaches you, decide:** + 1. **Mistake** (a typo, the wrong type by accident, a dropped required field) → fix the file in the next turn. + 2. **Intentional deviation** (the KB is evolving, a category genuinely needs a new variant, a field is shifting type) → surface the deviation to the user and propose updating `mdvs.toml` to absorb the change. **Do not silently fix the file**; the user decides whether the schema or the file is wrong. + +The schema is meant to evolve with the KB. Enforcement follows the KB's shape, it does not freeze it. Whenever you propose a schema update, also explain *why* — which file(s) drove the change and what convention the user might want to standardise on. + +Full mdvs documentation: . diff --git a/crates/mdvs/skills/mdvs/SKILL.md b/crates/mdvs/skills/mdvs/SKILL.md deleted file mode 100644 index 7f2cb98..0000000 --- a/crates/mdvs/skills/mdvs/SKILL.md +++ /dev/null @@ -1,347 +0,0 @@ ---- -name: mdvs -description: >- - Semantic search and frontmatter validation for markdown directories. - Use mdvs to find relevant notes by meaning (not just keywords), filter - results by frontmatter fields with SQL, validate schema consistency, - and detect field type or constraint violations. Use when the user asks - to search notes or docs, validate frontmatter, set up a schema for - markdown files, or when an mdvs.toml file exists in the project. ---- - -# mdvs — Markdown Validation & Search - -A CLI that treats markdown directories as databases: schema inference, frontmatter validation, and semantic search with SQL filtering. Single binary, no external services. - -**Frontmatter formats.** mdvs auto-detects YAML (`---`), TOML (`+++`), and JSON (`{...}`) per file from the leading delimiter, so a single vault can mix all three. The same schema validates them all uniformly. To force a single format vault-wide, set `[scan].frontmatter_format` in `mdvs.toml` to `"yaml"` / `"toml"` / `"json"` (default `"auto"`). - -## When to use which command - -| User intent | Command | -|---|---| -| Set up a schema for a markdown directory | `mdvs init ` | -| Validate frontmatter against the schema | `mdvs check ` | -| Re-scan after adding/changing/removing files | `mdvs update ` | -| Re-infer a field's type or constraints | `mdvs update reinfer ` | -| Build or rebuild the search index | `mdvs build ` | -| Search across notes | `mdvs search "" ` | -| Check what's configured and indexed | `mdvs info ` | -| Delete the search index | `mdvs clean ` | -| Emit the canonical JSON Schema of `mdvs.toml` | `mdvs export-jsonschema ` | -| Print this skill file to stdout | `mdvs skill` | - -`` defaults to `.` (current directory) for all commands. - -## Two layers - -mdvs has two independent layers: - -1. **Validation** (`init`, `check`, `update`) — works immediately, no model download, no build step. Reads markdown files and validates frontmatter against `mdvs.toml`. -2. **Search** (`build`, `search`) — downloads an embedding model, chunks markdown content, and builds a local LanceDB index in `.mdvs/`. - -Validation stands alone. You never need to build an index just to validate. - -## Key files - -- **`mdvs.toml`** — the schema config, committed to version control. Source of truth for field types, allowed/required paths, and constraints. Created by `init`, updated by `update`. -- **`.mdvs/`** — build artifacts (the Lance dataset under `index.lance/` plus a cached model). Gitignored. Recreatable with `mdvs build`. Never edit directly. - -## Command reference - -### `mdvs init` - -Scans markdown files, infers a typed schema from frontmatter, and writes `mdvs.toml`. - -- `--force` — overwrite an existing `mdvs.toml` (deletes `.mdvs/` too) -- `--dry-run` — show what would be inferred without writing anything -- `--ignore-bare-files` — exclude files that have no frontmatter -- `--from-jsonschema PATH` — import the schema from an external JSON Schema 2020-12 document (`.json` or `.toml`) instead of inferring from markdown. Round-trips with `mdvs export-jsonschema`. - -Use `init --force` to start over from scratch. Use `update` to incrementally add new fields. - -### `mdvs check` - -Validates all frontmatter against the schema in `mdvs.toml`. Reports violations: - -- **`MissingRequired`** — a required field is absent from a file -- **`WrongType`** — value doesn't match the declared type (e.g., string in an integer field) -- **`Disallowed`** — field appears in a file path not covered by its `allowed` globs -- **`InvalidCategory`** — value is not in the field's declared category list -- **`OutOfRange`** — numeric value is outside the declared `min`/`max` range - -New fields (present in files but not in `mdvs.toml`) are reported separately as informational — they don't cause a non-zero exit code. Run `update` to add them to the schema. - -- `--jsonschema PATH` — override the `[fields]` block in `mdvs.toml` for this run with an external JSON Schema. Useful for one-off CI validation against a stricter schema. - -Violation output is deterministic: violations are sorted by `(field, kind, rule)` and files within each violation are sorted by `path`. - -### `mdvs update` - -Re-scans files and adds newly discovered fields to `mdvs.toml`. Does not remove or change existing fields by default. - -- `mdvs update` — detect and add new fields only -- `mdvs update reinfer ` — re-infer type and constraints (heuristic defaults) -- `mdvs update reinfer --dry-run` — preview what reinfer would change -- `mdvs update reinfer --with=` — force specific constraint kinds -- `mdvs update reinfer --with=none` — strip all constraints - -The `--with` flag takes a comma-separated list of constraint kinds: `categorical`, `range`, or `none`. Examples: - -- `--with=categorical` — force categorical (skip heuristic threshold) -- `--with=range` — infer min/max from observed numeric values -- `--with=none` — strip all constraints from the field - -Incompatible kinds (like `range,categorical` on the same field) are rejected at parse time. `--with` requires named fields. - -Use `reinfer` when a field's type has changed (e.g., values evolved from integers to strings) or when you want to refresh its constraints. - -### `mdvs build` - -Validates frontmatter (runs `check` internally), then chunks markdown content, generates embeddings, and writes the Lance dataset to `.mdvs/`. - -- `--force` — full rebuild (ignore incremental cache) -- Incremental by default — only re-embeds new or edited files -- Aborts if `check` finds violations - -The first build downloads the default embedding model `minishlab/potion-base-8M` (~60 MB). Subsequent builds reuse the cached model. - -### `mdvs search` - -Search across the indexed notes — semantic (vector), full-text (BM25), or hybrid (RRF reranker over both). Requires a built index (auto-builds if needed). - -```bash -mdvs search "" [path] [--mode ] [--where ""] [--limit N] [-v] -``` - -- `--mode` — `semantic`, `fulltext`, or `hybrid` (default: `hybrid`) -- `--where` — SQL WHERE clause to filter on frontmatter fields -- `--limit` — max results (default: 10) -- `-v` — show best matching chunk text per result -- `--no-build` — skip auto-build, fail if no index exists -- `--no-update` — skip auto-update before building - -`--where` clauses on `Array(Float)` fields are rejected up front with a clear error. The workaround is to filter on a scalar field, or store the data as a parallel array of strings. - -#### `--where` filter examples - -```bash ---where "draft = false" ---where "status = 'published'" ---where "priority = 'high' AND author = 'Alice'" ---where "sample_count > 20" ---where "tags = 'rust'" # checks if array contains value ---where "status IN ('draft', 'published')" -``` - -String values use single quotes. Field names with spaces or special characters need double-quote escaping: `--where "\"lab section\" = 'Photonics'"`. - -### `mdvs info` - -Shows the current config and index status: scan settings, field definitions, build metadata (model, chunk size, file counts). Use `-v` for full field detail. - -### `mdvs clean` - -Deletes the `.mdvs/` directory. Does not touch `mdvs.toml`. - -### `mdvs export-jsonschema` - -Translates the `[fields]` block of `mdvs.toml` into a canonical JSON Schema 2020-12 document. Useful for sharing the schema with other tools, or for round-tripping through `mdvs init --from-jsonschema`. - -- `--format json|toml` — output format (default: `json`) -- `--output-file FILE` — write to a file instead of stdout - -### `mdvs skill` - -Prints this skill file to stdout. Useful for piping into another agent's context, or for confirming the skill mdvs ships matches what's installed. - -## Output format - -Three formats are available; the default is `pretty`. - -- `--output pretty` — box-drawing tables for a human-readable terminal. Adapts to terminal width. **This is the default.** -- `--output markdown` — GFM pipe tables and `##` section headers. **Best choice for agent consumption** — token-efficient and the format LLMs parse most fluently. -- `--output json` — strict structured JSON. Use this when you need to extract specific values programmatically (e.g. `mdvs check --output json | jq '.violations[]'`). - -When `--output` is omitted, mdvs picks the format using this priority chain: CLI flag > `default_output_format` in `mdvs.toml` > hard default (`pretty`). Same command always produces the same output, regardless of whether stdout is a terminal, a pipe, or captured by an agent harness. - -**For agent use**: pass `--output markdown` explicitly, or set `default_output_format = "markdown"` in the project's `mdvs.toml` once. The latter is the right choice for KBs co-maintained with an agent — every invocation produces Markdown without needing to remember the flag. - -Use `-v` (verbose) to get per-step pipeline output (with timings) in addition to the final result. Available for all three formats. - -## Exit codes - -- **0** — success (no violations) -- **1** — violations found (`check` and `build`) -- **2** — error (bad config, missing files, model mismatch, etc.) - -## Common workflows - -### First-time setup - -```bash -mdvs init # scan, infer schema, write mdvs.toml -``` - -### Ongoing validation - -```bash -mdvs check # after editing files — are they still valid? -mdvs update # after adding new frontmatter fields -mdvs check # validate again after update -``` - -### Fixing a field that was inferred wrong - -```bash -mdvs update reinfer priority --dry-run # preview heuristic re-run -mdvs update reinfer priority # apply -mdvs update reinfer priority --with=categorical # force categorical -mdvs update reinfer rating --with=range # infer min/max -mdvs update reinfer priority --with=none # strip all constraints -``` - -### Searching with filters - -```bash -mdvs search "calibration results" -mdvs search "setup" --where "draft = false" -mdvs search "budget" --where "status = 'active'" -v -mdvs search "experiment" --where "author = 'Giulia Ferretti'" --limit 5 -``` - -### CI validation - -```bash -mdvs check -# exit code 0 = all valid, 1 = violations found -``` - -## Things to know - -- `build` always runs `check` first — if validation fails, the build aborts. -- `search` auto-runs `update` and `build` if needed (unless `--no-update` or `--no-build`). -- Field types are inferred automatically: `String`, `Integer`, `Float`, `Boolean`, `Date` (`YYYY-MM-DD`), `DateTime` (RFC 3339 with mandatory timezone), `Array(T)`, `Array(Object{k: v, ...})`. Mixed scalar types widen (e.g., `Integer` + `String` becomes `String`). -- **Nested frontmatter uses dotted-name leaves.** A YAML key like `calibration.baseline.wavelength: 850.0` becomes a `[[fields.field]]` named `"calibration.baseline.wavelength"` of type `Float`. Top-level `Object` is rejected at config load — there are no nested `[[fields.field]]` blocks, only flat dotted names. `Array(Object{...})` is the one inline-Object form that stays. SQL filters use dot notation natively: `--where "calibration.baseline.wavelength > 800"`. -- **Preprocessors opt into widening.** Each `[[fields.field]]` carries a `preprocess` array. Two built-ins exist: `coerce_to_string` (accepts non-string scalars on a `String` field and stringifies them) and `widen_int_to_float` (accepts integers on a `Float` field). Inference auto-populates these when widening was observed. `preprocess = []` means strict — without the opt-in, a `Float` field rejects integer-backed numbers and a `String` field rejects bools/numbers. -- Fields with low-cardinality repeated values are automatically detected as categorical (e.g., `status: draft/published/archived`). Out-of-category values are reported as `InvalidCategory` violations. -- Constraint kinds available per type: `categories` (closed-set enum), `min`/`max` (numeric range), `min_length`/`max_length` (string and array length), `pattern` (regex on strings). Categorical is mutually exclusive with everything else. Range / length / pattern are not auto-inferred but can be added manually, or inferred on demand with `update reinfer --with=range`. -- `init --force` rewrites the entire config from scratch. `update` preserves existing config and only adds new fields. `update reinfer` re-infers specific fields. -- The model identity is tracked: if you change the model in `mdvs.toml`, `build` and `search` will require `--force` to confirm a full re-embed. -- `mdvs.toml` is the complete source of truth. There is no lock file. - -## Examples - -### Setting up a new vault from scratch - -```bash -# User has a directory of markdown notes and wants to add schema validation -mdvs init ~/notes -# → Scans files, infers fields (title: String, tags: Array(String), draft: Boolean, ...), -# writes ~/notes/mdvs.toml. Check runs automatically. - -# Preview what init would infer without writing anything -mdvs init ~/notes --dry-run -``` - -### User added a new field to some files - -The user started adding `category: tutorial` to blog posts. mdvs doesn't know about it yet. - -```bash -mdvs check ~/notes -# → Reports "category" as a new field (informational, not a violation) - -mdvs update ~/notes -# → Detects "category", adds it to mdvs.toml with inferred type, allowed/required paths - -mdvs check ~/notes -# → Clean — category is now part of the schema -``` - -### Fixing violations after check - -```bash -mdvs check ~/notes -# Output shows: -# MissingRequired: "title" missing in blog/drafts/untitled.md -# WrongType: "priority" expected Integer, got String in projects/alpha.md -# InvalidCategory: "status" got "wip", expected one of [draft, published, archived] -# OutOfRange: "rating" got 11, expected min=1, max=5 -``` - -For each violation type: -- **MissingRequired** — add the field to the file, or remove the path from `required` in `mdvs.toml` -- **WrongType** — fix the value in the file, or reinfer the field if the type should change: `mdvs update reinfer priority` -- **InvalidCategory** — fix the value in the file, or reinfer to update the category list: `mdvs update reinfer status --with=categorical` -- **OutOfRange** — fix the value in the file, or update the bounds: `mdvs update reinfer rating --with=range` -- **Disallowed** — the field shouldn't be in that file path. Remove it from the file, or widen the `allowed` globs in `mdvs.toml` - -### Searching with different filter patterns - -```bash -# Find all notes about "machine learning" that are published -mdvs search "machine learning" --where "status = 'published'" - -# Find high-priority items by a specific author -mdvs search "deadline" --where "priority = 'high' AND author = 'Alice'" - -# Numeric comparison -mdvs search "experiment" --where "sample_count >= 100" - -# Check if an array field contains a value -mdvs search "tutorial" --where "tags = 'beginner'" - -# Multiple possible values -mdvs search "update" --where "status IN ('draft', 'review')" - -# Verbose output — shows the matching text chunk from each result -mdvs search "calibration" -v -``` - -### Rebuilding after config changes - -```bash -# User changed the model in mdvs.toml -mdvs build ~/notes -# → Error: model mismatch (config model differs from existing index) - -mdvs build ~/notes --force -# → Full rebuild with the new model -``` - -### Working with subdirectories - -mdvs schemas are directory-scoped. A large vault might have different schemas for different sections: - -```bash -# Initialize just the blog section -mdvs init ~/notes/blog --glob "**" - -# Initialize the whole vault -mdvs init ~/notes - -# Search only within projects -mdvs search "budget" ~/notes/projects -``` - -### Edge cases - -**Files without frontmatter (bare files):** By default, `init` includes bare files in the scan. They have no fields, which affects inference (e.g., a field can't be `required` for `**` if bare files exist). Use `--ignore-bare-files` to exclude them, or set `include_bare_files = false` in `[scan]`. - -**Null values:** A field with `nullable = true` accepts null values. Null skips type and category checks. A field that is `required` but `nullable` passes if the key is present with a null value — it only fails if the key is entirely absent. - -**Mixed-type fields:** If a field has integers in some files and strings in others, it widens to `String`. The integer values are stored as their string representation (e.g., `1` becomes `"1"`). This is not data loss — it's intentional widening. - -**Field names with special characters:** Frontmatter keys with spaces, quotes, or other special characters work fine in `mdvs.toml` (TOML handles quoting). In `--where` clauses, wrap them in double quotes: `--where "\"author's note\" IS NOT NULL"`. - -**Large vaults:** Scanning and validation are fast (reads all files every time, no incremental scan). Embedding is the expensive part — builds are incremental by default, only re-embedding new or changed files. - -## Common errors - -| Error | Cause | Fix | -|---|---|---| -| `mdvs.toml already exists` | Running `init` twice | Use `init --force` or `update` | -| `no markdown files found` | Wrong path or glob | Check the path and `[scan].glob` in config | -| `model mismatch` | Config model differs from index | Run `build --force` to re-embed | -| `field 'X' is not in mdvs.toml` | `reinfer` on unknown field | Check spelling, or run `update` first to add it | -| violations on `check` | Frontmatter doesn't match schema | Read the violation list, fix files or adjust schema | diff --git a/crates/mdvs/src/cmd/build/config_mutate.rs b/crates/mdvs/src/cmd/build/config_mutate.rs index 16efc98..544df3f 100644 --- a/crates/mdvs/src/cmd/build/config_mutate.rs +++ b/crates/mdvs/src/cmd/build/config_mutate.rs @@ -18,7 +18,7 @@ use std::path::Path; // Unused under `cfg(any(test, feature = "testing-mocks"))` since the // default falls back to the mock embedder in that build flavor. #[cfg_attr(any(test, feature = "testing-mocks"), allow(dead_code))] -const DEFAULT_MODEL: &str = "minishlab/potion-base-8M"; +const DEFAULT_MODEL: &str = "minishlab/potion-multilingual-128M"; pub(super) const DEFAULT_CHUNK_SIZE: usize = 1024; /// Normalize a revision string: empty and "None" are treated as unset. @@ -45,26 +45,30 @@ pub(crate) fn mutate_config( match config.embedding_model { None => { - // With `--features testing-mocks` (CI fast lane), default to the - // deterministic mock embedder so tests that flow through init → - // build don't reach for Hugging Face. The feature is off in - // production builds, so end users always get model2vec. + // Production: write the real default to mdvs.toml so subsequent + // runs are deterministic. Test/mock builds: synthesize the mock + // default in-memory only — never persist it, or a dev binary + // running against a real vault corrupts the vault for the + // production binary on PATH. #[cfg(any(test, feature = "testing-mocks"))] - let default = EmbeddingModelConfig { - provider: "mock".to_string(), - name: set_model.unwrap_or("mock").to_string(), - revision: set_revision.and_then(normalize_revision), - dim: Some(256), - }; + { + config.embedding_model = Some(EmbeddingModelConfig { + provider: "mock".to_string(), + name: set_model.unwrap_or("mock").to_string(), + revision: set_revision.and_then(normalize_revision), + dim: Some(256), + }); + } #[cfg(not(any(test, feature = "testing-mocks")))] - let default = EmbeddingModelConfig { - provider: "model2vec".to_string(), - name: set_model.unwrap_or(DEFAULT_MODEL).to_string(), - revision: set_revision.and_then(normalize_revision), - dim: None, - }; - config.embedding_model = Some(default); - config_changed = true; + { + config.embedding_model = Some(EmbeddingModelConfig { + provider: "model2vec".to_string(), + name: set_model.unwrap_or(DEFAULT_MODEL).to_string(), + revision: set_revision.and_then(normalize_revision), + dim: None, + }); + config_changed = true; + } } Some(ref mut em) if set_model.is_some() || set_revision.is_some() => { if !force { @@ -195,4 +199,64 @@ mod tests { assert_eq!(normalize_revision("abc123"), Some("abc123".to_string())); assert_eq!(normalize_revision("not_none"), Some("not_none".to_string())); } + + /// Under `cfg(test)` the in-memory embedding default is `mock`, but it + /// must never reach disk — otherwise a dev binary running auto-build + /// against a real vault would write `provider = "mock"` into the user's + /// mdvs.toml and break that vault for the production binary on PATH. + #[test] + fn mock_embedder_default_is_not_persisted_to_mdvs_toml() { + let tmp = tempfile::tempdir().unwrap(); + let path = tmp.path(); + let config_path = path.join("mdvs.toml"); + + // Start with all build sections present except [embedding_model], + // so we isolate the mock-default branch from search/build/chunking + // defaults that legitimately persist. + let original = r#"[scan] +glob = "**" +include_bare_files = true +skip_gitignore = false +frontmatter_format = "auto" + +[check] +auto_update = true + +[chunking] +max_chunk_size = 1024 + +[build] +auto_update = true + +[search] +default_limit = 10 +auto_update = true +auto_build = true +internal_prefix = "" +aliases = {} + +[fields] +ignore = [] +"#; + std::fs::write(&config_path, original).unwrap(); + + let mut config: MdvsToml = toml::from_str(original).unwrap(); + let err = mutate_config(&mut config, path, None, None, None, false); + assert!(err.is_none(), "mutate_config errored: {err:?}"); + + // In-memory: mock was synthesized so the rest of the build chain works. + let em = config.embedding_model.as_ref().expect("in-memory default"); + assert_eq!(em.provider, "mock"); + + // On disk: file must be unchanged — no [embedding_model], no mock. + let on_disk = std::fs::read_to_string(&config_path).unwrap(); + assert!( + !on_disk.contains("mock"), + "mdvs.toml leaked mock embedder to disk:\n{on_disk}" + ); + assert!( + !on_disk.contains("[embedding_model]"), + "mdvs.toml gained an [embedding_model] block:\n{on_disk}" + ); + } } diff --git a/crates/mdvs/src/cmd/hook/handle.rs b/crates/mdvs/src/cmd/hook/handle.rs new file mode 100644 index 0000000..543a0fd --- /dev/null +++ b/crates/mdvs/src/cmd/hook/handle.rs @@ -0,0 +1,695 @@ +//! `mdvs hook handle` — runtime implementation. +//! +//! Reads a hook payload (JSON) from stdin, runs the kind-specific logic, +//! writes a hook-output envelope (JSON) to stdout, exits 0. Hooks are +//! non-blocking by design: violations and tips surface to the agent / +//! user; mdvs never rejects an edit at the harness layer. + +use std::collections::HashMap; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result}; +use serde::Deserialize; + +use crate::cmd::check; +use crate::cmd::hook::HookKind; +use crate::output::OutputFormat; +use crate::scaffold::{HooksConfig, Platform, template}; +use crate::step; + +/// Maximum number of lines to send through the user-visible `systemMessage` +/// channel. The agent channel (`additionalContext`) stays uncapped — the +/// agent reads all of it and decides what to surface in its reply. +/// +/// Past this point we truncate and append a `...` marker on its own line. +const MAX_USER_LINES: usize = 15; + +/// Run the hook handle. +/// +/// - Reads the harness payload (Claude Code / Codex / Cursor style stdin +/// JSON) from `stdin`. +/// - Loads the per-platform config (`scaffolding/platforms//`). +/// - Dispatches to the right kind: validate or search-nudge. +/// - Writes the platform-shaped JSON envelope to `stdout` (or nothing, +/// when there's nothing to surface — silent path). +/// - Always returns `Ok(())` on the happy path; the caller is expected to +/// exit 0 (non-blocking by design). +/// +/// Errors are surfaced for the caller to decide whether to ignore (recipe +/// users may want a malformed payload to be silent rather than noisy). +pub fn run( + stdin: R, + stdout: &mut W, + platform_name: &str, + kind: HookKind, +) -> Result<()> { + let platform = Platform::load(platform_name).context("loading platform config")?; + let payload = parse_payload(stdin).context("parsing stdin JSON")?; + + match kind { + HookKind::Validate => handle_validate(stdout, &platform, &payload), + HookKind::SearchNudge => handle_search_nudge(stdout, &platform, &payload), + } +} + +// ============================================================================ +// Stdin payload shape (lenient — extra fields ignored). +// ============================================================================ + +/// The subset of harness stdin payload we read. All fields are optional +/// because (a) different harnesses send different shapes and (b) we want +/// the hook to be resilient: a missing field falls back to a sensible +/// default (e.g. silent exit) rather than erroring out. +#[derive(Debug, Default, Deserialize)] +#[serde(default)] +struct HookPayload { + /// Harness's current working directory at the time of the tool call. + cwd: Option, + /// The tool-call input (file edits, bash commands, etc.). + tool_input: ToolInput, +} + +#[derive(Debug, Default, Deserialize)] +#[serde(default)] +struct ToolInput { + /// Set for Edit / Write / MultiEdit tool calls — the file being changed. + file_path: Option, + /// Set for Bash tool calls — the command being run. + command: Option, +} + +fn parse_payload(mut stdin: R) -> Result { + let mut buf = String::new(); + stdin.read_to_string(&mut buf).context("reading stdin")?; + if buf.trim().is_empty() { + return Ok(HookPayload::default()); + } + serde_json::from_str::(&buf).context("decoding hook payload JSON") +} + +// ============================================================================ +// HookKind::Validate +// ============================================================================ + +fn handle_validate( + stdout: &mut W, + platform: &Platform, + payload: &HookPayload, +) -> Result<()> { + // Only fire on .md edits. + let Some(file_path_str) = payload.tool_input.file_path.as_deref() else { + return Ok(()); + }; + if !file_path_str.ends_with(".md") { + return Ok(()); + } + + // Resolve to an absolute path, walking up from there to find mdvs.toml. + // If a relative path was sent, resolve against the harness's cwd. + let abs = resolve_path(file_path_str, payload.cwd.as_deref()); + let Some(vault_root) = walk_up_to_mdvs_toml(&abs) else { + return Ok(()); + }; + + // Run validation against the vault. `no_update = true` because hook- + // triggered validation shouldn't modify `mdvs.toml` (the user doesn't + // expect an edit to silently write new fields into the schema). + let result = check::run( + &vault_root, + /* no_update */ true, + /* verbose */ false, + None, + ); + + // Silent on clean: no violations AND no command-level failure. + let has_violations = step::has_violations(&result); + let has_failed = step::has_failed(&result); + if !has_violations && !has_failed { + return Ok(()); + } + + // Render twice: markdown for the agent (additionalContext), pretty for + // the user (systemMessage). Both reuse the same already-collected + // result; only the formatter differs. + let agent_body = result + .render(&OutputFormat::Markdown, /* verbose */ false) + .context("rendering agent markdown")?; + let user_body = result + .render(&OutputFormat::Pretty, /* verbose */ false) + .context("rendering user pretty")?; + + let agent_msg = append_skill_pointer(&agent_body, &platform.skill.install_path); + let user_msg = cap_lines(&user_body, MAX_USER_LINES); + + let Some(hooks) = platform.hooks.as_ref() else { + // Platform has no shell-hook surface (OpenCode, Antigravity). The + // user shouldn't be invoking this command for such a platform, but + // if they are, fail loudly so they know. + anyhow::bail!( + "platform '{}' has no [hooks] section — `mdvs hook handle` is not supported here. \ + Skill + snippet work via `mdvs scaffold skill|snippet`.", + platform.meta.name + ); + }; + + let envelope = build_envelope(hooks, &agent_msg, Some(&user_msg)); + writeln!(stdout, "{envelope}").context("writing stdout")?; + Ok(()) +} + +// ============================================================================ +// HookKind::SearchNudge +// ============================================================================ + +/// Search-tool prefixes that trigger the nudge. Each entry is matched as a +/// case-sensitive substring of the bash command — same as the shell-script +/// case-statement we extracted from earlier commits. +const SEARCH_TOOL_PATTERNS: &[&str] = &[ + "grep", "rg ", "ripgrep", "find ", "fd ", "fdfind ", "ag ", "ack ", "git grep", +]; + +fn handle_search_nudge( + stdout: &mut W, + platform: &Platform, + payload: &HookPayload, +) -> Result<()> { + let Some(hooks) = platform.hooks.as_ref() else { + anyhow::bail!( + "platform '{}' has no [hooks] section — `mdvs hook handle` is not supported here.", + platform.meta.name + ); + }; + + // Walk up from the agent's cwd. If we're not in an mdvs vault, the + // nudge stays silent. + let cwd = match payload.cwd.as_deref() { + Some(c) => PathBuf::from(c), + None => std::env::current_dir().context("reading current dir")?, + }; + if walk_up_to_mdvs_toml(&cwd).is_none() { + return Ok(()); + } + + // Only fire on search-tool commands. + let Some(command) = payload.tool_input.command.as_deref() else { + return Ok(()); + }; + if !is_search_command(command) { + return Ok(()); + } + + let tip = "Tip: `mdvs search` is also available for semantic / hybrid / SQL-filtered search over the KB."; + let envelope = build_envelope(hooks, tip, None); + writeln!(stdout, "{envelope}").context("writing stdout")?; + Ok(()) +} + +fn is_search_command(command: &str) -> bool { + SEARCH_TOOL_PATTERNS.iter().any(|pat| command.contains(pat)) +} + +// ============================================================================ +// Helpers — path resolution, walk-up, envelope construction, message shaping +// ============================================================================ + +fn resolve_path(file_path: &str, cwd: Option<&str>) -> PathBuf { + let raw = PathBuf::from(file_path); + if raw.is_absolute() { + raw + } else if let Some(cwd) = cwd { + PathBuf::from(cwd).join(raw) + } else { + raw + } +} + +/// Walk up from `start` (which may be a file path or a directory) looking +/// for an `mdvs.toml` in the directory itself or any ancestor. +fn walk_up_to_mdvs_toml(start: &Path) -> Option { + // If start is a file, begin from its parent. Stay tolerant of paths + // that don't exist on disk yet (the agent might have just deleted + // something or the file may exist but be unreadable due to perms). + let mut current = if start.is_file() { + start.parent()?.to_path_buf() + } else { + start.to_path_buf() + }; + loop { + if current.join("mdvs.toml").is_file() { + return Some(current); + } + if !current.pop() { + return None; + } + } +} + +/// Cap `body` at `max_lines` lines. If truncation happened, append a +/// `...` marker on its own line so the agent reader sees a clear "more +/// follows" signal. Always trims trailing newlines off `body` first so the +/// truncation marker lands on the right line. +fn cap_lines(body: &str, max_lines: usize) -> String { + let trimmed = body.trim_end_matches('\n'); + let mut lines: Vec<&str> = trimmed.lines().collect(); + if lines.len() <= max_lines { + return trimmed.to_string(); + } + lines.truncate(max_lines); + let mut out = lines.join("\n"); + out.push_str("\n..."); + out +} + +/// Append a pointer to the bundled mdvs skill. The pointer tells the agent +/// (a) where the skill file lives on disk for this platform, and (b) where +/// to find the schema-evolution loop section that covers how to react. +fn append_skill_pointer(body: &str, skill_install_path: &str) -> String { + let trimmed = body.trim_end_matches('\n'); + format!( + "{trimmed}\n\n---\n\n_To handle this correctly, load the mdvs skill at \ + `{skill_install_path}` (or run `mdvs scaffold skill` to print it). \ + The schema-evolution loop section covers when to fix the file vs. \ + propose a `mdvs.toml` update._" + ) +} + +/// Build the hook-output envelope by substituting `<>` and +/// `<>` into the platform's envelope template (from +/// `platform.toml`). When `user_msg` is `None`, the template's +/// `<>` marker (if any) gets pruned — the user-channel field +/// disappears from the output entirely. +/// +/// Per-platform JSON shapes live in `platform.toml`, not here. Different +/// harnesses can have wildly different envelopes (Claude Code's wrapped +/// `hookSpecificOutput`, Cursor's flat `additional_context`, etc.) and +/// this function doesn't care — it just feeds substitution values in. +fn build_envelope(hooks: &HooksConfig, agent_msg: &str, user_msg: Option<&str>) -> String { + let mut vars: HashMap<&str, Option> = HashMap::new(); + vars.insert("MSG", Some(agent_msg.to_string())); + vars.insert("USER_MSG", user_msg.map(|s| s.to_string())); + let envelope = template::substitute(&hooks.envelope, &vars); + envelope.to_string() +} + +// ============================================================================ +// Tests +// ============================================================================ + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::Value; + use std::io::Cursor; + use tempfile::TempDir; + + /// Build a minimal mdvs vault in `dir`: an `mdvs.toml` with one + /// categorical `status` field, plus one markdown file. + /// + /// The `status_value` parameter lets tests choose between a valid + /// value (no violations) and an invalid one (one violation surfaces). + fn write_fixture_vault(dir: &Path, status_value: &str) { + std::fs::write( + dir.join("mdvs.toml"), + r#" +[scan] +glob = "**" +include_bare_files = true +skip_gitignore = false +frontmatter_format = "auto" + +[check] +auto_update = false + +[[fields.field]] +name = "status" +type = "String" +constraints = { categories = ["active", "archived"] } +"#, + ) + .unwrap(); + std::fs::write( + dir.join("note.md"), + format!("---\nstatus: {status_value}\n---\n# Note\n"), + ) + .unwrap(); + } + + // --- parse_payload ---------------------------------------------------- + + #[test] + fn parse_payload_empty_stdin_returns_default() { + let payload = parse_payload(Cursor::new(b"")).unwrap(); + assert!(payload.cwd.is_none()); + assert!(payload.tool_input.file_path.is_none()); + assert!(payload.tool_input.command.is_none()); + } + + #[test] + fn parse_payload_full_claude_code_shape() { + let stdin = br#"{ + "session_id": "abc", + "cwd": "/some/dir", + "tool_input": { "file_path": "note.md", "command": null } + }"#; + let payload = parse_payload(Cursor::new(stdin)).unwrap(); + assert_eq!(payload.cwd.as_deref(), Some("/some/dir")); + assert_eq!(payload.tool_input.file_path.as_deref(), Some("note.md")); + assert!(payload.tool_input.command.is_none()); + } + + #[test] + fn parse_payload_extra_fields_ignored() { + let stdin = br#"{ + "tool_input": { "file_path": "x.md" }, + "unknown_field": [1, 2, 3] + }"#; + let payload = parse_payload(Cursor::new(stdin)).unwrap(); + assert_eq!(payload.tool_input.file_path.as_deref(), Some("x.md")); + } + + // --- walk_up_to_mdvs_toml -------------------------------------------- + + #[test] + fn walk_up_finds_vault_root_from_file() { + let dir = TempDir::new().unwrap(); + write_fixture_vault(dir.path(), "active"); + let file = dir.path().join("note.md"); + let found = walk_up_to_mdvs_toml(&file).unwrap(); + assert_eq!( + found.canonicalize().unwrap(), + dir.path().canonicalize().unwrap() + ); + } + + #[test] + fn walk_up_finds_vault_root_from_nested_file() { + let dir = TempDir::new().unwrap(); + write_fixture_vault(dir.path(), "active"); + let sub = dir.path().join("projects").join("alpha"); + std::fs::create_dir_all(&sub).unwrap(); + let file = sub.join("notes.md"); + std::fs::write(&file, "---\nstatus: active\n---\n# Nested\n").unwrap(); + let found = walk_up_to_mdvs_toml(&file).unwrap(); + assert_eq!( + found.canonicalize().unwrap(), + dir.path().canonicalize().unwrap() + ); + } + + #[test] + fn walk_up_returns_none_outside_vault() { + let dir = TempDir::new().unwrap(); + let file = dir.path().join("note.md"); + std::fs::write(&file, "x").unwrap(); + assert!(walk_up_to_mdvs_toml(&file).is_none()); + } + + // --- cap_lines -------------------------------------------------------- + + #[test] + fn cap_lines_passthrough_under_limit() { + let input = "a\nb\nc"; + let out = cap_lines(input, 5); + assert_eq!(out, "a\nb\nc"); + } + + #[test] + fn cap_lines_truncates_and_appends_marker() { + let input = "a\nb\nc\nd\ne\nf"; + let out = cap_lines(input, 3); + assert_eq!(out, "a\nb\nc\n..."); + } + + #[test] + fn cap_lines_strips_trailing_newlines_before_counting() { + let input = "a\nb\nc\n\n"; + let out = cap_lines(input, 5); + assert_eq!(out, "a\nb\nc"); + } + + // --- append_skill_pointer -------------------------------------------- + + #[test] + fn append_skill_pointer_uses_platform_path() { + let out = append_skill_pointer("Body", ".claude/skills/mdvs/SKILL.md"); + assert!(out.starts_with("Body\n\n---\n\n")); + assert!(out.contains("`.claude/skills/mdvs/SKILL.md`")); + assert!(out.contains("schema-evolution loop")); + } + + // --- is_search_command ---------------------------------------------- + + #[test] + fn is_search_command_recognises_common_tools() { + for cmd in [ + "grep foo .", + "rg pattern", + "find . -name '*.md'", + "fd extension md", + "git grep needle", + "ag pattern", + "ack -i bar", + ] { + assert!(is_search_command(cmd), "should match: {cmd}"); + } + } + + #[test] + fn is_search_command_ignores_unrelated() { + for cmd in ["cargo build", "mdvs search query", "echo hello", "cat file"] { + assert!(!is_search_command(cmd), "should NOT match: {cmd}"); + } + } + + // --- build_envelope -------------------------------------------------- + + /// With user_msg = Some(...), Claude Code's full Claude-Code-shaped + /// envelope renders cleanly: PostToolUse + additionalContext + + /// systemMessage. + #[test] + fn build_envelope_claude_code_validate_includes_both_channels() { + let p = Platform::load("claude-code").unwrap(); + let hooks = p.hooks.as_ref().unwrap(); + let env = build_envelope(hooks, "agent body", Some("user body")); + let parsed: Value = serde_json::from_str(&env).unwrap(); + assert_eq!(parsed["hookSpecificOutput"]["hookEventName"], "PostToolUse"); + assert_eq!( + parsed["hookSpecificOutput"]["additionalContext"], + "agent body" + ); + assert_eq!(parsed["systemMessage"], "user body"); + } + + /// With user_msg = None, the `<>` marker is pruned and the + /// resulting envelope omits `systemMessage` entirely. The wrapper + /// stays because its other field is still populated. + #[test] + fn build_envelope_claude_code_search_nudge_prunes_user_message() { + let p = Platform::load("claude-code").unwrap(); + let hooks = p.hooks.as_ref().unwrap(); + let env = build_envelope(hooks, "tip only", None); + let parsed: Value = serde_json::from_str(&env).unwrap(); + assert_eq!(parsed["hookSpecificOutput"]["hookEventName"], "PostToolUse"); + assert_eq!( + parsed["hookSpecificOutput"]["additionalContext"], + "tip only" + ); + assert!( + parsed.get("systemMessage").is_none(), + "systemMessage should be pruned" + ); + } + + // Cursor envelope test removed: mdvs no longer ships a hook config or + // envelope for cursor — the previous implementation was schema-correct + // per the docs but not observed firing in a live smoke test. + + // --- run() validate end-to-end -------------------------------------- + + #[test] + fn validate_silent_on_clean_vault() { + let dir = TempDir::new().unwrap(); + write_fixture_vault(dir.path(), "active"); + let file = dir.path().join("note.md"); + let stdin = format!(r#"{{"tool_input":{{"file_path":"{}"}}}}"#, file.display()); + let mut out = Vec::new(); + run( + Cursor::new(stdin), + &mut out, + "claude-code", + HookKind::Validate, + ) + .unwrap(); + assert!( + out.is_empty(), + "expected silent exit on clean vault, got: {}", + String::from_utf8_lossy(&out) + ); + } + + #[test] + fn validate_emits_envelope_with_violations() { + let dir = TempDir::new().unwrap(); + write_fixture_vault(dir.path(), "bogus"); + let file = dir.path().join("note.md"); + let stdin = format!(r#"{{"tool_input":{{"file_path":"{}"}}}}"#, file.display()); + let mut out = Vec::new(); + run( + Cursor::new(stdin), + &mut out, + "claude-code", + HookKind::Validate, + ) + .unwrap(); + assert!(!out.is_empty(), "expected envelope output for violations"); + + let env: Value = serde_json::from_slice(&out).unwrap(); + assert_eq!(env["hookSpecificOutput"]["hookEventName"], "PostToolUse"); + let context = env["hookSpecificOutput"]["additionalContext"] + .as_str() + .unwrap(); + assert!( + context.contains("status"), + "violation should mention the field: {context}" + ); + assert!( + context.contains(".claude/skills/mdvs/SKILL.md"), + "skill pointer should use claude-code's install path: {context}" + ); + assert!( + env["systemMessage"].is_string(), + "systemMessage should be populated" + ); + } + + #[test] + fn validate_silent_on_non_md_file() { + let dir = TempDir::new().unwrap(); + write_fixture_vault(dir.path(), "active"); + let stdin = format!( + r#"{{"tool_input":{{"file_path":"{}/some.rs"}}}}"#, + dir.path().display() + ); + let mut out = Vec::new(); + run( + Cursor::new(stdin), + &mut out, + "claude-code", + HookKind::Validate, + ) + .unwrap(); + assert!(out.is_empty()); + } + + #[test] + fn validate_silent_outside_vault() { + let dir = TempDir::new().unwrap(); + let lone_file = dir.path().join("note.md"); + std::fs::write(&lone_file, "x").unwrap(); + let stdin = format!( + r#"{{"tool_input":{{"file_path":"{}"}}}}"#, + lone_file.display() + ); + let mut out = Vec::new(); + run( + Cursor::new(stdin), + &mut out, + "claude-code", + HookKind::Validate, + ) + .unwrap(); + assert!(out.is_empty()); + } + + // Cursor end-to-end envelope test removed: mdvs no longer ships a hook + // config or envelope for cursor — the previous implementation was + // schema-correct per the docs but not observed firing in a live smoke + // test. The `validate_errors_on_platform_without_hooks` test below + // covers the "no hook config" path more generally. + + #[test] + fn validate_errors_on_platform_without_hooks() { + let dir = TempDir::new().unwrap(); + write_fixture_vault(dir.path(), "bogus"); + let file = dir.path().join("note.md"); + let stdin = format!(r#"{{"tool_input":{{"file_path":"{}"}}}}"#, file.display()); + let mut out = Vec::new(); + let err = run(Cursor::new(stdin), &mut out, "opencode", HookKind::Validate) + .expect_err("opencode has no [hooks] section, run should error"); + let msg = format!("{err}"); + assert!(msg.contains("opencode"), "{msg}"); + assert!(msg.contains("no [hooks] section"), "{msg}"); + } + + // --- run() search-nudge end-to-end ----------------------------------- + + #[test] + fn search_nudge_emits_when_in_vault_and_command_matches() { + let dir = TempDir::new().unwrap(); + write_fixture_vault(dir.path(), "active"); + let stdin = format!( + r#"{{"cwd":"{}","tool_input":{{"command":"grep foo ."}}}}"#, + dir.path().display() + ); + let mut out = Vec::new(); + run( + Cursor::new(stdin), + &mut out, + "claude-code", + HookKind::SearchNudge, + ) + .unwrap(); + assert!(!out.is_empty()); + let env: Value = serde_json::from_slice(&out).unwrap(); + assert_eq!(env["hookSpecificOutput"]["hookEventName"], "PostToolUse"); + assert!( + env["hookSpecificOutput"]["additionalContext"] + .as_str() + .unwrap() + .contains("mdvs search"), + "tip should mention mdvs search" + ); + assert!( + env.get("systemMessage").is_none(), + "search-nudge should not surface to user UI — only to model" + ); + } + + #[test] + fn search_nudge_silent_outside_vault() { + let dir = TempDir::new().unwrap(); + let stdin = format!( + r#"{{"cwd":"{}","tool_input":{{"command":"grep foo ."}}}}"#, + dir.path().display() + ); + let mut out = Vec::new(); + run( + Cursor::new(stdin), + &mut out, + "claude-code", + HookKind::SearchNudge, + ) + .unwrap(); + assert!(out.is_empty(), "no nudge outside a vault"); + } + + #[test] + fn search_nudge_silent_on_non_search_command() { + let dir = TempDir::new().unwrap(); + write_fixture_vault(dir.path(), "active"); + let stdin = format!( + r#"{{"cwd":"{}","tool_input":{{"command":"cargo build"}}}}"#, + dir.path().display() + ); + let mut out = Vec::new(); + run( + Cursor::new(stdin), + &mut out, + "claude-code", + HookKind::SearchNudge, + ) + .unwrap(); + assert!(out.is_empty()); + } +} diff --git a/crates/mdvs/src/cmd/hook/mod.rs b/crates/mdvs/src/cmd/hook/mod.rs new file mode 100644 index 0000000..355f296 --- /dev/null +++ b/crates/mdvs/src/cmd/hook/mod.rs @@ -0,0 +1,35 @@ +//! `mdvs hook` — the runtime that agent-harness PostToolUse hooks call. +//! +//! Reads a hook payload from stdin, runs validate or search-nudge logic, +//! writes a platform-shaped envelope to stdout. Cross-platform because +//! mdvs is a cross-platform Rust binary; no `jq` dependency. +//! +//! Usage from a hook config: +//! +//! ```text +//! mdvs hook handle --platform claude-code --kind validate +//! mdvs hook handle --platform claude-code --kind search-nudge +//! ``` +//! +//! See [`handle::run`] for the runtime behaviour, and `scaffold::Platform` +//! for the per-platform data this command reads. + +pub mod handle; + +/// Which kind of hook is being handled. +/// +/// Closed enum because the set is small and stable: `validate` runs +/// `mdvs check` after an edit; `search-nudge` emits a one-line tip when +/// the agent runs grep/rg/find/etc. inside an mdvs vault. +#[derive(Debug, Clone, Copy, PartialEq, Eq, clap::ValueEnum)] +#[clap(rename_all = "kebab-case")] +pub enum HookKind { + /// Run after an Edit/Write/MultiEdit tool call: validates frontmatter + /// in the vault containing the edited file and surfaces violations + /// (non-blocking). + Validate, + /// Run after a Bash tool call: if the command is a search tool (grep, + /// rg, find, fd, ag, ack, git grep) and the agent's cwd is inside an + /// mdvs vault, emit a one-line tip pointing at `mdvs search`. + SearchNudge, +} diff --git a/crates/mdvs/src/cmd/mod.rs b/crates/mdvs/src/cmd/mod.rs index 533238c..8d5169a 100644 --- a/crates/mdvs/src/cmd/mod.rs +++ b/crates/mdvs/src/cmd/mod.rs @@ -8,10 +8,14 @@ pub mod check; pub mod clean; /// Emit the canonical JSON Schema of the local `mdvs.toml`. pub mod export_jsonschema; +/// Agent-harness PostToolUse hook runtime (`mdvs hook handle`). +pub mod hook; /// Display project configuration and index status. pub mod info; /// Initialize a new mdvs project. pub mod init; +/// Install-time generator commands (`mdvs scaffold {skill,snippet,hook}`). +pub mod scaffold; /// Query the search index. pub mod search; /// Re-scan and update the schema in `mdvs.toml`. diff --git a/crates/mdvs/src/cmd/scaffold/hook.rs b/crates/mdvs/src/cmd/scaffold/hook.rs new file mode 100644 index 0000000..4afb23e --- /dev/null +++ b/crates/mdvs/src/cmd/scaffold/hook.rs @@ -0,0 +1,250 @@ +//! `mdvs scaffold hook` — print the per-platform PostToolUse hook config. +//! +//! The emitted config calls `mdvs hook handle --platform --kind +//! ` for each matcher. No shell scripts are generated; the runtime +//! lives in mdvs itself ([`crate::cmd::hook::handle`]). +//! +//! Refuses for platforms without a shell-hook surface (OpenCode's +//! TypeScript plugin API, Antigravity's undocumented hook spec) — both +//! cases fail with a pointer message explaining the user should still +//! install the skill and snippet via `mdvs scaffold skill|snippet`. + +use std::collections::HashMap; +use std::io::Write; + +use anyhow::{Context, Result}; +use serde_json::{Map, Value}; + +use crate::scaffold::{HookConfigFormat, HooksConfig, Platform, template}; + +/// Print the hook config JSON for `platform_name` to `stdout`. Writes a +/// short install hint to `stderr`. Returns an error if the platform has +/// no `[hooks]` section (OpenCode, Antigravity). +pub fn run(stdout: &mut W, stderr: &mut E, platform_name: &str) -> Result<()> { + let platform = Platform::load(platform_name)?; + let hooks = platform.hooks.as_ref().ok_or_else(|| { + anyhow::anyhow!( + "mdvs doesn't ship a verified hook config for '{name}'. The skill and snippet still \ + install normally:\n\n mdvs scaffold skill --platform {name}\n mdvs scaffold snippet --platform {name}\n\n\ + For the current status and integration guidance, see:\n \ + https://edochi.github.io/mdvs/recipes/agent-harnesses/{name}.html", + name = platform.meta.name + ) + })?; + + let config = build_config(&platform.meta.name, hooks); + let rendered = match hooks.config_format { + HookConfigFormat::Json => serde_json::to_string_pretty(&config)?, + HookConfigFormat::Toml => toml::to_string_pretty(&config)?, + }; + stdout + .write_all(rendered.as_bytes()) + .context("writing hook config to stdout")?; + stdout.write_all(b"\n").ok(); + + writeln!( + stderr, + "Merge into: {} (under {})", + hooks.config_path, platform.meta.display_name + ) + .context("writing install hint")?; + + Ok(()) +} + +/// Build the hooks config snippet for a platform by substituting into +/// the platform's bundled `hooks.config` template. +/// +/// The template (in `platform.toml`) declares the full per-harness JSON +/// shape, including any wrapper structure, matcher conventions, and +/// version fields. mdvs only fills in the two command strings via the +/// `<>` and `<>` placeholders. +/// +/// A `_comment` field is added at the top of the resulting object (if it +/// is one) so users opening the generated snippet see where to install +/// it. The comment is added programmatically rather than templated so +/// each platform.toml stays focused on the actual schema. +fn build_config(platform_name: &str, hooks: &HooksConfig) -> Value { + let vars: HashMap<&str, Option> = HashMap::from([ + ( + "COMMAND_VALIDATE", + Some(format!( + "mdvs hook handle --platform {platform_name} --kind validate" + )), + ), + ( + "COMMAND_SEARCH", + Some(format!( + "mdvs hook handle --platform {platform_name} --kind search-nudge" + )), + ), + ]); + + let mut config = template::substitute(&hooks.config, &vars); + + // Inject a leading _comment field so users see what to do with the + // emitted snippet. Only applies to object-rooted templates (every + // platform we ship is one). + let comment = format!( + "mdvs PostToolUse hooks generated by `mdvs scaffold hook --platform {platform_name}`. \ + Merge into {} under the existing `hooks` key. Both hooks self-scope by walking up \ + from the agent's cwd to find mdvs.toml; they stay silent outside an mdvs vault.", + hooks.config_path + ); + if let Value::Object(map) = &mut config { + // Rebuild with _comment first for readability. + let mut with_comment = Map::with_capacity(map.len() + 1); + with_comment.insert("_comment".into(), Value::String(comment)); + for (k, v) in std::mem::take(map) { + with_comment.insert(k, v); + } + *map = with_comment; + } + config +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn hook_emits_valid_json_for_claude_code() { + let mut out = Vec::new(); + let mut err = Vec::new(); + run(&mut out, &mut err, "claude-code").unwrap(); + let parsed: Value = serde_json::from_slice(&out).expect("output should be valid JSON"); + + // Comment field present. + assert!(parsed["_comment"].is_string()); + assert!( + parsed["_comment"] + .as_str() + .unwrap() + .contains(".claude/settings.json"), + "comment should reference the install path" + ); + + // PostToolUse with the two hooks under it. + let post = &parsed["hooks"]["PostToolUse"]; + assert!(post.is_array()); + let entries = post.as_array().unwrap(); + assert_eq!(entries.len(), 2, "expected validate + search-nudge entries"); + assert_eq!(entries[0]["matcher"], "Edit|Write|MultiEdit"); + assert_eq!(entries[1]["matcher"], "Bash"); + + // The command for each entry calls mdvs hook handle. + assert!( + entries[0]["hooks"][0]["command"] + .as_str() + .unwrap() + .contains("mdvs hook handle --platform claude-code --kind validate") + ); + assert!( + entries[1]["hooks"][0]["command"] + .as_str() + .unwrap() + .contains("mdvs hook handle --platform claude-code --kind search-nudge") + ); + + // Stderr hint. + let hint = String::from_utf8(err).unwrap(); + assert!(hint.contains(".claude/settings.json"), "{hint}"); + } + + /// Cursor uses a flat matcher entry shape and a top-level `version` + /// mdvs doesn't ship a verified hook config for Cursor — the previous + /// attempt was schema-correct per the docs but not observed firing in + /// a live smoke test. Until that's resolved, `scaffold hook --platform + /// cursor` refuses with a pointer at the per-platform mdbook page. + #[test] + fn hook_refuses_for_cursor_with_pointer() { + let mut out = Vec::new(); + let mut err = Vec::new(); + let res = run(&mut out, &mut err, "cursor"); + let e = res.unwrap_err(); + let msg = format!("{e}"); + assert!( + msg.contains("recipes/agent-harnesses/cursor.html"), + "expected mdbook URL pointer: {msg}" + ); + } + + /// Same as Cursor — Codex hook firing was inconclusive in a live smoke + /// test, so mdvs no longer ships a hook config for it. + #[test] + fn hook_refuses_for_codex_with_pointer() { + let mut out = Vec::new(); + let mut err = Vec::new(); + let res = run(&mut out, &mut err, "codex"); + let e = res.unwrap_err(); + let msg = format!("{e}"); + assert!( + msg.contains("recipes/agent-harnesses/codex.html"), + "expected mdbook URL pointer: {msg}" + ); + } + + #[test] + fn hook_refuses_for_opencode_with_pointer() { + let mut out = Vec::new(); + let mut err = Vec::new(); + let res = run(&mut out, &mut err, "opencode"); + let e = res.unwrap_err(); + let msg = format!("{e}"); + assert!(msg.contains("opencode"), "{msg}"); + // The error should still point users at scaffold skill / snippet + // (which DO work for opencode) plus the per-platform mdbook page. + assert!(msg.contains("mdvs scaffold skill"), "{msg}"); + assert!(msg.contains("mdvs scaffold snippet"), "{msg}"); + assert!( + msg.contains("recipes/agent-harnesses/opencode.html"), + "expected mdbook URL pointer: {msg}" + ); + } + + #[test] + fn hook_refuses_for_antigravity() { + let mut out = Vec::new(); + let mut err = Vec::new(); + let res = run(&mut out, &mut err, "antigravity"); + let e = res.unwrap_err(); + let msg = format!("{e}"); + assert!( + msg.contains("recipes/agent-harnesses/antigravity.html"), + "expected mdbook URL pointer: {msg}" + ); + } + + #[test] + fn hook_errors_on_unknown_platform() { + let mut out = Vec::new(); + let mut err = Vec::new(); + let res = run(&mut out, &mut err, "made-up-platform"); + assert!(res.is_err()); + } + + /// Round-trip: the emitted command strings should match the form + /// `mdvs hook handle` actually accepts. Smoke check by parsing the + /// command via clap. + #[test] + fn emitted_commands_parse_as_clap_input() { + // Quick check: every emitted command starts with `mdvs hook handle + // --platform --kind ` with `kind` one of the two + // values clap knows about. + let mut out = Vec::new(); + let mut err = Vec::new(); + run(&mut out, &mut err, "claude-code").unwrap(); + let parsed: Value = serde_json::from_slice(&out).unwrap(); + for entry in parsed["hooks"]["PostToolUse"].as_array().unwrap() { + let cmd = entry["hooks"][0]["command"].as_str().unwrap(); + assert!(cmd.starts_with("mdvs hook handle --platform claude-code --kind ")); + let kind = cmd + .strip_prefix("mdvs hook handle --platform claude-code --kind ") + .unwrap(); + assert!( + kind == "validate" || kind == "search-nudge", + "kind must match clap's value-enum: {kind}" + ); + } + } +} diff --git a/crates/mdvs/src/cmd/scaffold/mod.rs b/crates/mdvs/src/cmd/scaffold/mod.rs new file mode 100644 index 0000000..38187e0 --- /dev/null +++ b/crates/mdvs/src/cmd/scaffold/mod.rs @@ -0,0 +1,61 @@ +//! `mdvs scaffold` — install-time generator commands. +//! +//! Three subcommands, all reading from the bundled +//! [`crate::scaffold::SCAFFOLDING`] tree and the per-platform +//! [`crate::scaffold::Platform`] config: +//! +//! - [`skill`] — print the bundled `SKILL.md`. Pipe to the harness's skill +//! directory. +//! - [`snippet`] — print the project-rules snippet. Pipe / append to +//! `CLAUDE.md` / `AGENTS.md` / `.cursor/rules/mdvs.mdc`. +//! - [`hook`] — print the harness's PostToolUse hook config. Merge into +//! `.claude/settings.json` / `.codex/hooks.json` / `.cursor/hooks.json`. +//! +//! Each subcommand writes pure body content to stdout, so `mdvs scaffold +//! skill > .claude/skills/mdvs/SKILL.md` works without polluting the file. +//! Install hints are printed to stderr (and visible interactively but +//! invisible to a redirect). + +pub mod hook; +pub mod skill; +pub mod snippet; + +/// `mdvs scaffold` subcommand. +#[derive(Debug, clap::Subcommand)] +pub enum ScaffoldCommand { + /// Print the bundled mdvs skill file. + /// + /// Default: prints the universal `SKILL.md`. With `--platform`, also + /// emits an install-path hint on stderr (the body on stdout is + /// identical — pipe-safe). + Skill { + /// Target harness (claude-code, codex, cursor, opencode, antigravity). + #[arg(long)] + platform: Option, + }, + /// Print the project-rules snippet for the agent. + /// + /// Default: prints the universal `AGENTS.md`-flavoured block. With + /// `--platform`, picks the platform's preferred body (e.g. Cursor + /// uses the `.mdc`-wrapped variant for `.cursor/rules/`). + Snippet { + /// Target harness. + #[arg(long)] + platform: Option, + }, + /// Print the per-platform PostToolUse hook config (JSON) to merge into + /// the harness's hooks file. + /// + /// `--platform` is required because the JSON shape varies (config file + /// path, event-name capitalization). The emitted config calls + /// `mdvs hook handle --platform --kind ` for each matcher + /// — no shell scripts to install. + /// + /// Refuses for platforms without a shell-hook surface (OpenCode, + /// Antigravity). + Hook { + /// Target harness. + #[arg(long)] + platform: String, + }, +} diff --git a/crates/mdvs/src/cmd/scaffold/skill.rs b/crates/mdvs/src/cmd/scaffold/skill.rs new file mode 100644 index 0000000..8f5db15 --- /dev/null +++ b/crates/mdvs/src/cmd/scaffold/skill.rs @@ -0,0 +1,115 @@ +//! `mdvs scaffold skill` — print the bundled mdvs skill file. + +use std::io::Write; + +use anyhow::{Context, Result, anyhow}; + +use crate::scaffold::{Platform, SCAFFOLDING}; + +/// Print the bundled `SKILL.md` to `stdout`. If `platform_name` is `Some`, +/// look up the platform and write an install-path hint to `stderr` (the +/// body on stdout is identical regardless of platform — the skill content +/// is harness-agnostic). +pub fn run( + stdout: &mut W, + stderr: &mut E, + platform_name: Option<&str>, +) -> Result<()> { + let file = SCAFFOLDING + .get_file("skill/SKILL.md") + .ok_or_else(|| anyhow!("bundled skill/SKILL.md is missing — this is a build bug"))?; + let body = file + .contents_utf8() + .ok_or_else(|| anyhow!("bundled skill/SKILL.md is not valid UTF-8"))?; + + stdout + .write_all(body.as_bytes()) + .context("writing skill body to stdout")?; + + if let Some(name) = platform_name { + let platform = Platform::load(name)?; + writeln!( + stderr, + "Install to: {} (under {})", + platform.skill.install_path, platform.meta.display_name + ) + .context("writing install hint")?; + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn skill_emits_bundled_body() { + let mut out = Vec::new(); + let mut err = Vec::new(); + run(&mut out, &mut err, None).unwrap(); + let body = String::from_utf8(out).unwrap(); + // First line of SKILL.md is the YAML frontmatter delimiter. + assert!( + body.starts_with("---\n"), + "unexpected skill body start: {body:.60}" + ); + assert!( + body.contains("name: mdvs"), + "skill frontmatter should declare name" + ); + // No --platform → no stderr hint. + assert!(err.is_empty(), "expected no stderr hint without --platform"); + } + + #[test] + fn skill_emits_install_hint_to_stderr_when_platform_given() { + let mut out = Vec::new(); + let mut err = Vec::new(); + run(&mut out, &mut err, Some("claude-code")).unwrap(); + let body = String::from_utf8(out).unwrap(); + let hint = String::from_utf8(err).unwrap(); + // Body unchanged by --platform. + assert!(body.starts_with("---\n")); + // Hint mentions the platform's install path. + assert!( + hint.contains(".claude/skills/mdvs/SKILL.md"), + "hint: {hint}" + ); + assert!( + hint.contains("Claude Code"), + "hint should name the display: {hint}" + ); + } + + #[test] + fn skill_errors_on_unknown_platform() { + let mut out = Vec::new(); + let mut err = Vec::new(); + let res = run(&mut out, &mut err, Some("does-not-exist")); + assert!(res.is_err()); + // The body is still printed before the platform load errors — + // that's acceptable; the redirect target would still have a valid + // skill file, and the user sees the error explaining the hint + // couldn't be emitted. + } + + #[test] + fn skill_body_includes_pivot_era_content() { + // Sanity check that we're emitting the rewritten SKILL.md (Step 2 + // content), not some stale version: the new SKILL.md describes + // the schema-evolution loop explicitly. + let mut out = Vec::new(); + let mut err = Vec::new(); + run(&mut out, &mut err, None).unwrap(); + let body = String::from_utf8(out).unwrap(); + assert!( + body.contains("schema-evolution loop"), + "should mention the loop" + ); + assert!( + body.contains("mdvs scaffold"), + "should reference new commands" + ); + } +} diff --git a/crates/mdvs/src/cmd/scaffold/snippet.rs b/crates/mdvs/src/cmd/scaffold/snippet.rs new file mode 100644 index 0000000..dd018c2 --- /dev/null +++ b/crates/mdvs/src/cmd/scaffold/snippet.rs @@ -0,0 +1,141 @@ +//! `mdvs scaffold snippet` — print the project-rules snippet for the agent. + +use std::io::Write; + +use anyhow::{Context, Result, anyhow}; + +use crate::scaffold::{Platform, SCAFFOLDING, SnippetBody}; + +/// Print the project-rules snippet body to `stdout`. If `platform_name` is +/// `Some`, the platform's preferred body (per `snippet.body` in +/// `platform.toml`) is used and an install hint is written to `stderr`. +/// +/// Without `--platform`, defaults to the universal AGENTS.md-flavoured +/// body. That body works in any `AGENTS.md` / `CLAUDE.md` setup. +pub fn run( + stdout: &mut W, + stderr: &mut E, + platform_name: Option<&str>, +) -> Result<()> { + let (body_key, install_hint) = match platform_name { + Some(name) => { + let platform = Platform::load(name)?; + let body_key = body_file(platform.snippet.body); + let hint = format!( + "Install to: {} (under {})", + platform.snippet.target_file, platform.meta.display_name + ); + (body_key, Some(hint)) + } + None => (body_file(SnippetBody::AgentsMd), None), + }; + + let file = SCAFFOLDING + .get_file(body_key) + .ok_or_else(|| anyhow!("bundled {body_key} is missing — this is a build bug"))?; + let body = file + .contents_utf8() + .ok_or_else(|| anyhow!("bundled {body_key} is not valid UTF-8"))?; + + stdout + .write_all(body.as_bytes()) + .context("writing snippet body to stdout")?; + + if let Some(hint) = install_hint { + writeln!(stderr, "{hint}").context("writing install hint")?; + } + + Ok(()) +} + +/// Maps a [`SnippetBody`] variant to its bundled path under +/// `scaffolding/snippet/`. Closed-set match enforced by the enum. +fn body_file(body: SnippetBody) -> &'static str { + match body { + SnippetBody::AgentsMd => "snippet/agents-md.md", + SnippetBody::CursorRules => "snippet/cursor-rules.mdc", + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn snippet_default_emits_agents_md_body() { + let mut out = Vec::new(); + let mut err = Vec::new(); + run(&mut out, &mut err, None).unwrap(); + let body = String::from_utf8(out).unwrap(); + assert!( + body.contains("mdvs knowledge base"), + "snippet should mention the KB heading" + ); + // Universal body has no Cursor frontmatter wrapping. + assert!( + !body.starts_with("---\n"), + "AGENTS.md body shouldn't have YAML frontmatter" + ); + assert!(err.is_empty(), "no stderr hint without --platform"); + } + + #[test] + fn snippet_claude_code_emits_agents_md_body_with_install_hint() { + let mut out = Vec::new(); + let mut err = Vec::new(); + run(&mut out, &mut err, Some("claude-code")).unwrap(); + let body = String::from_utf8(out).unwrap(); + let hint = String::from_utf8(err).unwrap(); + assert!(body.contains("mdvs knowledge base")); + assert!( + !body.starts_with("---\n"), + "claude-code uses agents-md body, no frontmatter" + ); + assert!( + hint.contains("CLAUDE.md"), + "hint should target CLAUDE.md: {hint}" + ); + } + + #[test] + fn snippet_cursor_emits_mdc_body_with_frontmatter() { + let mut out = Vec::new(); + let mut err = Vec::new(); + run(&mut out, &mut err, Some("cursor")).unwrap(); + let body = String::from_utf8(out).unwrap(); + let hint = String::from_utf8(err).unwrap(); + // .mdc has Cursor frontmatter (alwaysApply: true). + assert!( + body.starts_with("---\n"), + ".mdc body should start with frontmatter" + ); + assert!( + body.contains("alwaysApply: true"), + ".mdc body should set alwaysApply" + ); + // Hint mentions the .cursor/rules/ target. + assert!(hint.contains(".cursor/rules/mdvs.mdc"), "hint: {hint}"); + } + + #[test] + fn snippet_opencode_uses_agents_md_with_agents_md_target() { + let mut out = Vec::new(); + let mut err = Vec::new(); + run(&mut out, &mut err, Some("opencode")).unwrap(); + let body = String::from_utf8(out).unwrap(); + let hint = String::from_utf8(err).unwrap(); + assert!(!body.starts_with("---\n")); + assert!( + hint.contains("AGENTS.md"), + "hint should target AGENTS.md: {hint}" + ); + } + + #[test] + fn snippet_errors_on_unknown_platform() { + let mut out = Vec::new(); + let mut err = Vec::new(); + let res = run(&mut out, &mut err, Some("does-not-exist")); + assert!(res.is_err()); + } +} diff --git a/crates/mdvs/src/cmd/search.rs b/crates/mdvs/src/cmd/search.rs index 2967349..e357b21 100644 --- a/crates/mdvs/src/cmd/search.rs +++ b/crates/mdvs/src/cmd/search.rs @@ -294,7 +294,7 @@ pub async fn run( } let search_start = Instant::now(); - let hits = match backend + let results = match backend .search( query_embedding, query, @@ -306,12 +306,12 @@ pub async fn run( ) .await { - Ok(hits) => { + Ok(r) => { steps.push(StepEntry::ok( - Outcome::ExecuteSearch(ExecuteSearchOutcome { hits: hits.len() }), + Outcome::ExecuteSearch(ExecuteSearchOutcome { hits: r.hits.len() }), search_start.elapsed().as_millis() as u64, )); - hits + r } Err(e) => { steps.push(StepEntry::err( @@ -329,9 +329,10 @@ pub async fn run( steps, result: Ok(Outcome::Search(Box::new(SearchOutcome { query: query.to_string(), - hits, + hits: results.hits, model_name, limit, + where_rewrites: results.where_rewrites, }))), elapsed_ms: start.elapsed().as_millis() as u64, } @@ -592,7 +593,7 @@ mod tests { ) .await .unwrap(); - assert_eq!(hits.len(), 1); + assert_eq!(hits.hits.len(), 1); } #[tokio::test] @@ -621,7 +622,7 @@ mod tests { .await .unwrap(); - for hit in &hits { + for hit in &hits.hits { assert_ne!(hit.filename, "blog/post2.md"); } } diff --git a/crates/mdvs/src/index/backend/mod.rs b/crates/mdvs/src/index/backend/mod.rs index 6a08c8d..a4c2ef5 100644 --- a/crates/mdvs/src/index/backend/mod.rs +++ b/crates/mdvs/src/index/backend/mod.rs @@ -1,5 +1,9 @@ mod read; mod search; +mod where_translator; + +pub use search::SearchResults; +pub use where_translator::WhereRewrite; use crate::discover::field_type::FieldType; use crate::index::storage::{ @@ -172,7 +176,7 @@ impl Backend { limit: usize, internal_prefix: &str, aliases: &std::collections::HashMap, - ) -> anyhow::Result> { + ) -> anyhow::Result { match self { Backend::Lance(b) => { b.search( @@ -631,7 +635,7 @@ mod tests { .unwrap(); // Query vector close to rust.md's embedding - let hits = backend + let results = backend .search( Some(vec![1.0, 0.0, 0.0, 0.0]), "rust", @@ -643,6 +647,7 @@ mod tests { ) .await .unwrap(); + let hits = results.hits; assert_eq!(hits.len(), 2); assert_eq!(hits[0].filename, "blog/rust.md"); @@ -660,6 +665,24 @@ mod tests { clause, &fields(children), &std::collections::HashSet::new(), + &std::collections::HashSet::new(), + "", + &std::collections::HashMap::new(), + ) + .unwrap() + .clause + } + + fn xlate_with_arrays( + clause: &str, + children: &[&str], + arrays: &[&str], + ) -> super::where_translator::TranslatedWhere { + translate_where_to_struct( + clause, + &fields(children), + &std::collections::HashSet::new(), + &fields(arrays), "", &std::collections::HashMap::new(), ) @@ -691,6 +714,7 @@ mod tests { "file_id = 'x'", &fields(&["file_id"]), &std::collections::HashSet::new(), + &std::collections::HashSet::new(), "", &std::collections::HashMap::new(), ) @@ -706,11 +730,12 @@ mod tests { "file_id = 'x' AND _file_id = 'y'", &fields(&["file_id"]), &std::collections::HashSet::new(), + &std::collections::HashSet::new(), "_", &std::collections::HashMap::new(), ) .unwrap(); - assert_eq!(out, "data.file_id = 'x' AND file_id = 'y'"); + assert_eq!(out.clause, "data.file_id = 'x' AND file_id = 'y'"); } // --- Translator: comparison operators --- @@ -725,8 +750,9 @@ mod tests { #[test] fn translate_where_not_equal_both_spellings() { + // sqlparser canonicalizes both `<>` and `!=` to `<>` on emit. assert_eq!(xlate("rating <> 5", &["rating"]), "data.rating <> 5"); - assert_eq!(xlate("rating != 5", &["rating"]), "data.rating != 5"); + assert_eq!(xlate("rating != 5", &["rating"]), "data.rating <> 5"); } #[test] @@ -783,16 +809,19 @@ mod tests { #[test] fn translate_where_keywords_case_insensitive() { + // sqlparser canonicalizes operators (AND/OR/etc.) to uppercase on + // emit regardless of the user's input case. Semantics unchanged. assert_eq!( xlate("a > 1 and b < 2 Or c = 3", &["a", "b", "c"]), - "data.a > 1 and data.b < 2 Or data.c = 3" + "data.a > 1 AND data.b < 2 OR data.c = 3" ); } #[test] - fn translate_where_boolean_literals_untouched() { + fn translate_where_boolean_literals_canonicalized() { + // sqlparser canonicalizes boolean literals to lowercase on emit. assert_eq!(xlate("draft = true", &["draft"]), "data.draft = true"); - assert_eq!(xlate("draft = FALSE", &["draft"]), "data.draft = FALSE"); + assert_eq!(xlate("draft = FALSE", &["draft"]), "data.draft = false"); } #[test] @@ -805,19 +834,21 @@ mod tests { #[test] fn translate_where_date_literal() { + // sqlparser canonicalizes the typed-string keyword to uppercase. assert_eq!( xlate("published >= date '2024-01-01'", &["published"]), - "data.published >= date '2024-01-01'" + "data.published >= DATE '2024-01-01'" ); } #[test] fn translate_where_date_as_field_name() { // A frontmatter field literally named `date` must be prefixed, while - // the `date '...'` literal keyword on the right is left alone. + // the `date '...'` literal keyword on the right is left alone (and + // canonicalized to `DATE`). assert_eq!( xlate("date >= date '2032-01-01'", &["date"]), - "data.date >= date '2032-01-01'" + "data.date >= DATE '2032-01-01'" ); } @@ -828,7 +859,7 @@ mod tests { "timestamp < timestamp '2024-01-01T00:00:00Z'", &["timestamp"] ), - "data.timestamp < timestamp '2024-01-01T00:00:00Z'" + "data.timestamp < TIMESTAMP '2024-01-01T00:00:00Z'" ); } @@ -839,7 +870,7 @@ mod tests { "synced_at < timestamp '2024-01-01T00:00:00Z'", &["synced_at"] ), - "data.synced_at < timestamp '2024-01-01T00:00:00Z'" + "data.synced_at < TIMESTAMP '2024-01-01T00:00:00Z'" ); } @@ -867,10 +898,11 @@ mod tests { xlate("abs(drift_rate) < 1", &["drift_rate"]), "abs(data.drift_rate) < 1" ); - // function with whitespace before the paren + // Function call with whitespace before the paren — sqlparser + // normalizes the whitespace on emit. assert_eq!( xlate("upper (status) = 'A'", &["status"]), - "upper (data.status) = 'A'" + "upper(data.status) = 'A'" ); } @@ -986,12 +1018,13 @@ mod tests { "file_id = 'x' AND fid = 'y'", &fields(&["file_id"]), &std::collections::HashSet::new(), + &std::collections::HashSet::new(), "", &aliases, ) .unwrap(); // bare `file_id` = frontmatter field → data.file_id; alias `fid` → internal file_id - assert_eq!(out, "data.file_id = 'x' AND file_id = 'y'"); + assert_eq!(out.clause, "data.file_id = 'x' AND file_id = 'y'"); } #[test] @@ -1015,6 +1048,7 @@ mod tests { clause, &fields(&["measurement_values"]), &float_lists, + &std::collections::HashSet::new(), "", &std::collections::HashMap::new(), ) @@ -1034,10 +1068,121 @@ mod tests { "filepath = 'a' AND title = 'b'", &fields(&["filepath", "title"]), &std::collections::HashSet::new(), + &std::collections::HashSet::new(), "", &std::collections::HashMap::new(), ) .unwrap_err(); assert!(err.to_string().contains("ambiguous column 'filepath'")); } + + // --- Translator: array-field rewrites (TODO-0191) --- + + #[test] + fn translate_where_array_equality_rewrites_to_array_has() { + let out = xlate_with_arrays("tags = 'rust'", &["tags"], &["tags"]); + // What we send to Lance: data.-qualified for the denormalized struct. + assert_eq!(out.clause, "array_has(data.tags, 'rust')"); + assert_eq!(out.rewrites.len(), 1); + assert_eq!(out.rewrites[0].original, "tags = 'rust'"); + // What we show the user in the translation note: bare column name, + // matching their mental model. `data.` is mdvs's internal column + // path; the note operates at the user's abstraction level. + assert_eq!(out.rewrites[0].rewritten, "array_has(tags, 'rust')"); + } + + #[test] + fn translate_where_array_equality_literal_on_left() { + let out = xlate_with_arrays("'rust' = tags", &["tags"], &["tags"]); + assert_eq!(out.clause, "array_has(data.tags, 'rust')"); + assert_eq!(out.rewrites.len(), 1); + } + + #[test] + fn translate_where_array_inequality_rewrites_to_not_array_has() { + let out = xlate_with_arrays("tags != 'rust'", &["tags"], &["tags"]); + assert_eq!(out.clause, "NOT array_has(data.tags, 'rust')"); + assert_eq!(out.rewrites.len(), 1); + } + + #[test] + fn translate_where_array_in_list_rewrites_to_or_chain() { + let out = xlate_with_arrays("tags IN ('rust', 'go')", &["tags"], &["tags"]); + assert_eq!( + out.clause, + "array_has(data.tags, 'rust') OR array_has(data.tags, 'go')" + ); + assert_eq!(out.rewrites.len(), 1); + } + + #[test] + fn translate_where_array_not_in_list_rewrites_to_not_or_chain() { + let out = xlate_with_arrays("tags NOT IN ('rust', 'go')", &["tags"], &["tags"]); + // Wrapped in NOT(...) so OR-chain precedence is correct. + assert!( + out.clause + .contains("NOT (array_has(data.tags, 'rust') OR array_has(data.tags, 'go'))") + || out.clause.starts_with( + "NOT (array_has(data.tags, 'rust') OR array_has(data.tags, 'go'))" + ), + "expected NOT-wrapped OR-chain, got: {}", + out.clause + ); + assert_eq!(out.rewrites.len(), 1); + } + + #[test] + fn translate_where_array_inside_array_has_not_double_rewritten() { + // Explicit array_has(...) the user typed must stay as is (modulo + // data. prefixing on the column argument). + let out = xlate_with_arrays("array_has(tags, 'rust')", &["tags"], &["tags"]); + assert_eq!(out.clause, "array_has(data.tags, 'rust')"); + assert!( + out.rewrites.is_empty(), + "no rewrite fired (already canonical form), got: {:?}", + out.rewrites + ); + } + + #[test] + fn translate_where_array_inside_array_length_not_rewritten() { + // array_length(tags) > 2 — `tags` is inside a function, not a direct + // operand of equality. Must stay as a plain qualified identifier. + let out = xlate_with_arrays("array_length(tags) > 2", &["tags"], &["tags"]); + assert_eq!(out.clause, "array_length(data.tags) > 2"); + assert!(out.rewrites.is_empty()); + } + + #[test] + fn translate_where_array_equality_with_data_prefix() { + // User pre-qualified the array field as `data.tags`. The rewrite + // still fires. + let out = xlate_with_arrays("data.tags = 'rust'", &["tags"], &["tags"]); + assert_eq!(out.clause, "array_has(data.tags, 'rust')"); + assert_eq!(out.rewrites.len(), 1); + } + + #[test] + fn translate_where_scalar_equality_not_rewritten_when_not_array() { + // `status` is NOT an array field — `=` stays as `=`, no rewrite. + let out = xlate_with_arrays("status = 'active'", &["status"], &[]); + assert_eq!(out.clause, "data.status = 'active'"); + assert!(out.rewrites.is_empty()); + } + + #[test] + fn translate_where_array_equality_combined_with_scalar_clauses() { + // Mixed clause: scalar AND array. Only the array side fires the + // rewrite; the scalar side stays as a plain equality. + let out = xlate_with_arrays( + "status = 'active' AND tags = 'rust'", + &["status", "tags"], + &["tags"], + ); + assert_eq!( + out.clause, + "data.status = 'active' AND array_has(data.tags, 'rust')" + ); + assert_eq!(out.rewrites.len(), 1); + } } diff --git a/crates/mdvs/src/index/backend/search.rs b/crates/mdvs/src/index/backend/search.rs index 212d9a8..c8686da 100644 --- a/crates/mdvs/src/index/backend/search.rs +++ b/crates/mdvs/src/index/backend/search.rs @@ -26,7 +26,7 @@ use lancedb::query::{ExecutableQuery, QueryBase, Select}; const OVER_FETCH_FACTOR: usize = 3; /// Reserved top-level columns of the denormalized Lance table. -const RESERVED_COLS: &[&str] = &[ +pub(super) const RESERVED_COLS: &[&str] = &[ COL_CHUNK_ID, COL_FILE_ID, COL_CHUNK_INDEX, @@ -44,7 +44,7 @@ const RESERVED_COLS: &[&str] = &[ /// `DATE`/`TIMESTAMP` are intentionally absent: they're keywords only when /// introducing a literal (`date '...'`), which is handled contextually, so a /// plain frontmatter field named `date` still resolves correctly. -const SQL_KEYWORDS: &[&str] = &[ +pub(super) const SQL_KEYWORDS: &[&str] = &[ "AND", "OR", "NOT", @@ -82,30 +82,39 @@ impl LanceBackend { limit: usize, internal_prefix: &str, aliases: &std::collections::HashMap, - ) -> anyhow::Result> { + ) -> anyhow::Result { // `--limit 0` means no results; LanceDB rejects a zero `k`, so short- // circuit rather than surface a cryptic "k must be positive" error. if limit == 0 { - return Ok(vec![]); + return Ok(SearchResults { + hits: vec![], + where_rewrites: vec![], + }); } let Some(table) = self.open_table().await? else { - return Ok(vec![]); + return Ok(SearchResults { + hits: vec![], + where_rewrites: vec![], + }); }; - let translated = match where_clause { + let (translated, where_rewrites) = match where_clause { Some(w) => { let schema = table.schema().await?; let data_children = data_child_names(schema.as_ref()); let float_lists = float_list_child_names(schema.as_ref()); - Some(translate_where_to_struct( + let array_fields = array_child_names(schema.as_ref(), &float_lists); + let result = translate_where_to_struct( w, &data_children, &float_lists, + &array_fields, internal_prefix, aliases, - )?) + )?; + (Some(result.clause), result.rewrites) } - None => None, + None => (None, vec![]), }; let k = limit.saturating_mul(OVER_FETCH_FACTOR); let fts = || FullTextSearchQuery::new(query_text.to_string()); @@ -224,128 +233,26 @@ impl LanceBackend { .unwrap_or(std::cmp::Ordering::Equal) }); hits.truncate(limit); - Ok(hits) + Ok(SearchResults { + hits, + where_rewrites, + }) } } -/// Schema-aware `--where` translator. Frontmatter fields (children of the -/// `data` Struct) are prefixed with `data.`; genuine internal columns are left -/// top-level. A name that is *both* a frontmatter field and an internal column -/// is a collision: it's resolved toward the frontmatter field when -/// `internal_prefix`/aliases give the internal column another name, otherwise -/// it errors (mirroring the old engine). Single-quoted string literals are -/// never rewritten. -pub(super) fn translate_where_to_struct( - clause: &str, - data_children: &std::collections::HashSet, - float_list_fields: &std::collections::HashSet, - internal_prefix: &str, - aliases: &std::collections::HashMap, -) -> anyhow::Result { - // Reverse alias lookup: alias name -> real internal column. - let alias_to_internal: std::collections::HashMap<&str, &str> = aliases - .iter() - .map(|(col, alias)| (alias.as_str(), col.as_str())) - .collect(); - let has_aliasing = !internal_prefix.is_empty() || !aliases.is_empty(); - - // A string literal, optionally preceded by a `date`/`timestamp` keyword so - // the whole `date '...'` literal is protected as one unit (otherwise a - // frontmatter field named `date` and the `date` literal keyword are - // indistinguishable once the literal is split off). - // - // `lazy_regex::regex!` parses these patterns at compile time, so a - // syntax error fails `cargo build` and can never reach a user at - // runtime. - let lit = lazy_regex::regex!(r"(?i)(?:\b(?:date|timestamp)\s+)?'(?:[^']|'')*'"); - let ident = lazy_regex::regex!(r"[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)*"); - - let rewrite_segment = |segment: &str| -> anyhow::Result { - let mut out = String::new(); - let mut last = 0; - for m in ident.find_iter(segment) { - out.push_str(&segment[last..m.start()]); - last = m.end(); - let chain = m.as_str(); - let first = chain.split('.').next().unwrap_or(chain); - - // A bare identifier immediately followed by `(` is a function call - // (lower, length, abs, cast, coalesce, array_length, ...); leave the - // function name and let its column arguments be rewritten normally. - if !chain.contains('.') && segment[last..].trim_start().starts_with('(') { - out.push_str(chain); - continue; - } - - // Keyword / function — leave untouched. - if SQL_KEYWORDS.iter().any(|k| k.eq_ignore_ascii_case(chain)) { - out.push_str(chain); - continue; - } - // Explicit reference to an internal column via its alias or prefix. - if let Some(internal) = alias_to_internal.get(first) { - out.push_str(internal); - continue; - } - if !internal_prefix.is_empty() - && let Some(stripped) = first.strip_prefix(internal_prefix) - && RESERVED_COLS.contains(&stripped) - { - out.push_str(stripped); - continue; - } - - // Reject filters on Array(Float) fields up front — Lance panics on - // them (TODO-0159). The referenced field is `first`, or the segment - // after `data.` when the user pre-qualified. - let fm_name = if first == "data" { - chain.split('.').nth(1) - } else { - Some(first) - }; - if let Some(name) = fm_name - && float_list_fields.contains(name) - { - return Err(anyhow::anyhow!( - "filtering on Array(Float) field '{name}' is not supported in --where. \ - Filter on a different field or store the values in a parallel scalar field." - )); - } - - let is_reserved = RESERVED_COLS.contains(&first); - let is_frontmatter = data_children.contains(first); - let rewritten = if is_reserved && is_frontmatter { - if has_aliasing { - format!("data.{chain}") - } else { - return Err(anyhow::anyhow!( - "ambiguous column '{first}' in --where: it is both a frontmatter field and \ - an internal column. Disambiguate by setting [search].internal_prefix \ - (e.g. \"_\") or [search.aliases].{first} = \"\"" - )); - } - } else if is_reserved { - chain.to_string() - } else { - format!("data.{chain}") - }; - out.push_str(&rewritten); - } - out.push_str(&segment[last..]); - Ok(out) - }; - - let mut out = String::with_capacity(clause.len()); - let mut last = 0; - for m in lit.find_iter(clause) { - out.push_str(&rewrite_segment(&clause[last..m.start()])?); - out.push_str(m.as_str()); - last = m.end(); - } - out.push_str(&rewrite_segment(&clause[last..])?); - Ok(out) +/// Search results bundled with any array-field rewrites that fired during +/// `--where` translation. Surfaced to the user as a "Note" block at the top +/// of the search output so the rewrite isn't magic. +pub struct SearchResults { + /// File-deduped hits, ordered by descending score. + pub hits: Vec, + /// Array-field rewrites — empty when no `--where` clause was passed or + /// when nothing needed rewriting. + pub where_rewrites: Vec, } +pub(super) use super::where_translator::translate_where_to_struct; + /// First-level child field names of the `data` Struct column — i.e. the /// top-level frontmatter field names. Used by the `--where` translator to /// tell frontmatter fields from internal columns. @@ -361,6 +268,35 @@ fn data_child_names(schema: &arrow::datatypes::Schema) -> std::collections::Hash names } +/// Top-level `data` Struct children whose Arrow type is `List` or +/// `LargeList` for any element type *other than* float — i.e. fields +/// declared as `Array(String)`, `Array(Integer)`, `Array(Boolean)`, +/// `Array(Date)`, or `Array(DateTime)` in `mdvs.toml`. Used by the `--where` +/// translator to recognise array-field comparisons and rewrite them as +/// `array_has(...)` calls. `Array(Float)` lists are returned by +/// [`float_list_child_names`] and are rejected up front rather than +/// rewritten. +pub(super) fn array_child_names( + schema: &arrow::datatypes::Schema, + float_list_fields: &std::collections::HashSet, +) -> std::collections::HashSet { + let mut names = std::collections::HashSet::new(); + if let Ok(field) = schema.field_with_name(crate::index::storage::COL_DATA) + && let DataType::Struct(children) = field.data_type() + { + for child in children { + let is_list = matches!( + child.data_type(), + DataType::List(_) | DataType::LargeList(_) + ); + if is_list && !float_list_fields.contains(child.name()) { + names.insert(child.name().clone()); + } + } + } + names +} + /// Top-level `data` Struct children that are lists of floats (`Array(Float)`, /// Arrow `List`). Filtering on these via `--where` panics inside /// lance-encoding 6.0 (TODO-0159), so the translator rejects such references diff --git a/crates/mdvs/src/index/backend/where_translator.rs b/crates/mdvs/src/index/backend/where_translator.rs new file mode 100644 index 0000000..6f16a25 --- /dev/null +++ b/crates/mdvs/src/index/backend/where_translator.rs @@ -0,0 +1,503 @@ +//! AST-based `--where` clause translator. +//! +//! Parses the user's SQL fragment via [`sqlparser`], walks the `Expr` tree, +//! qualifies bare frontmatter field names with `data.`, rewrites array-field +//! comparisons (`=` / `!=` / `IN` / `NOT IN`) to `array_has(...)` forms that +//! Lance can execute, rejects `Array(Float)` references up front, and emits +//! canonical SQL via [`Expr::to_string`]. +//! +//! The translator returns both the rewritten clause and a list of +//! [`WhereRewrite`] entries (one per array-field rewrite that fired) so the +//! caller can surface a translation note to the user. +//! +//! ## Design +//! +//! The previous shape was a regex pass: two regexes (`lit` for literals, +//! `ident` for identifiers) and a walk-and-rewrite loop. Operator-blind by +//! construction — to detect `tags = 'rust'` and rewrite it to +//! `array_has(tags, 'rust')` would have required peek-ahead regexes, +//! function-call heuristics, and special IN-list parsing. Each rule +//! compounded fragility. +//! +//! The AST-based shape gets structural awareness for free: +//! `Expr::BinaryOp { op: Eq, left: Identifier, right: Value(...) }` is +//! exactly the pattern we want to rewrite; an `Identifier` nested inside an +//! `Expr::Function` is structurally distinct from a top-level operand, so +//! `array_length(tags) > 2` never triggers the array-equality rewrite. See +//! TODO-0191 for the full rationale. +//! +//! ## Canonicalization +//! +//! `Expr::to_string()` emits canonical SQL — `AND` / `OR` / `BETWEEN` are +//! uppercased, typed literals (`DATE '…'`, `TIMESTAMP '…'`) keep their +//! keyword, redundant whitespace is normalized. Semantics are preserved; +//! whitespace and case may differ from the user's input. The translation +//! note shows both forms so the user can see what changed. + +use std::collections::{HashMap, HashSet}; + +use serde::Serialize; +use sqlparser::ast::{ + BinaryOperator, Expr, FunctionArg, FunctionArgExpr, FunctionArgumentList, FunctionArguments, + Ident, ObjectName, ObjectNamePart, UnaryOperator, +}; +use sqlparser::dialect::GenericDialect; +use sqlparser::parser::Parser; + +use super::search::{RESERVED_COLS, SQL_KEYWORDS}; + +/// A single array-field comparison that was auto-rewritten. Surfaced to the +/// user in the search outcome's "Note" block so the rewrite isn't magic. +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +pub struct WhereRewrite { + /// The original sub-expression as written by the user (canonicalized via + /// `Expr::to_string` — case and whitespace may differ from the literal + /// input). + pub original: String, + /// The `array_has(...)` form mdvs sent to Lance. + pub rewritten: String, +} + +/// Result of translating a `--where` clause. +#[derive(Debug, Clone)] +pub struct TranslatedWhere { + /// The rewritten clause to hand to Lance. + pub clause: String, + /// Array-field rewrites that fired. Empty if the clause needed no + /// rewriting. + pub rewrites: Vec, +} + +/// Schema-aware `--where` translator. Frontmatter fields (children of the +/// `data` Struct) are prefixed with `data.`; genuine internal columns are +/// left top-level. A name that is *both* a frontmatter field and an internal +/// column is a collision: it's resolved toward the frontmatter field when +/// `internal_prefix`/aliases give the internal column another name, +/// otherwise it errors (mirroring the original regex translator). Comparisons +/// against `Array(*)` fields (other than `Array(Float)`, which is rejected +/// up front) are rewritten to `array_has(...)` forms. +pub(super) fn translate_where_to_struct( + clause: &str, + data_children: &HashSet, + float_list_fields: &HashSet, + array_fields: &HashSet, + internal_prefix: &str, + aliases: &HashMap, +) -> anyhow::Result { + // Empty / whitespace-only input → caller convention is empty in, empty + // out (the search method treats `None` as "no filter"). + if clause.trim().is_empty() { + return Ok(TranslatedWhere { + clause: String::new(), + rewrites: vec![], + }); + } + + // Reverse alias lookup: alias name -> real internal column. + let alias_to_internal: HashMap<&str, &str> = aliases + .iter() + .map(|(col, alias)| (alias.as_str(), col.as_str())) + .collect(); + + let ctx = WalkCtx { + data_children, + float_list_fields, + array_fields, + alias_to_internal, + internal_prefix, + }; + + let mut expr = Parser::new(&GenericDialect {}) + .try_with_sql(clause) + .map_err(|e| anyhow::anyhow!("failed to tokenize --where '{clause}': {e}"))? + .parse_expr() + .map_err(|e| anyhow::anyhow!("failed to parse --where '{clause}': {e}"))?; + + let mut rewrites = Vec::new(); + walk_expr(&mut expr, &ctx, &mut rewrites)?; + + Ok(TranslatedWhere { + clause: expr.to_string(), + rewrites, + }) +} + +struct WalkCtx<'a> { + data_children: &'a HashSet, + float_list_fields: &'a HashSet, + array_fields: &'a HashSet, + alias_to_internal: HashMap<&'a str, &'a str>, + internal_prefix: &'a str, +} + +impl<'a> WalkCtx<'a> { + fn has_aliasing(&self) -> bool { + !self.internal_prefix.is_empty() || !self.alias_to_internal.is_empty() + } +} + +/// Walk the AST in-place. At each node: +/// 1. Try to apply an array-field rewrite (replaces the node entirely). +/// 2. Recurse into children, qualifying any bare identifiers with `data.` +/// and rejecting `Array(Float)` references. +fn walk_expr( + expr: &mut Expr, + ctx: &WalkCtx, + rewrites: &mut Vec, +) -> anyhow::Result<()> { + // Pre-rewrite check: is this whole node an array-field comparison we + // should convert to array_has(...)? If so, replace it and capture the + // before/after pair for the translation note. + if let Some(new_expr) = try_array_rewrite(expr, ctx)? { + let original = expr.to_string(); + let rewritten = new_expr.to_string(); + rewrites.push(WhereRewrite { + original, + rewritten, + }); + *expr = new_expr; + // Fall through and recurse into children of the new expr to qualify + // the array_has(...) argument identifiers. The rewritten form puts + // `tags` inside a Function; the recursion won't re-trigger + // try_array_rewrite (it only fires on BinaryOp/InList, not Function + // arguments). + } + + match expr { + Expr::Identifier(ident) => qualify_single_ident(ident, ctx), + Expr::CompoundIdentifier(parts) => qualify_compound(parts, ctx), + Expr::BinaryOp { left, right, .. } => { + walk_expr(left, ctx, rewrites)?; + walk_expr(right, ctx, rewrites) + } + Expr::UnaryOp { expr: inner, .. } => walk_expr(inner, ctx, rewrites), + Expr::Nested(inner) => walk_expr(inner, ctx, rewrites), + Expr::IsNull(inner) | Expr::IsNotNull(inner) => walk_expr(inner, ctx, rewrites), + Expr::IsTrue(inner) + | Expr::IsNotTrue(inner) + | Expr::IsFalse(inner) + | Expr::IsNotFalse(inner) + | Expr::IsUnknown(inner) + | Expr::IsNotUnknown(inner) => walk_expr(inner, ctx, rewrites), + Expr::Between { + expr: e, low, high, .. + } => { + walk_expr(e, ctx, rewrites)?; + walk_expr(low, ctx, rewrites)?; + walk_expr(high, ctx, rewrites) + } + Expr::InList { expr: e, list, .. } => { + walk_expr(e, ctx, rewrites)?; + for item in list.iter_mut() { + walk_expr(item, ctx, rewrites)?; + } + Ok(()) + } + Expr::Like { + expr: e, + pattern, + escape_char: _, + .. + } => { + walk_expr(e, ctx, rewrites)?; + walk_expr(pattern, ctx, rewrites) + } + Expr::ILike { + expr: e, pattern, .. + } => { + walk_expr(e, ctx, rewrites)?; + walk_expr(pattern, ctx, rewrites) + } + Expr::Function(func) => { + // Recurse into function arguments to qualify any column references. + if let FunctionArguments::List(arg_list) = &mut func.args { + for arg in arg_list.args.iter_mut() { + if let FunctionArg::Unnamed(FunctionArgExpr::Expr(inner)) = arg { + walk_expr(inner, ctx, rewrites)?; + } else if let FunctionArg::Named { + arg: FunctionArgExpr::Expr(inner), + .. + } = arg + { + walk_expr(inner, ctx, rewrites)?; + } + } + } + Ok(()) + } + Expr::Cast { expr: inner, .. } => walk_expr(inner, ctx, rewrites), + Expr::Value(_) | Expr::TypedString { .. } => Ok(()), + // Other variants (subqueries, CASE, INTERVAL, JSON ops, array + // constructors, etc.) we don't touch — anything user-written that + // contains a bare identifier of ours and falls in one of these + // shapes will be passed through to Lance unchanged. The set of + // walked variants covers everything mdvs's existing test suite + // exercises plus the four new array-equality rewrite cases. + _ => Ok(()), + } +} + +fn qualify_single_ident(ident: &mut Ident, ctx: &WalkCtx) -> anyhow::Result<()> { + let name = ident.value.as_str(); + + // SQL keywords / function-like tokens — leave alone. + if SQL_KEYWORDS.iter().any(|k| k.eq_ignore_ascii_case(name)) { + return Ok(()); + } + + // Internal column accessed via alias. + if let Some(internal) = ctx.alias_to_internal.get(name) { + ident.value = (*internal).to_string(); + return Ok(()); + } + + // Internal column accessed via prefix. + if !ctx.internal_prefix.is_empty() + && let Some(stripped) = name.strip_prefix(ctx.internal_prefix) + && RESERVED_COLS.contains(&stripped) + { + ident.value = stripped.to_string(); + return Ok(()); + } + + // Array(Float) rejection — the column is unfilterable. + if ctx.float_list_fields.contains(name) { + return Err(array_float_error(name)); + } + + let is_reserved = RESERVED_COLS.contains(&name); + let is_frontmatter = ctx.data_children.contains(name); + + if is_reserved && is_frontmatter { + if !ctx.has_aliasing() { + return Err(ambiguous_column_error(name)); + } + // With aliasing, bare name resolves to frontmatter — qualify. + // We can't mutate `Ident` into a CompoundIdentifier in place; the + // caller's enclosing Expr is what holds the variant. To work around + // this without restructuring the walk, we encode `data.` as a + // dotted value inside a single Ident — sqlparser's display emits the + // value verbatim, which for our purposes is fine because the + // resulting clause is reparseable. The same trick is used below. + ident.value = format!("data.{name}"); + } else if !is_reserved { + ident.value = format!("data.{name}"); + } + // is_reserved && !is_frontmatter → genuine internal column, leave alone. + + Ok(()) +} + +fn qualify_compound(parts: &mut Vec, ctx: &WalkCtx) -> anyhow::Result<()> { + let Some(first) = parts.first() else { + return Ok(()); + }; + let first_name = first.value.as_str(); + + // Already `data.` qualified — verify the segment after `data.` isn't an + // Array(Float) field (those would panic Lance), then leave alone. + if first_name == "data" { + if let Some(second) = parts.get(1) + && ctx.float_list_fields.contains(second.value.as_str()) + { + return Err(array_float_error(second.value.as_str())); + } + return Ok(()); + } + + // Internal column via alias (e.g. `fid.something` if alias maps to a + // dotted path — uncommon but mirrors the regex translator's behavior). + if let Some(internal) = ctx.alias_to_internal.get(first_name) { + parts[0].value = (*internal).to_string(); + return Ok(()); + } + + // Prefix-resolved internal column. + if !ctx.internal_prefix.is_empty() + && let Some(stripped) = first_name.strip_prefix(ctx.internal_prefix) + && RESERVED_COLS.contains(&stripped) + { + parts[0].value = stripped.to_string(); + return Ok(()); + } + + // Array(Float) rejection — the leading segment is the field name. + if ctx.float_list_fields.contains(first_name) { + return Err(array_float_error(first_name)); + } + + let is_reserved = RESERVED_COLS.contains(&first_name); + let is_frontmatter = ctx.data_children.contains(first_name); + + if is_reserved && is_frontmatter { + if !ctx.has_aliasing() { + return Err(ambiguous_column_error(first_name)); + } + parts.insert(0, Ident::new("data")); + } else if !is_reserved { + parts.insert(0, Ident::new("data")); + } + Ok(()) +} + +/// If `expr` is an array-field equality (`=`, `!=`, `IN`, `NOT IN`) against +/// scalar literals, return the rewritten `array_has(...)` form. Otherwise +/// return `Ok(None)`. +fn try_array_rewrite(expr: &Expr, ctx: &WalkCtx) -> anyhow::Result> { + match expr { + Expr::BinaryOp { left, op, right } => { + let is_eq = matches!(op, BinaryOperator::Eq); + let is_neq = matches!(op, BinaryOperator::NotEq); + if !is_eq && !is_neq { + return Ok(None); + } + // Identify which side is the array-field identifier and which is + // the scalar literal. Both orderings supported. + let (field_name, literal) = if let (Some(f), Some(l)) = + (array_field_name(left, ctx), as_literal(right)) + { + (f, l) + } else if let (Some(f), Some(l)) = (array_field_name(right, ctx), as_literal(left)) { + (f, l) + } else { + return Ok(None); + }; + let array_has = make_array_has(&field_name, literal); + Ok(Some(if is_eq { array_has } else { negate(array_has) })) + } + Expr::InList { + expr: e, + list, + negated, + } => { + let Some(field_name) = array_field_name(e, ctx) else { + return Ok(None); + }; + // Every list entry must be a scalar literal — if any isn't, fall + // back to the un-rewritten clause (Lance will error if needed). + let mut literals = Vec::with_capacity(list.len()); + for item in list { + let Some(lit) = as_literal(item) else { + return Ok(None); + }; + literals.push(lit); + } + // `literals` is guaranteed non-empty here: the early returns + // above bail out on empty / non-literal IN lists. `pop` returns + // None only when the source is empty. + let mut iter = literals.into_iter(); + let Some(first_lit) = iter.next() else { + return Ok(None); + }; + // Build (array_has(f, v1) OR array_has(f, v2) OR ...). + let first = make_array_has(&field_name, first_lit); + let or_chain = iter.fold(first, |acc, lit| Expr::BinaryOp { + left: Box::new(acc), + op: BinaryOperator::Or, + right: Box::new(make_array_has(&field_name, lit)), + }); + Ok(Some(if *negated { + Expr::Nested(Box::new(negate(Expr::Nested(Box::new(or_chain))))) + } else { + or_chain + })) + } + _ => Ok(None), + } +} + +/// If `expr` is a bare identifier (`tags`) or a `data.` compound that +/// names a known array field, return the unqualified field name (so the +/// rewrite can re-emit it as a `data.`-qualified arg to `array_has`). +/// Returns `None` for anything else — function calls, literals, nested +/// expressions, identifiers that don't name array fields. +fn array_field_name(expr: &Expr, ctx: &WalkCtx) -> Option { + match expr { + Expr::Identifier(ident) => { + let name = &ident.value; + if ctx.array_fields.contains(name) { + Some(name.clone()) + } else { + None + } + } + Expr::CompoundIdentifier(parts) => { + // Accept `data.` where is an array field. + if parts.len() == 2 + && parts[0].value == "data" + && ctx.array_fields.contains(&parts[1].value) + { + Some(parts[1].value.clone()) + } else { + None + } + } + _ => None, + } +} + +/// Is this expression a scalar literal we can use as the second arg of +/// `array_has`? Accepts string / number / boolean / typed-string (`DATE`, +/// `TIMESTAMP`) literals. +fn as_literal(expr: &Expr) -> Option { + match expr { + Expr::Value(_) | Expr::TypedString { .. } => Some(expr.clone()), + // A unary minus on a number — `-5` parses as UnaryOp(Minus, Value(5)) + // — is also a scalar literal for our purposes. + Expr::UnaryOp { + op: UnaryOperator::Minus, + expr: inner, + } if matches!(inner.as_ref(), Expr::Value(_)) => Some(expr.clone()), + _ => None, + } +} + +/// Construct `array_has(, )` with a bare-identifier +/// column argument. The recursive walk in [`walk_expr`] qualifies the +/// identifier to `data.` before the clause reaches Lance — but +/// the translation note is captured upstream of that qualification, so the +/// user-facing rewrite reads as `array_has(tags, 'x')` instead of leaking +/// the internal `data.` prefix. +fn make_array_has(field_name: &str, literal: Expr) -> Expr { + Expr::Function(sqlparser::ast::Function { + name: ObjectName(vec![ObjectNamePart::Identifier(Ident::new("array_has"))]), + uses_odbc_syntax: false, + parameters: FunctionArguments::None, + args: FunctionArguments::List(FunctionArgumentList { + duplicate_treatment: None, + args: vec![ + FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::Identifier(Ident::new( + field_name, + )))), + FunctionArg::Unnamed(FunctionArgExpr::Expr(literal)), + ], + clauses: vec![], + }), + filter: None, + null_treatment: None, + over: None, + within_group: vec![], + }) +} + +fn negate(expr: Expr) -> Expr { + Expr::UnaryOp { + op: UnaryOperator::Not, + expr: Box::new(expr), + } +} + +fn array_float_error(name: &str) -> anyhow::Error { + anyhow::anyhow!( + "filtering on Array(Float) field '{name}' is not supported in --where. \ + Filter on a different field or store the values in a parallel scalar field." + ) +} + +fn ambiguous_column_error(name: &str) -> anyhow::Error { + anyhow::anyhow!( + "ambiguous column '{name}' in --where: it is both a frontmatter field and \ + an internal column. Disambiguate by setting [search].internal_prefix \ + (e.g. \"_\") or [search.aliases].{name} = \"\"" + ) +} diff --git a/crates/mdvs/src/index/embed.rs b/crates/mdvs/src/index/embed.rs index ea00b53..e59b597 100644 --- a/crates/mdvs/src/index/embed.rs +++ b/crates/mdvs/src/index/embed.rs @@ -18,7 +18,7 @@ pub enum ModelConfig { /// in production builds. #[cfg(any(test, feature = "testing-mocks"))] Mock { - /// Embedding dimension. Defaults to 256 (matches potion-base-8M). + /// Embedding dimension. Defaults to 256 (matches the potion family). dim: usize, }, } @@ -182,6 +182,11 @@ mod tests { /// invocation skips them — they would otherwise download /// `minishlab/potion-base-8M` from Hugging Face. Run locally with /// `cargo test -- --ignored` once the model is cached. + /// + /// Deliberately the small 8M model, not the production default + /// (`minishlab/potion-multilingual-128M`, ~480 MB) — these tests + /// exercise the model2vec loader path, not embedding quality, so + /// the small model keeps first-run download cheap. const TEST_MODEL: &str = "minishlab/potion-base-8M"; fn test_embedder() -> Embedder { diff --git a/crates/mdvs/src/index/storage.rs b/crates/mdvs/src/index/storage.rs index 5c0ab3f..3f3854d 100644 --- a/crates/mdvs/src/index/storage.rs +++ b/crates/mdvs/src/index/storage.rs @@ -1097,7 +1097,7 @@ mod tests { let meta = BuildMetadata { embedding_model: EmbeddingModelConfig { provider: "model2vec".into(), - name: "minishlab/potion-base-8M".into(), + name: "minishlab/potion-multilingual-128M".into(), revision: None, dim: None, }, diff --git a/crates/mdvs/src/lib.rs b/crates/mdvs/src/lib.rs index e146a7d..27cad83 100644 --- a/crates/mdvs/src/lib.rs +++ b/crates/mdvs/src/lib.rs @@ -29,6 +29,10 @@ pub mod output; pub mod preprocess; /// Shared formatters (`format_pretty`, `format_markdown`) that consume `Vec`. pub mod render; +/// Per-platform scaffolding config (`Platform`, loaded from bundled +/// `scaffolding/platforms//platform.toml`). Drives `mdvs scaffold` +/// and `mdvs hook handle`. +pub mod scaffold; /// Configuration file types (`mdvs.toml`) and shared data structures. pub mod schema; /// Step tree types for the unified command output architecture. diff --git a/crates/mdvs/src/main.rs b/crates/mdvs/src/main.rs index 52777a8..49b0e6d 100644 --- a/crates/mdvs/src/main.rs +++ b/crates/mdvs/src/main.rs @@ -92,7 +92,7 @@ enum Command { /// SQL WHERE clause for filtering #[arg( long = "where", - long_help = "SQL WHERE clause for filtering.\n\nExamples:\n --where \"draft = false\"\n --where \"tags = 'rust'\"\n --where \"author = 'O''Brien'\" (escape ' by doubling)\n\nField names with special characters require SQL quoting:\n --where \"\\\"author's note\\\" = 'value'\"" + long_help = "SQL WHERE clause for filtering.\n\nExamples:\n --where \"draft = false\"\n --where \"tags = 'rust'\" (array fields — auto-rewritten to array_has)\n --where \"tags IN ('rust', 'go')\" (auto-rewritten to OR-chain of array_has)\n --where \"author = 'O''Brien'\" (escape ' by doubling)\n\nField names with special characters require SQL quoting:\n --where \"\\\"author's note\\\" = 'value'\"" )] where_clause: Option, /// Retrieval mode: semantic (vector), fulltext (BM25), or hybrid (both) @@ -154,8 +154,36 @@ enum Command { #[arg(long, value_name = "PATH")] output_file: Option, }, - /// Print the agent skill file to stdout - Skill, + /// Generate install-time artifacts for an agent harness. + /// + /// Subcommands emit either the bundled SKILL.md, the project-rules + /// snippet, or the PostToolUse hook config — to stdout, ready to + /// pipe into the right file under the harness's config dir. + Scaffold { + #[command(subcommand)] + subcommand: mdvs::cmd::scaffold::ScaffoldCommand, + }, + /// Agent-harness hook runtime — called by PostToolUse hooks. + /// + /// Subcommands handle one tool-call payload at a time, reading JSON + /// from stdin and writing a platform-specific JSON envelope to stdout. + Hook { + #[command(subcommand)] + subcommand: HookCommand, + }, +} + +#[derive(Subcommand)] +enum HookCommand { + /// Handle one hook invocation (reads stdin, writes envelope to stdout). + Handle { + /// Platform name (must match a bundled `scaffolding/platforms//`) + #[arg(long)] + platform: String, + /// Which kind of hook this invocation is — drives the runtime logic. + #[arg(long, value_enum)] + kind: mdvs::cmd::hook::HookKind, + }, } #[derive(Subcommand)] @@ -358,8 +386,23 @@ async fn main() -> anyhow::Result<()> { } Ok(()) } - Command::Skill => { - print!("{}", include_str!("../skills/mdvs/SKILL.md")); + Command::Scaffold { subcommand } => { + use mdvs::cmd::scaffold::ScaffoldCommand; + let stdout = std::io::stdout(); + let stderr = std::io::stderr(); + let mut out = stdout.lock(); + let mut err = stderr.lock(); + match subcommand { + ScaffoldCommand::Skill { platform } => { + mdvs::cmd::scaffold::skill::run(&mut out, &mut err, platform.as_deref())?; + } + ScaffoldCommand::Snippet { platform } => { + mdvs::cmd::scaffold::snippet::run(&mut out, &mut err, platform.as_deref())?; + } + ScaffoldCommand::Hook { platform } => { + mdvs::cmd::scaffold::hook::run(&mut out, &mut err, &platform)?; + } + } Ok(()) } Command::Info { path } => { @@ -394,6 +437,14 @@ async fn main() -> anyhow::Result<()> { } Ok(()) } + Command::Hook { subcommand } => match subcommand { + HookCommand::Handle { platform, kind } => { + let stdin = std::io::stdin(); + let mut stdout = std::io::stdout(); + mdvs::cmd::hook::handle::run(stdin.lock(), &mut stdout, &platform, kind)?; + Ok(()) + } + }, } } diff --git a/crates/mdvs/src/outcome/commands/build.rs b/crates/mdvs/src/outcome/commands/build.rs index 4faa375..cde255b 100644 --- a/crates/mdvs/src/outcome/commands/build.rs +++ b/crates/mdvs/src/outcome/commands/build.rs @@ -71,26 +71,6 @@ impl Render for BuildOutcome { .join("\n") }; - let embedded_files_str = if self.embedded_files.is_empty() { - "(none)".into() - } else { - self.embedded_files - .iter() - .map(|f| format!("{} ({})", f.filename, format_chunk_count(f.chunks))) - .collect::>() - .join("\n") - }; - - let removed_files_str = if self.removed_files.is_empty() { - "(none)".into() - } else { - self.removed_files - .iter() - .map(|f| format!("{} ({})", f.filename, format_chunk_count(f.chunks))) - .collect::>() - .join("\n") - }; - let rows = vec![ vec!["full rebuild".into(), self.full_rebuild.to_string()], vec!["files total".into(), self.files_total.to_string()], @@ -102,8 +82,6 @@ impl Render for BuildOutcome { vec!["chunks unchanged".into(), self.chunks_unchanged.to_string()], vec!["chunks removed".into(), self.chunks_removed.to_string()], vec!["new fields".into(), new_fields_str], - vec!["embedded files".into(), embedded_files_str], - vec!["removed files".into(), removed_files_str], ]; blocks.push(Block::Table { @@ -114,6 +92,149 @@ impl Render for BuildOutcome { }, }); + // Per-file lists go in their own Section blocks. Stuffing a 500-row + // file list into a key-value table cell renders as a single + //
-joined string in markdown and a giant tabled cell in the + // terminal — neither is readable. A Section gives both formats a + // proper heading + one-line-per-file structure. + if !self.embedded_files.is_empty() { + blocks.push(Block::Section { + label: "embedded files".into(), + children: self + .embedded_files + .iter() + .map(|f| { + Block::Line(format!("{} ({})", f.filename, format_chunk_count(f.chunks))) + }) + .collect(), + }); + } + + if !self.removed_files.is_empty() { + blocks.push(Block::Section { + label: "removed files".into(), + children: self + .removed_files + .iter() + .map(|f| { + Block::Line(format!("{} ({})", f.filename, format_chunk_count(f.chunks))) + }) + .collect(), + }); + } + blocks } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::render::{format_markdown, format_pretty}; + + fn outcome_with_two_files() -> BuildOutcome { + BuildOutcome { + full_rebuild: true, + files_total: 2, + files_embedded: 2, + files_unchanged: 0, + files_removed: 0, + chunks_total: 3, + chunks_embedded: 3, + chunks_unchanged: 0, + chunks_removed: 0, + new_fields: vec![], + embedded_files: vec![ + BuildFileDetail { + filename: "notes/alpha.md".into(), + chunks: 2, + }, + BuildFileDetail { + filename: "notes/beta.md".into(), + chunks: 1, + }, + ], + removed_files: vec![], + } + } + + /// Per-file lists must not land in the key-value summary table — that + /// produced an unreadable
-joined GFM cell with hundreds of files + /// jammed into it. They belong in their own Section. + #[test] + fn embedded_files_render_as_section_not_table_cell() { + let blocks = outcome_with_two_files().render(); + + // Summary table must NOT contain an "embedded files" row. + let summary_row_cells: Vec<&str> = blocks + .iter() + .filter_map(|b| match b { + Block::Table { rows, .. } => Some(rows), + _ => None, + }) + .flatten() + .map(|row| row[0].as_str()) + .collect(); + assert!( + !summary_row_cells.contains(&"embedded files"), + "summary table still has an 'embedded files' row: {summary_row_cells:?}" + ); + assert!( + !summary_row_cells.contains(&"removed files"), + "summary table still has a 'removed files' row: {summary_row_cells:?}" + ); + + // A Section labeled "embedded files" must exist with one child per file. + let section = blocks + .iter() + .find_map(|b| match b { + Block::Section { label, children } if label == "embedded files" => Some(children), + _ => None, + }) + .expect("expected an 'embedded files' Section"); + assert_eq!(section.len(), 2); + } + + /// Markdown output must put each file on its own line under a `##` + /// heading, NOT inside a `
`-joined table cell. + #[test] + fn markdown_renders_embedded_files_as_heading_and_lines() { + let md = format_markdown(&outcome_with_two_files().render()); + assert!( + md.contains("## embedded files"), + "expected '## embedded files' heading, got:\n{md}" + ); + assert!(md.contains("notes/alpha.md (2 chunks)")); + assert!(md.contains("notes/beta.md (1 chunk)")); + assert!( + !md.contains("notes/alpha.md (2 chunks)
notes/beta.md"), + "file list leaked back into a
-joined table cell:\n{md}" + ); + } + + /// Pretty output must put each file on its own indented line under the + /// section header. + #[test] + fn pretty_renders_embedded_files_as_indented_list() { + let pretty = format_pretty(&outcome_with_two_files().render()); + assert!(pretty.contains("embedded files:")); + assert!(pretty.contains("notes/alpha.md (2 chunks)")); + assert!(pretty.contains("notes/beta.md (1 chunk)")); + } + + /// Empty file lists do NOT emit empty Section blocks (no `## embedded + /// files` heading with nothing under it). + #[test] + fn empty_file_lists_emit_no_section() { + let mut outcome = outcome_with_two_files(); + outcome.embedded_files.clear(); + outcome.removed_files.clear(); + let blocks = outcome.render(); + assert!( + !blocks + .iter() + .any(|b| matches!(b, Block::Section { label, .. } if label == "embedded files" || label == "removed files")), + "empty file lists should not produce Section blocks" + ); + } +} diff --git a/crates/mdvs/src/outcome/commands/search.rs b/crates/mdvs/src/outcome/commands/search.rs index 1ea11c8..70b07d8 100644 --- a/crates/mdvs/src/outcome/commands/search.rs +++ b/crates/mdvs/src/outcome/commands/search.rs @@ -3,7 +3,7 @@ use serde::Serialize; use crate::block::{Block, Render, TableStyle}; -use crate::index::backend::SearchHit; +use crate::index::backend::{SearchHit, WhereRewrite}; /// Full outcome for the search command. #[derive(Debug, Serialize)] @@ -16,12 +16,36 @@ pub struct SearchOutcome { pub model_name: String, /// Result limit that was applied. pub limit: usize, + /// Array-field `--where` rewrites that fired during translation. Empty + /// when no `--where` clause was passed or when nothing needed rewriting. + /// Surfaced as a "Note" block at the top of the rendered output so users + /// see what mdvs sent to Lance. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub where_rewrites: Vec, } impl Render for SearchOutcome { fn render(&self) -> Vec { let mut blocks = vec![]; + // Translation note (above everything else, so users see the rewrite + // before reading results). Suppressed when nothing fired. + if !self.where_rewrites.is_empty() { + let n = self.where_rewrites.len(); + let expr_word = if n == 1 { "expression" } else { "expressions" }; + blocks.push(Block::Section { + label: format!( + "Note — rewrote {n} array-field {expr_word} for List-column matching" + ), + children: self + .where_rewrites + .iter() + .map(|r| Block::Line(format!("{} → {}", r.original, r.rewritten))) + .collect(), + }); + blocks.push(Block::Line(String::new())); + } + // Summary line let hit_word = if self.hits.len() == 1 { "hit" } else { "hits" }; blocks.push(Block::Line(format!( diff --git a/crates/mdvs/src/scaffold/mod.rs b/crates/mdvs/src/scaffold/mod.rs new file mode 100644 index 0000000..4f574ca --- /dev/null +++ b/crates/mdvs/src/scaffold/mod.rs @@ -0,0 +1,24 @@ +//! Agent-harness scaffolding: per-platform configuration and helpers for the +//! `mdvs scaffold {skill,snippet,hook}` and `mdvs hook handle` subcommands. +//! +//! Per-platform behaviour lives as data, not code: each +//! `scaffolding/platforms//platform.toml` declares the harness's skill +//! install path, snippet target file + body selection, and (where applicable) +//! the hook config path + event names + matcher patterns. Adding a new +//! harness is a new toml file — no Rust changes, no release for end users. +//! +//! See [`Platform`] for the deserialised shape. + +use include_dir::{Dir, include_dir}; + +/// Bundled scaffolding tree. Embedded at compile time so the production +/// binary needs no disk access to read per-platform configs or the skill / +/// snippet content. +pub static SCAFFOLDING: Dir<'_> = include_dir!("$CARGO_MANIFEST_DIR/scaffolding"); + +pub mod platform; +pub mod template; + +pub use platform::{ + HookConfigFormat, HooksConfig, Meta, Platform, SkillConfig, SnippetBody, SnippetConfig, +}; diff --git a/crates/mdvs/src/scaffold/platform.rs b/crates/mdvs/src/scaffold/platform.rs new file mode 100644 index 0000000..bb951fb --- /dev/null +++ b/crates/mdvs/src/scaffold/platform.rs @@ -0,0 +1,285 @@ +//! Per-platform configuration loaded from +//! `scaffolding/platforms//platform.toml`. +//! +//! The [`Platform`] struct is the in-memory shape of a `platform.toml`. It's +//! plain data — no enum (so new platforms come from new toml files, not +//! recompiles) and no `dyn Trait` (mdvs is a binary, not a library; the +//! "extensibility surface" lives in toml files, not in Rust types). +//! +//! Loading goes through [`Platform::load`]; enumerating bundled platforms +//! goes through [`Platform::list`]. Both read from the bundled +//! [`super::SCAFFOLDING`] tree, so there's no disk access at runtime. + +use anyhow::{Context, Result, anyhow}; +use serde::{Deserialize, Deserializer}; +use serde_json::Value; + +/// One agent harness's configuration. Deserialised from +/// `scaffolding/platforms//platform.toml`. +#[derive(Debug, Clone, Deserialize)] +pub struct Platform { + /// Identification + display metadata. + pub meta: Meta, + /// Where `mdvs scaffold skill` suggests installing the bundled + /// `SKILL.md`, and where `mdvs hook handle` expects to find it. + pub skill: SkillConfig, + /// Where `mdvs scaffold snippet` writes the project-rules block, and + /// which bundled body template to use. + pub snippet: SnippetConfig, + /// Hook configuration. `None` for harnesses without a shell-command + /// hook surface (OpenCode's TypeScript plugin API, Antigravity's + /// undocumented post-rebrand surface). `mdvs scaffold hook` refuses + /// for these and points the user at the recipe page. + pub hooks: Option, +} + +/// Identification and display metadata for one platform. +#[derive(Debug, Clone, Deserialize)] +pub struct Meta { + /// Canonical platform name. Must match the directory name under + /// `scaffolding/platforms/`. + pub name: String, + /// Human-readable name for help text + documentation. + pub display_name: String, + /// Optional pointer to the harness's hooks / skills documentation. + /// Surfaced in `mdvs scaffold hook` output so users can verify the + /// generated config against the upstream reference. + pub documentation_url: Option, +} + +/// Skill install configuration. +#[derive(Debug, Clone, Deserialize)] +pub struct SkillConfig { + /// Path where the harness expects to find the skill file, relative to + /// the project / workspace root. Surfaced as a help-text hint by + /// `mdvs scaffold skill --platform `. + pub install_path: String, +} + +/// Snippet (project-rules) install configuration. +#[derive(Debug, Clone, Deserialize)] +pub struct SnippetConfig { + /// Target file where the snippet should land (relative to the project + /// root). Examples: `CLAUDE.md`, `AGENTS.md`, `.cursor/rules/mdvs.mdc`. + pub target_file: String, + /// Which bundled body template to use. + pub body: SnippetBody, +} + +/// The available snippet body templates under `scaffolding/snippet/`. +/// +/// New body types require both a new variant here and a corresponding +/// `scaffolding/snippet/.` file — that's why this is a closed +/// enum rather than a free-form string. The set is small and stable. +#[derive(Debug, Clone, Copy, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "kebab-case")] +pub enum SnippetBody { + /// `scaffolding/snippet/agents-md.md` — plain markdown, no frontmatter. + /// Drops into `AGENTS.md`, `CLAUDE.md`, or any always-on rules file. + AgentsMd, + /// `scaffolding/snippet/cursor-rules.mdc` — same body wrapped in + /// Cursor's `.mdc` frontmatter (`alwaysApply: true`). + CursorRules, +} + +/// Hook configuration for a platform. +/// +/// Present only for harnesses with a shell-command hook surface (Claude +/// Code, Codex, Cursor). OpenCode and Antigravity have `Platform::hooks == +/// None`. +/// +/// Per-harness JSON shapes (the envelope `mdvs hook handle` emits and the +/// config `mdvs scaffold hook` emits) live in the [`envelope`] and +/// [`config`] templates as data — see [`super::template`] for the +/// substitution rules. New harnesses with novel shapes can be supported +/// by adding a new `platform.toml` file; no Rust changes required. +#[derive(Debug, Clone, Deserialize)] +pub struct HooksConfig { + /// Path to the harness's hooks config file (relative to the project + /// root). Examples: `.claude/settings.json`, `.codex/hooks.json`, + /// `.cursor/hooks.json`. + pub config_path: String, + /// On-disk format of the hooks config file. + pub config_format: HookConfigFormat, + /// The envelope template `mdvs hook handle` emits. Parsed as JSON at + /// platform-load time; available placeholders are `<>` (agent + /// context, always populated) and `<>` (user channel, + /// populated for `validate` only — pruned for `search-nudge`). + #[serde(deserialize_with = "deserialize_template")] + pub envelope: Value, + /// The config-snippet template `mdvs scaffold hook` emits. Parsed as + /// JSON at platform-load time; available placeholders are + /// `<>` and `<>` (the full `mdvs + /// hook handle …` commands, built from the platform name). + #[serde(deserialize_with = "deserialize_template")] + pub config: Value, +} + +/// Custom serde deserializer that reads a TOML table `{ template = "…" }` +/// and parses the inner string as JSON. The template must be valid JSON at +/// load time — a malformed template surfaces here, not at runtime. +fn deserialize_template<'de, D: Deserializer<'de>>(deserializer: D) -> Result { + #[derive(Deserialize)] + struct TemplateTable { + template: String, + } + let wrapper = TemplateTable::deserialize(deserializer)?; + serde_json::from_str(&wrapper.template).map_err(serde::de::Error::custom) +} + +/// File format of a harness's hook config file. +/// +/// Closed enum because the set is small and any new format requires emit +/// support in `mdvs scaffold hook`. +#[derive(Debug, Clone, Copy, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum HookConfigFormat { + /// JSON, the common case (Claude Code, Codex, Cursor). + Json, + /// TOML — placeholder for harnesses that ship a `[hooks]` table + /// instead of a separate JSON file (Codex also accepts this in + /// `~/.codex/config.toml`). + Toml, +} + +impl Platform { + /// Load the platform config for `name` from the bundled + /// `scaffolding/platforms//platform.toml`. Returns an error if no + /// such platform exists or the toml is malformed. + pub fn load(name: &str) -> Result { + let path = format!("platforms/{name}/platform.toml"); + let file = super::SCAFFOLDING.get_file(&path).ok_or_else(|| { + anyhow!( + "unknown platform '{name}' — bundled platforms are: {}", + Self::list().join(", ") + ) + })?; + let content = file + .contents_utf8() + .ok_or_else(|| anyhow!("platform.toml for '{name}' is not valid UTF-8"))?; + toml::from_str(content).with_context(|| format!("parsing scaffolding/{path}")) + } + + /// List the names of all bundled platforms, sorted alphabetically. + /// Reads from the embedded scaffolding directory; no disk access. + pub fn list() -> Vec { + let Some(platforms_dir) = super::SCAFFOLDING.get_dir("platforms") else { + return Vec::new(); + }; + let mut names: Vec = platforms_dir + .dirs() + .filter_map(|d| { + d.path() + .file_name() + .map(|n| n.to_string_lossy().into_owned()) + }) + .collect(); + names.sort(); + names + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// All five bundled platforms load without error. + #[test] + fn bundled_platforms_load() { + for name in ["claude-code", "codex", "cursor", "opencode", "antigravity"] { + let p = Platform::load(name).unwrap_or_else(|e| { + panic!("failed to load platform '{name}': {e:?}"); + }); + assert_eq!( + p.meta.name, name, + "platform.toml `meta.name` must match dir name" + ); + } + } + + /// `Platform::list` returns the five bundled names in alphabetical order. + #[test] + fn list_returns_bundled_platforms_sorted() { + let got = Platform::list(); + let expected = vec![ + "antigravity".to_string(), + "claude-code".to_string(), + "codex".to_string(), + "cursor".to_string(), + "opencode".to_string(), + ]; + assert_eq!(got, expected); + } + + /// Unknown platform name surfaces a helpful error listing the available + /// platforms. + #[test] + fn unknown_platform_error_lists_known() { + let err = Platform::load("does-not-exist").unwrap_err(); + let msg = format!("{err}"); + assert!(msg.contains("unknown platform 'does-not-exist'"), "{msg}"); + assert!( + msg.contains("claude-code"), + "available platforms should be listed: {msg}" + ); + } + + /// Claude Code's envelope template carries the PostToolUse PascalCase + /// event name baked into the JSON shape. The structural detail — + /// whether event_name lives in a field, deep in a wrapper, or + /// anywhere else — is platform.toml's concern, not Rust's. + #[test] + fn claude_code_envelope_template_uses_post_tool_use() { + let p = Platform::load("claude-code").unwrap(); + let hooks = p.hooks.as_ref().expect("claude-code has [hooks]"); + assert_eq!(hooks.config_path, ".claude/settings.json"); + assert_eq!(hooks.config_format, HookConfigFormat::Json); + // Envelope template contains the right event name as a literal. + let envelope_str = serde_json::to_string(&hooks.envelope).unwrap(); + assert!( + envelope_str.contains("PostToolUse"), + "claude-code envelope should bake in PostToolUse literally: {envelope_str}" + ); + } + + /// Cursor's snippet uses the `.mdc` body (with frontmatter), targeting + /// `.cursor/rules/`. + #[test] + fn cursor_snippet_uses_mdc_body() { + let p = Platform::load("cursor").unwrap(); + assert_eq!(p.snippet.body, SnippetBody::CursorRules); + assert_eq!(p.snippet.target_file, ".cursor/rules/mdvs.mdc"); + } + + /// Only Claude Code ships a verified [hooks] config today. The other + /// four platforms either don't have a shell-hook surface (OpenCode, + /// Antigravity) or had their previous hook config retracted after live + /// smoke tests failed to confirm firing (Codex, Cursor). + #[test] + fn only_claude_code_ships_hooks() { + assert!(Platform::load("claude-code").unwrap().hooks.is_some()); + for name in ["codex", "cursor", "opencode", "antigravity"] { + let p = Platform::load(name).unwrap(); + assert!(p.hooks.is_none(), "{name} should not declare [hooks]"); + } + } + + /// Claude Code's config template references the + /// `<>` and `<>` placeholders. + /// Regression check — `mdvs scaffold hook` won't be able to fill the + /// commands in if claude-code/platform.toml drops the markers. + #[test] + fn config_template_references_command_markers() { + let p = Platform::load("claude-code").unwrap(); + let hooks = p.hooks.as_ref().expect("claude-code has [hooks]"); + let config_str = serde_json::to_string(&hooks.config).unwrap(); + assert!( + config_str.contains("<>"), + "claude-code config template missing COMMAND_VALIDATE marker" + ); + assert!( + config_str.contains("<>"), + "claude-code config template missing COMMAND_SEARCH marker" + ); + } +} diff --git a/crates/mdvs/src/scaffold/template.rs b/crates/mdvs/src/scaffold/template.rs new file mode 100644 index 0000000..966bf23 --- /dev/null +++ b/crates/mdvs/src/scaffold/template.rs @@ -0,0 +1,331 @@ +//! JSON-tree substitution for `<>` placeholders in platform +//! templates. Lets per-platform `platform.toml` files declare arbitrary +//! envelope and config shapes without any Rust code knowing the details +//! of each harness's JSON layout. +//! +//! ## Marker syntax +//! +//! A marker is a JSON string whose **entire content** is `<>` where +//! `NAME` is an uppercase identifier. The full-string-match rule means +//! `"hello <>"` is NOT a marker; it's a literal string with text +//! that happens to contain angle brackets. This avoids the escaping and +//! partial-substitution edge cases of `printf`-style templating. +//! +//! ## Substitution rules +//! +//! Given a `vars: HashMap<&str, Option>`: +//! +//! - `Some(value)` — the marker is replaced with that string value (as a +//! JSON string node). +//! - `None` — the marker is **pruned**: its containing object key (or +//! array element) is removed entirely. Use this for fields that don't +//! apply to a particular invocation (e.g. `systemMessage` on search- +//! nudge hooks that have no user-facing content). +//! - Marker name not present in `vars` — same as `None`, pruned. +//! +//! ## What this enables +//! +//! Per-platform JSON shapes — including substantially different ones — +//! live in `platform.toml` as data, not in Rust. Adding a new harness +//! with a novel envelope shape is one toml file. + +use std::collections::HashMap; + +use serde_json::{Map, Value}; + +/// Walk `template` (a parsed JSON Value) and replace `<>` placeholder +/// strings according to `vars`. See module docs for the substitution rules. +/// +/// Returns a new `Value`; the input is left untouched. +pub fn substitute(template: &Value, vars: &HashMap<&str, Option>) -> Value { + match template { + Value::String(s) => match parse_marker(s) { + Some(name) => match vars.get(name) { + Some(Some(value)) => Value::String(value.clone()), + _ => Value::Null, // sentinel; only reachable if substitute is + // called with a bare-marker root. Callers + // walking objects/arrays prune before recursion. + }, + None => Value::String(s.clone()), + }, + Value::Object(map) => { + let mut out = Map::new(); + for (k, v) in map { + if should_prune(v, vars) { + continue; + } + out.insert(k.clone(), substitute(v, vars)); + } + Value::Object(out) + } + Value::Array(items) => { + let mut out = Vec::new(); + for item in items { + if should_prune(item, vars) { + continue; + } + out.push(substitute(item, vars)); + } + Value::Array(out) + } + other => other.clone(), + } +} + +/// If `s` is a bare marker (entire content matches `<>`), return +/// `Some("NAME")`. Otherwise `None`. +fn parse_marker(s: &str) -> Option<&str> { + s.strip_prefix("<<") + .and_then(|s| s.strip_suffix(">>")) + // Require non-empty name so `"<<>>"` isn't a marker. + .filter(|n| !n.is_empty()) +} + +/// Should this value be pruned from its parent container? True only if +/// the value is a marker string whose name resolves to `None` (or isn't in +/// `vars` at all). +fn should_prune(value: &Value, vars: &HashMap<&str, Option>) -> bool { + let Value::String(s) = value else { + return false; + }; + let Some(name) = parse_marker(s) else { + return false; + }; + matches!(vars.get(name), Some(None) | None) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn vars(pairs: &[(&'static str, Option<&str>)]) -> HashMap<&'static str, Option> { + pairs + .iter() + .map(|(k, v)| (*k, v.map(|s| s.to_string()))) + .collect() + } + + #[test] + fn marker_string_substitutes_when_value_present() { + let template = json!({ "name": "<>" }); + let out = substitute(&template, &vars(&[("NAME", Some("alice"))])); + assert_eq!(out, json!({ "name": "alice" })); + } + + #[test] + fn marker_key_pruned_when_value_is_none() { + let template = json!({ + "name": "<>", + "extra": "<>", + }); + let out = substitute( + &template, + &vars(&[("NAME", Some("alice")), ("MISSING", None)]), + ); + assert_eq!(out, json!({ "name": "alice" })); + } + + #[test] + fn marker_key_pruned_when_name_not_in_vars() { + let template = json!({ + "name": "<>", + "extra": "<>", + }); + let out = substitute(&template, &vars(&[("NAME", Some("alice"))])); + assert_eq!(out, json!({ "name": "alice" })); + } + + #[test] + fn partial_marker_left_as_literal() { + // Only entire-string matches are substituted. Partial = literal. + let template = json!({ + "greet": "hello <>", + "name": "<>", + }); + let out = substitute(&template, &vars(&[("NAME", Some("alice"))])); + assert_eq!(out, json!({ "greet": "hello <>", "name": "alice" })); + } + + #[test] + fn nested_objects_substituted() { + let template = json!({ + "outer": { + "inner": "<>", + "kept": "<>", + } + }); + let out = substitute(&template, &vars(&[("X", Some("foo")), ("Y", Some("bar"))])); + assert_eq!(out, json!({ "outer": { "inner": "foo", "kept": "bar" } })); + } + + #[test] + fn nested_object_prunes_only_keys_not_outer_value() { + // Inner marker missing → inner key pruned. Outer remains as {}. + let template = json!({ + "outer": { "inner": "<>" } + }); + let out = substitute(&template, &vars(&[("MISSING", None)])); + assert_eq!(out, json!({ "outer": {} })); + } + + #[test] + fn array_elements_pruned() { + let template = json!(["<>", "<>", "<>", "literal",]); + let out = substitute( + &template, + &vars(&[("A", Some("a")), ("B", Some("b")), ("MISSING", None)]), + ); + assert_eq!(out, json!(["a", "b", "literal"])); + } + + #[test] + fn non_string_values_pass_through() { + let template = json!({ + "n": 42, + "b": true, + "x": null, + "a": [1, 2, 3], + "s": "<>", + }); + let out = substitute(&template, &vars(&[("NAME", Some("alice"))])); + assert_eq!( + out, + json!({ "n": 42, "b": true, "x": null, "a": [1, 2, 3], "s": "alice" }) + ); + } + + #[test] + fn empty_marker_treated_as_literal() { + // `<<>>` is NOT a valid marker (no name), so it should pass through. + let template = json!({ "field": "<<>>" }); + let out = substitute(&template, &HashMap::new()); + assert_eq!(out, json!({ "field": "<<>>" })); + } + + #[test] + fn empty_string_substitutes_to_empty_string() { + // An empty-string value is Some(""), not None. The key stays in + // the output with an empty string. Important: this differs from + // pruning. Callers can use Some("") to mean "include with empty + // body" or None to mean "remove the field entirely". + let template = json!({ "msg": "<>" }); + let out = substitute(&template, &vars(&[("MSG", Some(""))])); + assert_eq!(out, json!({ "msg": "" })); + } + + #[test] + fn claude_code_validate_envelope_full_substitution() { + // Realistic test: a Claude-Code-shaped envelope template, both + // markers populated. + let template = json!({ + "hookSpecificOutput": { + "hookEventName": "PostToolUse", + "additionalContext": "<>" + }, + "systemMessage": "<>" + }); + let out = substitute( + &template, + &vars(&[ + ("MSG", Some("agent body")), + ("USER_MSG", Some("pretty body")), + ]), + ); + assert_eq!( + out, + json!({ + "hookSpecificOutput": { + "hookEventName": "PostToolUse", + "additionalContext": "agent body" + }, + "systemMessage": "pretty body" + }) + ); + } + + #[test] + fn claude_code_search_nudge_envelope_prunes_user_msg() { + // Realistic test: search-nudge has no user message, so the + // systemMessage key is pruned. + let template = json!({ + "hookSpecificOutput": { + "hookEventName": "PostToolUse", + "additionalContext": "<>" + }, + "systemMessage": "<>" + }); + let out = substitute( + &template, + &vars(&[("MSG", Some("tip body")), ("USER_MSG", None)]), + ); + assert_eq!( + out, + json!({ + "hookSpecificOutput": { + "hookEventName": "PostToolUse", + "additionalContext": "tip body" + } + }) + ); + } + + #[test] + fn cursor_envelope_only_uses_msg() { + // Cursor's shape: snake_case field, no wrapper, no user channel + // from postToolUse. The USER_MSG var isn't referenced anywhere in + // the template — whether it's Some or None makes no difference. + let template = json!({ "additional_context": "<>" }); + let with_user = substitute( + &template, + &vars(&[("MSG", Some("body")), ("USER_MSG", Some("ignored"))]), + ); + let without_user = substitute( + &template, + &vars(&[("MSG", Some("body")), ("USER_MSG", None)]), + ); + assert_eq!(with_user, json!({ "additional_context": "body" })); + assert_eq!(without_user, json!({ "additional_context": "body" })); + } + + #[test] + fn realistic_config_template_with_multiple_markers() { + // Mirrors what scaffold::hook will substitute. + let template = json!({ + "hooks": { + "PostToolUse": [ + { + "matcher": "Edit|Write|MultiEdit", + "hooks": [ + { "type": "command", "command": "<>" } + ] + }, + { + "matcher": "Bash", + "hooks": [ + { "type": "command", "command": "<>" } + ] + } + ] + } + }); + let out = substitute( + &template, + &vars(&[ + ( + "COMMAND_VALIDATE", + Some("mdvs hook handle --platform claude-code --kind validate"), + ), + ( + "COMMAND_SEARCH", + Some("mdvs hook handle --platform claude-code --kind search-nudge"), + ), + ]), + ); + let validate = &out["hooks"]["PostToolUse"][0]["hooks"][0]["command"]; + assert_eq!( + validate.as_str().unwrap(), + "mdvs hook handle --platform claude-code --kind validate" + ); + } +} diff --git a/crates/mdvs/src/schema/config.rs b/crates/mdvs/src/schema/config.rs index e3b622e..abb984a 100644 --- a/crates/mdvs/src/schema/config.rs +++ b/crates/mdvs/src/schema/config.rs @@ -619,7 +619,7 @@ mod tests { }, embedding_model: Some(EmbeddingModelConfig { provider: "model2vec".into(), - name: "minishlab/potion-base-8M".into(), + name: "minishlab/potion-multilingual-128M".into(), revision: None, dim: None, }), @@ -687,7 +687,7 @@ mod tests { }, embedding_model: Some(EmbeddingModelConfig { provider: "model2vec".into(), - name: "minishlab/potion-base-8M".into(), + name: "minishlab/potion-multilingual-128M".into(), revision: Some("abc123".into()), dim: None, }), @@ -732,7 +732,7 @@ allowed = ["blog/**"] required = [] [embedding_model] -name = "minishlab/potion-base-8M" +name = "minishlab/potion-multilingual-128M" [chunking] max_chunk_size = 1024 @@ -1720,7 +1720,7 @@ skip_gitignore = true [embedding_model] provider = "model2vec" -name = "minishlab/potion-base-8M" +name = "minishlab/potion-multilingual-128M" [chunking] max_chunk_size = 1024 diff --git a/crates/mdvs/src/schema/shared.rs b/crates/mdvs/src/schema/shared.rs index 90b02a7..26af83e 100644 --- a/crates/mdvs/src/schema/shared.rs +++ b/crates/mdvs/src/schema/shared.rs @@ -352,7 +352,7 @@ pub struct EmbeddingModelConfig { /// Provider name (e.g. `"model2vec"`). #[serde(default = "default_provider")] pub provider: String, - /// HuggingFace model ID (e.g. `"minishlab/potion-base-8M"`). + /// HuggingFace model ID (e.g. `"minishlab/potion-multilingual-128M"`). pub name: String, /// Pinned revision (commit SHA). #[serde(skip_serializing_if = "Option::is_none")] @@ -741,7 +741,7 @@ mod tests { let w = Wrapper { model: EmbeddingModelConfig { provider: "model2vec".into(), - name: "minishlab/potion-base-8M".into(), + name: "minishlab/potion-multilingual-128M".into(), revision: Some("abc123".into()), dim: None, }, @@ -760,7 +760,7 @@ mod tests { let w = Wrapper { model: EmbeddingModelConfig { provider: "model2vec".into(), - name: "minishlab/potion-base-8M".into(), + name: "minishlab/potion-multilingual-128M".into(), revision: None, dim: None, }, diff --git a/docs/spec/architecture.md b/docs/spec/architecture.md index f15d5ef..42b67f5 100644 --- a/docs/spec/architecture.md +++ b/docs/spec/architecture.md @@ -9,6 +9,8 @@ mdvs has two layers: - **Validation layer** (init, update, check) — scans markdown, infers schema, validates frontmatter. No model needed. Operates on `mdvs.toml`. - **Search layer** (build, search) — chunks markdown, embeds text, stores in a single Lance dataset, queries via LanceDB's native search (semantic / fulltext / hybrid). Requires an embedding model. Operates on `.mdvs/`. +A third install-time subsystem — **agent-harness scaffolding** (`mdvs scaffold {skill,snippet,hook}` install commands plus the `mdvs hook handle` runtime) — wires mdvs into Claude Code / Codex / Cursor / OpenCode / Antigravity from per-platform config files. Per-platform JSON shapes (envelope, install-time config) live as data, not Rust. See [scaffolding.md](scaffolding.md) for the full design. + `mdvs.toml` is the single source of truth for schema. There is no lock file. Build metadata (model identity, chunk size, schema hash) is stored as Lance table-level key-value metadata. ```mermaid @@ -83,7 +85,15 @@ src/ │ ├── search.rs — load model → embed query → execute search │ ├── info.rs — show config + index status │ ├── clean.rs — delete .mdvs/ -│ └── export_jsonschema.rs — translate mdvs.toml → JSON Schema (json/toml output) +│ ├── export_jsonschema.rs — translate mdvs.toml → JSON Schema (json/toml output) +│ ├── scaffold/ — install-time `mdvs scaffold {skill,snippet,hook}` (see scaffolding.md) +│ │ ├── mod.rs — ScaffoldCommand clap enum +│ │ ├── skill.rs — print bundled SKILL.md +│ │ ├── snippet.rs — print project-rules body, picking variant per platform.toml +│ │ └── hook.rs — substitute platform's config template, emit JSON snippet +│ └── hook/ — runtime `mdvs hook handle --platform --kind ` +│ ├── mod.rs — HookKind value-enum +│ └── handle.rs — stdin parse → walk-up → check or search-nudge → envelope │ ├── discover/ │ ├── scan.rs — directory walking, per-file frontmatter dispatch (YAML / TOML / JSON), ScannedFile @@ -98,6 +108,11 @@ src/ │ ├── preprocess.rs — ValueStage enum (Stage 2), Pipeline, infer_value_stages │ +├── scaffold/ — per-platform agent-harness integration data (see scaffolding.md) +│ ├── mod.rs — SCAFFOLDING: Dir<'_> (include_dir bundle of scaffolding/) +│ ├── platform.rs — Platform, HooksConfig, deserialize-with for JSON templates +│ └── template.rs — substitute() with <> placeholders and prune-on-None +│ ├── schema/ │ ├── config.rs — MdvsToml, TomlField, FieldsConfig, validate(), from_inferred() │ ├── shared.rs — ScanConfig, EmbeddingModelConfig, ChunkingConfig, FieldTypeSerde diff --git a/docs/spec/scaffolding.md b/docs/spec/scaffolding.md new file mode 100644 index 0000000..69cf632 --- /dev/null +++ b/docs/spec/scaffolding.md @@ -0,0 +1,217 @@ +# Scaffolding + +How mdvs integrates with agent harnesses (Claude Code, Codex, Cursor, OpenCode, Antigravity) and how to add a new platform. + +This page documents the internal architecture. User-facing instructions live in [book/src/recipes/agentic-harnesses-and-agentic-ides.md](../../book/src/recipes/agentic-harnesses-and-agentic-ides.md). + +## Overview + +Three install-time commands write content to disk in the harness's expected location: + +- `mdvs scaffold skill [--platform ]` — emits a comprehensive `SKILL.md` (Agent Skills standard). +- `mdvs scaffold snippet [--platform ]` — emits a project-rules block to paste into `AGENTS.md` / `CLAUDE.md` / `.cursor/rules/`. +- `mdvs scaffold hook --platform ` — emits a per-platform PostToolUse hook config (JSON snippet) that the harness's settings file ingests. + +One runtime command handles every hook invocation: + +- `mdvs hook handle --platform --kind ` — reads the harness's stdin JSON, walks up to find `mdvs.toml`, runs the requested logic, writes a platform-shaped envelope to stdout, exits 0. + +All four commands share the same per-platform data: `scaffolding/platforms//platform.toml`. Adding a new harness is adding one toml file; no Rust changes. + +## Directory layout (`crates/mdvs/scaffolding/`) + +``` +scaffolding/ +├── skill/SKILL.md — universal skill body, harness-agnostic +├── snippet/ +│ ├── agents-md.md — universal AGENTS.md / CLAUDE.md snippet +│ └── cursor-rules.mdc — same body wrapped in Cursor `.mdc` frontmatter +└── platforms/ + ├── claude-code/platform.toml + ├── codex/platform.toml + ├── cursor/platform.toml + ├── opencode/platform.toml — skill + snippet only, no [hooks] + └── antigravity/platform.toml — skill + snippet only, no [hooks] +``` + +The entire `scaffolding/` tree is bundled into the binary at build time via `include_dir!`. No disk access at runtime; users get a single binary that knows about every platform. + +## `platform.toml` schema + +```toml +[meta] +name = "..." # canonical name, must match the directory name +display_name = "..." # human-readable label for help text +documentation_url = "..." # optional, surfaced for users to verify shape + +[skill] +install_path = "..." # where the harness expects `SKILL.md` + +[snippet] +target_file = "..." # where to append/write the snippet +body = "agents-md" # which bundled body to use; values: agents-md | cursor-rules + +[hooks] # OPTIONAL — omit for harnesses without a shell-command hook surface +config_path = "..." # path to the harness's hooks file +config_format = "json" # current valid: "json" (only) + +[hooks.envelope] +template = """ # JSON, parsed at load time, with <> placeholders +{ ... } +""" + +[hooks.config] +template = """ # JSON, parsed at load time, with <> placeholders +{ ... } +""" +``` + +The `[hooks]` section is what makes a platform "hook-capable." Platforms without it (OpenCode, Antigravity) get full skill + snippet support, but `mdvs scaffold hook --platform ` and `mdvs hook handle --platform ` both refuse with a pointer at the recipe page. + +## Template substitution (`scaffold/template.rs`) + +`platform.toml` declares the JSON shapes the harness expects. Variable parts (the agent message, the user message, the install-time command strings) are filled in at the point of use via `<>` placeholders. + +### Marker syntax + +A marker is a JSON string whose **entire content** matches `<>` where `NAME` is an uppercase identifier. The full-string-match rule means `"hello <>"` is **NOT** a marker — it's a literal string with text that happens to contain angle brackets. This avoids the escaping and partial-substitution edge cases of `printf`-style templating. + +### Substitution rules + +Given `vars: HashMap<&str, Option>`: + +| `vars[NAME]` | Behavior | +|---|---| +| `Some(value)` | Replace the marker string with the value as a JSON string node. | +| `None` | **Prune**: remove the parent key (in objects) or array element. | +| Not present in vars | Same as `None`. | + +The prune-on-`None` rule handles per-kind variance cleanly. Example: Claude Code's envelope template has both `<>` and `<>`. For `validate`, both are populated → full envelope. For `search-nudge`, only `<>` is populated → the `<>` marker is pruned, taking `"systemMessage"` with it. The agent sees the tip in `additionalContext`; the user UI stays quiet. + +Cursor's envelope template doesn't reference `<>` at all (Cursor's `postToolUse` has no user channel). Whether mdvs passes `USER_MSG=Some(...)` or `None` makes no difference — same output either way. + +### Implementation + +```rust +pub fn substitute(template: &Value, vars: &HashMap<&str, Option>) -> Value; +``` + +Walks the parsed `serde_json::Value` tree, returning a new tree with markers replaced and pruned keys/elements removed. No string concatenation, no escaping required from the template author — substitution operates on JSON nodes, not raw text. + +Tests covering all the rules: `scaffold::template::tests` (14 tests). + +## The two consumers + +### `mdvs scaffold hook` (install-time) + +```text +$ mdvs scaffold hook --platform claude-code +``` + +Loads `Platform`, builds two vars: + +- `<>` → `"mdvs hook handle --platform claude-code --kind validate"` +- `<>` → `"mdvs hook handle --platform claude-code --kind search-nudge"` + +Substitutes into `hooks.config` template. Injects a `_comment` field at the top of the resulting object explaining where to merge it. Emits to stdout (clean for piping); install-path hint goes to stderr. + +For platforms without `[hooks]`: refuses with a pointer that still directs users to `mdvs scaffold skill|snippet`. + +### `mdvs hook handle` (runtime) + +```text +$ echo '{"tool_input":{"file_path":"kb/note.md"},"cwd":"/path"}' \ + | mdvs hook handle --platform claude-code --kind validate +``` + +For `--kind validate`: + +1. Read stdin JSON, extract `tool_input.file_path`. Silent if absent or not `.md`. +2. Walk up from the file's directory looking for `mdvs.toml`. Silent if no vault. +3. Run `cmd::check::run` on the vault directly (no subprocess; no shell). +4. If no violations and no error → silent exit 0. +5. Render the result twice: `--output markdown` for the agent channel (uncapped), `--output pretty` for the user channel (capped at `MAX_USER_LINES = 15` with a `...` truncation marker). +6. Append a skill pointer to the agent markdown. +7. Build vars: `<>` = agent markdown, `<>` = pretty (`Some` for validate; `None` would prune). +8. Substitute into `hooks.envelope`. Emit to stdout, exit 0. + +For `--kind search-nudge`: + +1. Read stdin JSON, extract `cwd` (or `pwd` fallback). +2. Walk up to find `mdvs.toml`. Silent if no vault. +3. Match `tool_input.command` against search-tool patterns (`grep`, `rg `, `ripgrep`, `find `, `fd `, `fdfind `, `ag `, `ack `, `git grep`). +4. Build vars: `<>` = the static tip, `<>` = `None` (search-nudge stays out of the user UI). +5. Substitute into `hooks.envelope`. Emit to stdout, exit 0. + +The hook **never blocks** (always exits 0). Violations and tips surface to the agent through `additionalContext`-style channels; the agent decides what to do next. The "warning, not block" design intentionally keeps mdvs out of the harness's permission flow — the schema is meant to evolve with the KB. + +For platforms without `[hooks]`: refuses with the same pointer as `mdvs scaffold hook`. + +## Adding a new platform + +Three categories, ordered by ease. + +### A) New harness with a Claude-Code-style hook config + +Examples of what would qualify: a hypothetical harness that uses the same `{"hooks": {"": [{"matcher": "...", "hooks": [{"type": "command", "command": "..."}]}]}}` nesting but at a different config path or with a different event-name capitalization. + +Steps: + +1. Create `scaffolding/platforms//platform.toml` with the right `[meta]`, `[skill]`, `[snippet]`, `[hooks]` sections. +2. Copy `claude-code/platform.toml`'s `[hooks.envelope]` and `[hooks.config]` templates as a starting point; adjust event names and matcher strings to match the harness's docs. +3. `cargo test --features testing-mocks scaffold::platform` — verifies the toml parses cleanly and the template is valid JSON. Add a per-platform test if the new shape has distinctive landmarks. + +No Rust code changes needed. + +### B) New harness with a completely different JSON shape + +Cursor was the original test case: snake-case `additional_context` at the top level, no `hookSpecificOutput` wrapper, no `systemMessage`-equivalent, flat matcher entries on the config side, top-level `"version": 1`. + +Same three steps as (A), but the templates are written from scratch to match the harness's actual schema. The `<>` and `<>` markers can appear anywhere in the envelope template (or not at all, if the harness has no equivalent channel). The `<>` and `<>` markers are required somewhere in the config template (otherwise `mdvs scaffold hook` would emit an unfilled snippet). + +Still no Rust code changes. + +### C) New harness with a non-shell hook surface + +OpenCode (TypeScript plugin API) and Antigravity (undocumented post-rebrand) are current examples. There's no JSON-config + shell-command pathway for mdvs to slot into. + +Steps: + +1. Create `scaffolding/platforms//platform.toml` with `[meta]`, `[skill]`, `[snippet]` only — omit `[hooks]` entirely. +2. `mdvs scaffold skill` and `mdvs scaffold snippet` will work; `mdvs scaffold hook --platform ` and `mdvs hook handle --platform ` will refuse with a helpful pointer. +3. If/when the harness adds a documented shell-command hook surface, fill in `[hooks]` to upgrade to category (A) or (B). + +### What `<>` names are available + +Hard-coded set, defined in `cmd/hook/handle.rs::build_envelope` (envelope) and `cmd/scaffold/hook.rs::build_config` (config): + +| Marker | Available in | Source | +|---|---|---| +| `<>` | envelope | agent-context message (markdown violation report + skill pointer for validate; static tip for search-nudge) | +| `<>` | envelope | user-visible message (pretty render of violations, capped). `None` for search-nudge (causes prune). | +| `<>` | config | `"mdvs hook handle --platform --kind validate"` | +| `<>` | config | `"mdvs hook handle --platform --kind search-nudge"` | + +Adding a new marker requires a small Rust change in the consumer that fills it; the template engine itself doesn't know about marker names. + +## Rust shape (`crates/mdvs/src/scaffold/`) + +``` +scaffold/ +├── mod.rs — re-exports + SCAFFOLDING: Dir<'_> (the include_dir bundle) +├── platform.rs — Platform, HooksConfig, deserialize-with for templates +└── template.rs — substitute() +``` + +`Platform` is a plain struct loaded from `platform.toml`. No enum (platforms aren't a fixed set), no `dyn Trait` (no library-style downstream extension; this is a binary). New harnesses come from new toml files, never from new Rust types. The project's "enum dispatch, no `dyn Trait`" rule (from [architecture.md](architecture.md#enum-dispatch-pattern)) still applies to concerns where finiteness matters (backends, embedders, value stages); platforms aren't one of those. + +## Why `mdvs hook handle` lives in Rust and not in shell scripts + +mdvs is already a cross-platform Rust binary. The shell-script approach we shipped initially (six `.sh` files across three platforms) only worked on POSIX systems and required `jq` on `PATH` — a real blocker for Windows users and a friction point even on Mac/Linux. Pulling the logic into a single mdvs subcommand: + +- Eliminates the per-OS implementation matrix; one Rust build target serves every OS mdvs already supports. +- Drops the `jq` runtime dependency. +- Lets the install-time `command:` field in the harness's settings file be a one-liner pointing at `mdvs hook handle` — no separate script files to install, no shell-escaping concerns. +- Centralizes the hook contract in one testable place. Bug fixes ship via a normal mdvs release rather than per-user shell-script edits. + +The trade-off is that the per-harness JSON shape now lives inside the binary (via the embedded `platform.toml` files) instead of in editable shell scripts. The template-driven design above mitigates this: users (and contributors) can override by adding a new `platform.toml` to the source tree, no Rust touch required. diff --git a/docs/spec/todos/TODO-0186.md b/docs/spec/todos/TODO-0186.md index 482ee51..aad2779 100644 --- a/docs/spec/todos/TODO-0186.md +++ b/docs/spec/todos/TODO-0186.md @@ -5,7 +5,8 @@ status: todo priority: high created: 2026-06-06 depends_on: [] -blocks: [187] +blocks: [] +related: [190] --- # TODO-0186: `mdvs explain` — proactive schema query @@ -217,3 +218,12 @@ schema." Without this, the agent loop is write→check→fix→repeat; with it, the agent gets the rules upfront and writes valid frontmatter on the first try. Direct enabler for [TODO-0187] (agent harness hook recipe) — especially its pre-explain mode. + +### Follow-up: pre-explain hook mode (v2) + +[TODO-0190](TODO-0190.md) ships `mdvs scaffold hook` with v1 +(post-check) only. **v2 (pre-explain)** — the `PreToolUse` hook +that injects the applicable schema into agent context before the +edit — needs this TODO (`mdvs explain`) to exist. Once 0186 lands, +open a follow-up TODO to add the v2 mode to the per-platform +scaffolding hook scripts. diff --git a/docs/spec/todos/TODO-0187.md b/docs/spec/todos/TODO-0187.md index eda915f..40e9070 100644 --- a/docs/spec/todos/TODO-0187.md +++ b/docs/spec/todos/TODO-0187.md @@ -1,15 +1,23 @@ --- id: 187 title: Recipe — agent harness hook for auto-check (and optional auto-explain / pre-validate) -status: todo +status: done priority: medium created: 2026-06-06 +completed: 2026-06-23 depends_on: [186] blocks: [] +subsumed_by: 190 --- # TODO-0187: Agent harness hook recipe +## Resolution + +Subsumed by [TODO-0190](TODO-0190.md). The `mdvs scaffold hook` command surface and the per-harness recipe pages under `book/src/recipes/agent-harnesses/` ship exactly the auto-check recipe this TODO proposed, plus the cross-platform `mdvs hook handle` runtime (which 0187 sketched as a hand-written shell script). The hook-mode taxonomy (auto-check shipped, auto-explain and pre-validate deferred) is preserved as part of 0190's `--kind` design and the deferral lives in TODO-0186 / TODO-0188. + +## Original scope + ## Problem Today, an agent that writes markdown into an mdvs vault needs to be diff --git a/docs/spec/todos/TODO-0188.md b/docs/spec/todos/TODO-0188.md index 620b77b..fb02d7b 100644 --- a/docs/spec/todos/TODO-0188.md +++ b/docs/spec/todos/TODO-0188.md @@ -6,6 +6,7 @@ priority: medium created: 2026-06-06 depends_on: [] blocks: [] +related: [190] --- # TODO-0188: `mdvs check --stdin` — validate file content in memory @@ -163,3 +164,13 @@ The validation logic is already path-agnostic in spirit — this TODO just plumbs the input boundary so callers can supply content instead of a file path. Small surface change; meaningful capability addition. + +### Follow-up: pre-validate hook mode (v3) + +[TODO-0190](TODO-0190.md) ships `mdvs scaffold hook` with v1 +(post-check) only. **v3 (pre-validate)** — the `PreToolUse` hook +that validates the proposed file content in-memory before the +write reaches disk, blocking bad writes entirely — needs this TODO +(`mdvs check --stdin`) to exist. Once 0188 lands, open a follow-up +TODO to add the v3 mode to the per-platform scaffolding hook +scripts. diff --git a/docs/spec/todos/TODO-0190.md b/docs/spec/todos/TODO-0190.md new file mode 100644 index 0000000..eea8e71 --- /dev/null +++ b/docs/spec/todos/TODO-0190.md @@ -0,0 +1,522 @@ +--- +id: 190 +title: Design `mdvs scaffold` — unified agent-harness integration command surface +status: done +priority: high +created: 2026-06-22 +completed: 2026-06-23 +depends_on: [] +blocks: [] +related: [186, 188] +subsumed: [187] +files_created: + - crates/mdvs/src/scaffold/mod.rs + - crates/mdvs/src/scaffold/platform.rs + - crates/mdvs/src/scaffold/template.rs + - crates/mdvs/src/cmd/scaffold/mod.rs + - crates/mdvs/src/cmd/scaffold/skill.rs + - crates/mdvs/src/cmd/scaffold/snippet.rs + - crates/mdvs/src/cmd/scaffold/hook.rs + - crates/mdvs/src/cmd/hook/mod.rs + - crates/mdvs/src/cmd/hook/handle.rs + - crates/mdvs/scaffolding/skill/SKILL.md + - crates/mdvs/scaffolding/snippet/SNIPPET.md + - crates/mdvs/scaffolding/claude-code/platform.toml + - crates/mdvs/scaffolding/codex/platform.toml + - crates/mdvs/scaffolding/cursor/platform.toml + - crates/mdvs/scaffolding/antigravity/platform.toml + - crates/mdvs/scaffolding/opencode/platform.toml + - crates/mdvs/examples/opencode-plugin/mdvs-hooks.ts + - book/src/recipes/agent-harnesses.md + - book/src/recipes/agent-harnesses/claude-code.md + - book/src/recipes/agent-harnesses/codex.md + - book/src/recipes/agent-harnesses/cursor.md + - book/src/recipes/agent-harnesses/antigravity.md + - book/src/recipes/agent-harnesses/opencode.md + - docs/spec/scaffolding.md +files_updated: + - crates/mdvs/src/cmd/mod.rs + - crates/mdvs/src/main.rs + - crates/mdvs/src/lib.rs + - crates/mdvs/Cargo.toml + - book/src/SUMMARY.md +--- + +# TODO-0190: Design `mdvs scaffold` — unified agent-harness integration command surface + +> **PIVOT 2026-06-22 (afternoon).** Hook logic moves into mdvs itself as `mdvs hook handle --platform --kind {validate|search-nudge}` rather than living in per-platform shell scripts. Driver: Windows support — the shell-based scaffolding shipped in commits `63311cf` + `01d8ff8` only runs on Mac/Linux. Pulling the logic into mdvs (already a cross-platform Rust binary) eliminates the OS dependency, drops the `jq` runtime dependency, and lets new platforms be added via a `platform.toml` config file with no Rust changes. See ["Pivot to cross-platform mdvs-internal hooks"](#pivot-to-cross-platform-mdvs-internal-hooks) below for the new architecture. The shell-script content from `63311cf` + `01d8ff8` is transitional reference material; the production implementation goes through the Rust path. + +## Summary + +Replace the narrow `mdvs skill` command with a broader `mdvs scaffold` surface that emits all three agent-harness integration artifacts — the bundled skill file, the project-rules snippet, and platform-specific hook configs — with `--platform` awareness for the parts that vary across harnesses (Claude Code, Codex, OpenCode, Cursor, Antigravity). Add `mdvs hook handle` as the cross-platform runtime for the hooks themselves: configured per platform via `scaffolding/platforms//platform.toml`, no shell scripts shipped, no `jq` dependency, works on Windows for free. + +This TODO captures the design conversation that happened on 2026-06-22 around the agent-harnesses recipe page; the recipe revealed that hand-written hook JSON, ad-hoc skill install paths, and a not-yet-existing snippet flag all want to live under a single, coherent command. Mid-implementation (commits `63311cf` + `01d8ff8`) the shell-script scaffolding showed it wouldn't work on Windows and the design pivoted to mdvs-internal hooks. + +## Resolution + +Shipped on `feat/mdvs-wire` (PR #66), merged into `main`. The full command surface: + +- `mdvs scaffold skill --platform ` — writes a 368-line agent skill (content identical across platforms; install path varies) into the harness-native `*/skills/mdvs/SKILL.md` location. +- `mdvs scaffold snippet --platform ` — appends a rule snippet ("prefer `mdvs search` over `Grep`") to the harness's rules file (`CLAUDE.md`, `AGENTS.md`, etc.). +- `mdvs scaffold hook --platform ` — emits the harness's PostToolUse hook config wired to `mdvs hook handle`. Refuses with a pointer for OpenCode (TypeScript plugin API only — see `crates/mdvs/examples/opencode-plugin/mdvs-hooks.ts`) and Antigravity (user-level hooks only — documented in the recipe page). +- `mdvs hook handle --platform --kind {validate|search-nudge}` — cross-platform runtime that reads the harness's tool-call JSON on stdin, walks up to find `mdvs.toml`, runs `check` (validate) or formats a search nudge (search-nudge), and prints the platform-specific envelope. Silent on clean / outside-a-vault / hook-disabled — never interrupts the agent. + +**Per-platform shapes are template-driven.** `crates/mdvs/scaffolding//platform.toml` declares the JSON skeleton with `<>` placeholders (prune-on-None). Adding a sixth harness is a config edit, not a new Rust enum variant — confirmed during the design conversation as the right altitude. + +**Five platforms shipped.** `claude-code` (hooks verified end-to-end), `codex` (hook schema-correct per docs; firing status unclear in initial smoke test), `cursor` (hook schema-correct per docs; not observed firing in initial smoke test — needs investigation), `antigravity` (skill + snippet only — no project-level hooks upstream), `opencode` (skill + snippet only — TypeScript bridge plugin documented but did not fire in initial smoke). The Refractions personal vault was used as the install target for all five during development. The skill and snippet halves work everywhere; only Claude Code's hook half is known to work end-to-end. See `book/src/recipes/agent-harnesses.md` "What's tested" section for the current honest status of each. + +**Documentation.** New `book/src/recipes/agent-harnesses/` chapter (overview + one page per harness, alphabetical). First chapter in the Recipes section. `docs/spec/scaffolding.md` captures the architectural decisions (template-driven shapes, walk-up-silent behavior, refusal patterns) for future maintainers. + +**Subsumes [TODO-0187](TODO-0187.md)** (agent harness hook recipe) — the recipe page that 0187 proposed is part of what shipped here. + +**Defers** to follow-ups: TODO-0186 (`mdvs explain`, deferred to post-launch), TODO-0188 (`check --stdin`, deferred). The shipped hook surface assumes the simpler "post-edit validate" semantics; explain and stdin-mode are orthogonal additions. + +The design conversation captured below remains as the audit trail — including the mid-implementation pivot from shell scripts to mdvs-internal hooks — because the pivot's reasoning (Windows support, drop `jq` dependency, no per-platform shell variance) is load-bearing for anyone proposing a v2 design change. + +## Background + +Two things triggered this: + +1. While writing `book/src/recipes/agentic-harnesses-and-agentic-ides.md` we realized the recipe was teaching users to hand-write per-harness hook JSON blocks. That's exactly the kind of platform-specific glue a tool should emit, not the user. +2. We discovered the current validation-hook example was silently broken: the hook's stdout goes to Claude Code's debug log (per [hooks reference](https://code.claude.com/docs/en/hooks)), not to the model. For the agent to see violation feedback, the hook must emit a JSON envelope (`{hookSpecificOutput: {hookEventName: "PostToolUse", additionalContext: "..."}}`) and exit 0. Writing that inline as a `jq` pipeline in every user's `settings.json` is unreasonable; mdvs should emit it. + +## Proposed surface + +``` +mdvs scaffold skill [--platform ] # default: standard .agents/skills/ SKILL.md +mdvs scaffold snippet [--platform ] # default: harness-neutral AGENTS.md block +mdvs scaffold hook --platform # platform required (no sensible default) +``` + +Aliases like `claude` / `cc`, `codex`, `oc`, `cursor`, `antigravity` for ergonomics. + +### What each subcommand emits + +- **`scaffold skill`** — the bundled `SKILL.md` content (refreshed from the existing `crates/mdvs/skills/mdvs/SKILL.md`, 347 lines). Platform variants differ only in install-path hints in surrounding help text; the SKILL.md body itself follows the [Agent Skills open standard](https://agentskills.io) and is the same across harnesses. +- **`scaffold snippet`** — a short ~10–15 line block to paste into the harness's project-rules file (`AGENTS.md` for most; `CLAUDE.md` for Claude Code). Content covers: KB presence, `mdvs search` preference over Grep/Glob, `mdvs check` validation contract, the schema-evolution "warning, not block" rule. +- **`scaffold hook`** — emits both the `.claude/settings.json`-style config block AND the shell-script body for the validate-on-write and search-nudge hooks. Format differs per harness: Claude Code JSON, Codex JSON/TOML, Cursor JSON with camelCase event names. OpenCode and Antigravity refuse with a pointer to the recipe page (different surfaces / undocumented). + +### Envelope wrapping lives in the shell scripts, NOT in mdvs + +An earlier draft of this TODO proposed adding per-platform `OutputFormat` variants (`claude-code-hook`, `codex-hook`, etc.) that would emit the hook-output JSON envelope directly from `mdvs check`. That approach is rejected because: + +- mdvs would have to track each harness's envelope schema, which changes on the harness's release cadence; +- mdvs's CLI surface would gain platform-specific format names that aren't really mdvs concerns; +- per-platform glue belongs in per-platform files — the bundled hook scripts — not in mdvs Rust. + +Instead: + +- **`mdvs check` stays harness-agnostic.** It emits `markdown` or `json` per existing `OutputFormat`. No new variants. +- **Per-platform shell logic wraps mdvs's output in the platform's envelope.** When a harness changes its envelope schema, the update is contained to that platform's shell, not in mdvs Rust. + +### Concrete shell sketches + +Two distinct hook shapes, both for Claude Code's PostToolUse (Codex follows the same pattern unchanged; Cursor swaps `PostToolUse` → `postToolUse` in the envelope and the matcher): + +**Validate-on-write hook** — separate `.sh` file, captures mdvs output, wraps it in the envelope when non-empty. Lives at `scaffolding/hooks/claude-code/validate.sh`: + +```bash +#!/usr/bin/env bash +set -euo pipefail +f=$(jq -r '.tool_input.file_path // empty') +[ -z "$f" ] && exit 0 +case "$f" in *.md) ;; *) exit 0 ;; esac +dir=$(dirname "$f") +while [ "$dir" != "/" ] && [ ! -f "$dir/mdvs.toml" ]; do + dir=$(dirname "$dir") +done +[ ! -f "$dir/mdvs.toml" ] && exit 0 +out=$(mdvs check "$dir" --output markdown 2>&1) || true +[ -z "$out" ] && exit 0 +jq -n --arg msg "$out" \ + '{hookSpecificOutput: {hookEventName: "PostToolUse", additionalContext: $msg}}' +``` + +The wrapping `.claude/settings.json` just calls it: + +```json +{ + "matcher": "Edit|Write|MultiEdit", + "hooks": [{ "type": "command", "command": "/abs/path/.claude/hooks/mdvs-validate.sh" }] +} +``` + +**Search-nudge hook** — inline in `settings.json`, pure literal envelope, no captured output: + +```json +{ + "matcher": "Bash", + "hooks": [{ + "type": "command", + "command": "cmd=$(jq -r '.tool_input.command // empty'); case \"$cmd\" in *kb/*) case \"$cmd\" in *grep*|*rg\\ *|*find\\ *|*fd\\ *|*ag\\ *|*ack\\ *|*\"git grep\"*) echo '{\"hookSpecificOutput\":{\"hookEventName\":\"PostToolUse\",\"additionalContext\":\"Tip: `mdvs search` is also available for semantic / hybrid / SQL-filtered search over the KB.\"}}' ;; esac ;; esac" + }] +} +``` + +Both hooks exit 0 unconditionally; the harness treats the JSON envelope on stdout as "additional context the model should see" — the non-blocking warning path. + +### Per-artifact variants — what actually differs per platform + +Not every artifact needs platform-specific content. Concrete count: + +| Artifact | Variants | What differs | What stays universal | +|---|---|---|---| +| `SKILL.md` | **1** | Suggested install path comment in help text | Body content — the Agent Skills standard is shared across all five harnesses; the SKILL.md teaches the agent what mdvs is and when to call it, which is harness-agnostic. | +| Snippet (project-rules) | **2** | (a) Plain markdown for `CLAUDE.md` / `AGENTS.md`, (b) `.mdc`-wrapped (frontmatter with `alwaysApply: true`) for Cursor `.cursor/rules/mdvs.mdc`. | Body instructions to the agent — same everywhere. | +| Validate-on-write hook | **3** | Settings file path (`.claude/settings.json` vs `.codex/hooks.json` vs `.cursor/hooks.json`); event-name capitalization (`PostToolUse` vs `postToolUse`); tool matcher names (`Edit\|Write\|MultiEdit` vs equivalents). | The walk-up-to-`mdvs.toml` logic; the `mdvs check --output markdown` invocation; the empty-output-means-silent contract. The `.sh` body is ~95% identical across platforms — only the envelope's event-name string differs. | +| Search-nudge hook | **3** | Same platform-specific surface as validate hook, embedded inline in the settings file. | The case-match against `grep`/`rg`/`find`/`fd`/etc.; the KB-path scoping; the literal-envelope echo pattern. | + +So `mdvs scaffold skill` and `mdvs scaffold snippet` have very small platform surface area. `mdvs scaffold hook` is where the per-platform code lives — three variants covering Claude Code, Codex, Cursor. + +### `mdvs check` operates on the vault, not on a file + +`mdvs check [PATH]` takes a directory containing `mdvs.toml`, not a file path. There is no per-file validation mode today (that would need [TODO-0188](TODO-0188.md) `mdvs check --stdin`). The hook flow is therefore: + +1. Hook fires on `Edit` / `Write`. +2. Read `file_path` from the stdin JSON. +3. Walk up the file's parent directories to find `mdvs.toml` — this is the vault root. +4. Run `mdvs check --output markdown`. +5. Wrap the output in the platform's envelope (shell script's job). +6. Exit 0. + +## Decisions (Step 1 complete) + +Twelve open design questions surfaced during the 2026-06-22 design conversation; all settled. Notes below for the record. + +1. ~~**Name.**~~ **Decided: `scaffold`.** The verb specifically means "generate boilerplate," which fits exactly. Directory under the crate is `scaffolding/` (singular mass noun, like `documentation/`). Alternatives considered: `wire` (shorter but more abstract), `adapter`, `agent`. +2. ~~**Subcommand order.**~~ **Decided: verb-object.** `scaffold skill | snippet | hook` — reads as an install command. +3. ~~**Hook output shape.**~~ **Decided: stdout-only in v1.** `scaffold hook` prints one big text block with clearly-delimited config + script sections, each prefixed by header comments explaining destination paths. `--write` flag deferred to v2 once paths and behavior are stable. +4. ~~`mdvs check --output ` naming.~~ **Dropped** — envelope wrapping moved to per-platform shell scripts (see "Envelope wrapping lives in the shell scripts, NOT in mdvs" above). +5. ~~**Relationship to [TODO-0187](TODO-0187.md).**~~ **Decided: subsume.** 0187's three composable hook modes (post-check / pre-explain / pre-validate) carry over as v1/v2/v3 of `scaffold hook`. 0190 ships **v1 (post-check) only**; v2 and v3 unlock when [TODO-0186](TODO-0186.md) and [TODO-0188](TODO-0188.md) land respectively, and are tracked in those TODOs' follow-up notes. 0187's exit-2/stderr output is replaced by the JSON envelope + exit 0 path. 0187 gets marked `subsumed_by: 190` when 0190 v1 ships. +6. ~~**Pre-1.0 migration.**~~ **Decided: hard rename.** `mdvs skill` is removed entirely; `mdvs scaffold skill` replaces it. No alias surface. Help text for the old name (if a stub is kept transiently) errors with a pointer. +7. ~~**Cursor split.**~~ **Decided: emit AGENTS.md form by default; the `.mdc` variant ships as a second template under `scaffolding/snippet/cursor-rules.mdc`** (selected by `--platform cursor` with optional `--target cursor-rules`). Body content identical; only the `.mdc` frontmatter wrapper differs. +8. ~~**OpenCode hooks.**~~ **Decided: refuse with pointer in v1.** OpenCode's hook surface is a TypeScript plugin API ([source](https://opencode.ai/docs/agents/)); no shell-command config. `scaffold hook --platform opencode` exits with a message pointing at the recipe page section that describes the community shell-hook package. Re-evaluate when/if OpenCode adds a first-class shell-hook config. +9. ~~**Antigravity hooks.**~~ **Decided: refuse with pointer in v1.** Upstream documentation is incomplete; inherited Gemini CLI hook docs use different event names (`BeforeTool` / `AfterTool`) but Antigravity's post-rebrand schema isn't published. Skill + snippet work on Antigravity (verified via [Google's authoring codelab](https://codelabs.developers.google.com/getting-started-with-antigravity-skills)); hooks await documentation. +10. ~~**Auto-detect platform.**~~ **Decided: no in v1.** Explicit `--platform` for every invocation. Auto-detect from cwd (`.claude/`, `.codex/`, etc.) is v2 work. +11. ~~**Inline shell vs separate script file.**~~ **Decided (initial): separate `.sh` for validate, inline for search-nudge.** **Re-decided (pivot 2026-06-22): neither — both move into mdvs as `mdvs hook handle`.** See [Pivot section](#pivot-to-cross-platform-mdvs-internal-hooks). The shell approach was reversed because it couldn't ship on Windows. +12. ~~**stdin parser choice.**~~ **Decided (initial): `jq`.** **Made moot by the pivot:** mdvs reads stdin JSON natively via serde, no external parser needed. The skill content references to `jq` are now historical / pedagogical (still useful for users who want to understand the contract). + +## Pivot to cross-platform mdvs-internal hooks + +**When:** 2026-06-22, mid-Step-3, after shipping `63311cf` (Claude Code .sh) and `01d8ff8` (Codex + Cursor .sh + search-nudge extraction + example_kb dogfooding). + +**Why:** Everything shipped is POSIX shell, requires `jq` on PATH, and won't run on Windows. mdvs is a cross-platform Rust binary; the hook glue should be too. The architectural argument that drove decision #4 ("envelope wrapping in shell, not mdvs") was based on the assumption that envelope schemas change per-harness on a fast cadence — in practice they don't (Claude/Codex/Cursor all converged on `{hookSpecificOutput: {hookEventName, additionalContext}}` with only the event-name capitalization differing). The cross-platform benefit outweighs the keep-mdvs-thin argument. + +### New architecture + +Three changes: + +1. **`mdvs hook handle --platform --kind `** — new subcommand. Reads stdin JSON, runs the same logic the shell scripts did (walk-up to `mdvs.toml`, run `mdvs check` internally for validate / pattern-match command for search-nudge, wrap output in the per-platform envelope), prints to stdout, exits 0. All cross-platform Rust. No `jq`. No shell. + +2. **Per-platform behaviour is data, not code.** `scaffolding/platforms//platform.toml` declares everything that varies between harnesses: + + ```toml + [meta] + name = "claude-code" + display_name = "Claude Code" + + [skill] + install_path = ".claude/skills/mdvs/SKILL.md" + + [snippet] + target_file = "CLAUDE.md" + body = "agents-md" # key into scaffolding/snippet/ + + [hooks] + config_path = ".claude/settings.json" + config_format = "json" # or "toml" for Codex's [hooks] table form + event_name_validate = "PostToolUse" + event_name_search = "PostToolUse" + matcher_validate = "Edit|Write|MultiEdit" + matcher_search = "Bash" + + [hooks.envelope] + template = ''' + { + "hookSpecificOutput": { + "hookEventName": "$event_name", + "additionalContext": $msg + }, + "systemMessage": $user_msg + } + ''' + + [hooks.stdin_paths] + file_path = ".tool_input.file_path" + command = ".tool_input.command" + cwd = ".cwd" + ``` + + Adding a new harness = adding a new `platform.toml`. No Rust changes, no release for end users. + +3. **`scaffolding/` becomes config-only.** No more `.sh` files under `scaffolding/hooks//`. The bundled tree at ship time: + + ``` + scaffolding/ + ├── skill/SKILL.md ← universal + ├── snippet/ + │ ├── agents-md.md ← universal body + │ └── cursor-rules.mdc ← Cursor frontmatter wrap + └── platforms/ + ├── claude-code/platform.toml + ├── codex/platform.toml + ├── cursor/platform.toml + ├── opencode/platform.toml ← skill+snippet only, no [hooks] + └── antigravity/platform.toml ← same + ``` + + Bundled into the binary via `include_dir!` at build time. Loaded as `Platform` structs at runtime (not enums, not `dyn Trait` — plain data, see "Rust shape" below). + +### What the settings.json / hooks.json templates emit + +After pivot, `mdvs scaffold hook --platform claude-code` emits a config that calls `mdvs` directly — no shell wrapper, no script files: + +```json +{ + "hooks": { + "PostToolUse": [ + { "matcher": "Edit|Write|MultiEdit", + "hooks": [{ "type": "command", + "command": "mdvs hook handle --platform claude-code --kind validate" }]}, + { "matcher": "Bash", + "hooks": [{ "type": "command", + "command": "mdvs hook handle --platform claude-code --kind search-nudge" }]} + ] + } +} +``` + +That's it. Users only need `mdvs` on PATH (which they already do for everything else). No `jq`, no `.sh` files, works the same on every OS. + +### Rust shape (plain struct, no enum, no `dyn Trait`) + +```rust +pub struct Platform { + pub meta: Meta, + pub skill: SkillConfig, + pub snippet: SnippetConfig, + pub hooks: Option, // None for OpenCode (TS plugin) / Antigravity (undocumented) +} + +impl Platform { + pub fn load(name: &str) -> Result { + let toml = scaffolding::read(&format!("platforms/{name}/platform.toml"))?; + toml::from_str(&toml).context("parsing platform config") + } + + pub fn emit_envelope(&self, msg: &str, user_msg: Option<&str>, event_kind: HookKind) -> String { ... } +} + +pub fn list_platforms() -> Vec { + scaffolding::dir("platforms").iter().map(|d| d.name()).collect() +} +``` + +No enum because platforms aren't a fixed set known at compile time — the whole point of the config-driven design is that new platforms come from new toml files. The project's "enum dispatch, no `dyn Trait`" rule still applies elsewhere (backends, embedders, value stages — concerns where finiteness genuinely matters); platforms aren't one of those. + +### What this supersedes / preserves from the work already shipped + +- **Superseded:** `scaffolding/hooks//{validate,search-nudge}.sh` and `{settings,hooks}.json` templates from commits `63311cf` + `01d8ff8`. The shell scripts get deleted; the JSON templates get regenerated to call `mdvs hook handle`. +- **Preserved:** `scaffolding/skill/SKILL.md` (universal, unchanged). `scaffolding/snippet/{agents-md.md,cursor-rules.mdc}` (universal, unchanged). The `example_kb/AGENTS.md` and `CLAUDE.md` symlink (unchanged). +- **Reshaped:** `example_kb/.{claude,codex,cursor}/` get cleaned up — no more `.sh` symlinks under `hooks/`, just config files calling `mdvs hook handle`. `example_kb/.agents/skills/` stays as-is (cross-harness skill path). + +### Why the existing scaffolding scripts are not lost work + +They documented the exact shell logic the new Rust subcommand needs to implement. They're the executable spec for `mdvs hook handle`. The Rust impl is a direct port: + +- Read stdin JSON → `serde_json::from_reader(stdin())` +- Walk up to `mdvs.toml` → loop with `std::fs::metadata` +- Run `mdvs check` → call the existing `cmd::check` module directly (no subprocess) +- Wrap in envelope → templated string substitution per `platform.toml` +- Emit to stdout → `println!` +- Exit 0 + +## Platforms — confirmed conventions (research summary) + +| Harness | Skill path | Snippet file | Hook config | Hook events | Hook input | Hook output for "model context" | +|---|---|---|---|---|---|---| +| Claude Code | `.claude/skills//SKILL.md` | `CLAUDE.md` | `.claude/settings.json` | `PreToolUse`, `PostToolUse` | stdin JSON | JSON envelope `{hookSpecificOutput: {hookEventName, additionalContext}}` | +| Codex | `.agents/skills//SKILL.md` | `AGENTS.md` (or `AGENTS.override.md`) | `.codex/hooks.json` or `[hooks]` in `config.toml` | `PreToolUse`, `PostToolUse` | stdin JSON | JSON envelope, same shape as Claude Code | +| OpenCode | `.opencode/skills/`, `.claude/skills/`, `.agents/skills/` (all read) | `AGENTS.md` | TypeScript plugin API (`tool.execute.before/after`) — no shell-config file | n/a (plugin) | n/a (plugin) | n/a (plugin) | +| Cursor | `.cursor/skills/`, `.agents/skills/`, `.claude/skills/`, `.codex/skills/` (all read) | `AGENTS.md` or `.cursor/rules/*.mdc` | `.cursor/hooks.json` | camelCase: `postToolUse` etc. | stdin JSON | JSON envelope (verify exact shape during impl) | +| Antigravity | `.agents/skills//SKILL.md` | `AGENTS.md` | undocumented at time of writing | unknown | unknown | unknown | + +Sources (saved for citation): + +- [Agent Skills open standard](https://agentskills.io) +- Claude Code: [skills](https://code.claude.com/docs/en/skills), [hooks](https://code.claude.com/docs/en/hooks) +- Codex: [skills](https://developers.openai.com/codex/skills/), [hooks](https://developers.openai.com/codex/hooks), [AGENTS.md guide](https://developers.openai.com/codex/guides/agents-md) +- OpenCode: [skills](https://opencode.ai/docs/skills/), [agents](https://opencode.ai/docs/agents/), [rules](https://opencode.ai/docs/rules/) +- Cursor: [skills](https://cursor.com/docs/context/skills), [rules](https://cursor.com/docs/rules), [hooks](https://cursor.com/docs/hooks) +- Antigravity: [authoring skills codelab](https://codelabs.developers.google.com/getting-started-with-antigravity-skills), [Gemini CLI configuration (inherited)](https://github.com/google-gemini/gemini-cli/tree/main/docs/hooks) + +## Implementation impact (sketch — for sizing only) + +Updated for the post-pivot architecture. Pre-pivot bullets (shell-script bundling, jq dependency, no new mdvs subcommands) are obsolete — see the "Pivot to cross-platform mdvs-internal hooks" section above for what replaced them. + +- **New CLI subcommand `mdvs hook handle --platform --kind `** — the runtime that hooks call. Reads stdin JSON via serde, walks up to `mdvs.toml`, invokes `cmd::check` directly (no subprocess), wraps the markdown body in the per-platform envelope from `platform.toml`, emits to stdout, exits 0. +- **New CLI subcommand structure under `crates/mdvs/src/cmd/scaffold/`** (scaffold) and `crates/mdvs/src/cmd/hook/` (hook). The two share a `Platform` loader. +- **Reshape `crates/mdvs/scaffolding/` to be config-only:** + - Keep: `skill/SKILL.md`, `snippet/{agents-md.md,cursor-rules.mdc}`. + - Add: `platforms//platform.toml` for each supported harness (claude-code, codex, cursor, opencode, antigravity). + - Remove: `hooks//*.sh` and `{settings,hooks}.json` templates (the templates get *generated* by `mdvs scaffold hook` from `platform.toml` rather than bundled). +- **`Platform` struct + loader in `crates/mdvs/src/scaffold/platform.rs`** (new module). Plain struct, deserialised from `platform.toml` via `serde + toml`. No enum, no `dyn Trait` — the project's enum-dispatch rule still applies for backends/embedders/etc., but platforms are data-driven (new platform = new toml file, no Rust release). +- **Bundle `scaffolding/` into the binary** via `include_dir!` (or equivalent). Users get a single mdvs binary that contains all the per-platform data. +- **Skill body refresh** (`crates/mdvs/scaffolding/skill/SKILL.md`, currently 488 lines): drop the section that taught the agent to expect `jq`-wrapped envelopes (mdvs handles the wrapping now); the agent only needs to know about the `additionalContext` semantic. Replace `mdvs skill` references with `mdvs scaffold skill`. The schema-evolution loop content stays. +- **Hard rename `mdvs skill` → `mdvs scaffold skill`** (breaking, pre-1.0 so acceptable). +- **Update `book/src/recipes/agentic-harnesses-and-agentic-ides.md`** to use the new commands and the no-shell install path. Drop references to `validate.sh` / `search-nudge.sh` files; the recipe shows users dropping a one-line `command:` into their harness config. +- **Delete the shell scripts from commits `63311cf` + `01d8ff8`** as part of the cutover. The work isn't lost — it documented the executable spec that `mdvs hook handle` now implements in Rust. + +## Implementation plan + +Restructured after the 2026-06-22 pivot. Steps 1–3 of the original plan shipped in commits `63311cf` + `01d8ff8` and produced the shell-script scaffolding that's now being superseded — those steps are kept for history; new pivot-aware steps follow. + +### Step 1 — Settle the load-bearing design decisions (≈ 30 min) ✓ DONE 2026-06-22 + +All twelve open design questions settled. See "Decisions" section above. (Decisions #11 and #12 were later reversed by the pivot.) + +### Step 2 — Author all content artifacts (3–4 hours) ✓ DONE 2026-06-22 (commit `63311cf`) + +Produced under `crates/mdvs/scaffolding/`: +- `skill/SKILL.md` +- `snippet/agents-md.md`, `snippet/cursor-rules.mdc` +- `hooks/claude-code/validate.sh` + `settings.json` + +End-to-end tested against `example_kb` on Claude Code. Caught one real bug (silent gate using output emptiness instead of exit code). + +### Step 3 — Add Codex + Cursor hook variants (1–2 hours) ✓ DONE 2026-06-22 (commit `01d8ff8`) + +Codex + Cursor `.sh` + config templates, plus extraction of the search-nudge to its own script (with cwd-based walk-up replacing the brittle `*kb/*` pattern), plus full `example_kb` dogfooding across `.claude` / `.agents` / `.codex` / `.cursor`. + +### Step 3b — Pivot decision (≈ 0 min, conversational) ✓ DONE 2026-06-22 + +Recognised that the shell-script approach won't ship on Windows. Decision to pull hook logic into `mdvs hook handle` as a cross-platform Rust subcommand and make platform behaviour data-driven via `scaffolding/platforms//platform.toml`. See ["Pivot to cross-platform mdvs-internal hooks"](#pivot-to-cross-platform-mdvs-internal-hooks). + +### Step 4 — Design and write `platform.toml` for each supported harness (1–2 hours) + +Write the per-platform config files that the new `mdvs hook handle` + `mdvs scaffold` commands read at runtime: + +- `crates/mdvs/scaffolding/platforms/claude-code/platform.toml` +- `crates/mdvs/scaffolding/platforms/codex/platform.toml` +- `crates/mdvs/scaffolding/platforms/cursor/platform.toml` +- `crates/mdvs/scaffolding/platforms/opencode/platform.toml` (skill + snippet only, no `[hooks]` table) +- `crates/mdvs/scaffolding/platforms/antigravity/platform.toml` (same) + +Schema sketch is in the Pivot section. Validate each one parses with `toml::from_str` round-tripping into the eventual `Platform` struct. + +**Verification:** Each toml file parses cleanly. The set of declared fields covers everything the shell scripts encoded as constants (event names, matchers, install paths, envelope shapes). + +### Step 5 — Implement `Platform` struct + loader in Rust (2–3 hours) + +New module `crates/mdvs/src/scaffold/platform.rs`: + +- `pub struct Platform { meta, skill, snippet, hooks: Option }` +- `Platform::load(name: &str) -> Result` — reads `scaffolding/platforms//platform.toml` (bundled via `include_dir!`) +- `Platform::list() -> Vec` — directory enumeration over bundled platforms +- `HooksConfig::emit_envelope(&self, msg, user_msg, kind) -> String` — templated substitution + +Add `include_dir` (or equivalent) as a dependency to embed `scaffolding/` at build time. + +**Verification:** Unit tests load every bundled platform.toml, exercise the envelope-emit path with a few values, assert the output matches expected shapes. + +### Step 6 — Implement `mdvs hook handle --platform --kind ` (3–4 hours) + +The runtime that hooks call. Replaces `validate.sh` + `search-nudge.sh` from Step 2/3. + +- New module `crates/mdvs/src/cmd/hook/mod.rs` with the `Hook` subcommand enum and `Handle` variant. +- `crates/mdvs/src/cmd/hook/handle.rs` — reads stdin JSON via `serde_json::from_reader(stdin())`, walks up from `cwd` (or current cwd if not in stdin) to find `mdvs.toml`, runs `cmd::check` directly (no subprocess), wraps the output via `Platform::emit_envelope`, prints to stdout, exits 0. +- The validate path also runs check twice (markdown for agent, pretty for user) and caps the user message at `MAX_USER_LINES=15` with `...` truncation — matches the shell-script behaviour. +- The search-nudge path checks the command-line pattern for grep/rg/find/etc. and only emits if cwd is in a vault. + +**Verification:** Unit tests for both kinds (validate, search-nudge) across all three hook-supporting platforms (claude-code, codex, cursor). Hand-install on `example_kb` for Claude Code by swapping the `.claude/settings.json` `command:` to `mdvs hook handle --platform claude-code --kind validate` — verify the existing test cycle (bogus status edit → violation in additionalContext + systemMessage) still works. + +### Step 7 — Implement `mdvs scaffold {skill,snippet,hook}` (2–3 hours) + +The install-time generator commands. All three read `Platform` config to know what to emit. + +- `crates/mdvs/src/cmd/scaffold/mod.rs` with `Scaffold` subcommand enum (Skill, Snippet, Hook). +- `scaffold skill` — prints `scaffolding/skill/SKILL.md`. `--platform ` only changes the install-path comment in help text. +- `scaffold snippet` — prints `scaffolding/snippet/.md` where `` comes from `platform.toml`. For Cursor, also offers the `.mdc` wrap. +- `scaffold hook` — emits the platform-specific config (e.g. `.claude/settings.json` block) with the `command:` filled in as `mdvs hook handle --platform --kind `. No more `.sh` files emitted. + +Remove `crates/mdvs/src/cmd/skill.rs` (hard rename, decision #6). Wire `scaffold` and `hook` into `main.rs`. + +**Verification:** `mdvs scaffold {skill,snippet,hook} --platform ` for each platform; output of `scaffold hook` parses as valid JSON. + +### Step 8 — Cutover `example_kb` to the no-shell install (≈ 30 min) + +Delete the `.sh` symlinks under `example_kb/.{claude,codex,cursor}/hooks/`. Regenerate each platform's `settings.json` / `hooks.json` to call `mdvs hook handle` directly. Verify by re-running the bogus-status edit cycle (should produce identical agent+user output). + +**Verification:** Hooks fire the same way as before. No `jq` invocations show up in any installed config. Files removed: `.sh` symlinks; files modified: `settings.json` / `hooks.json` per platform. + +### Step 9 — Delete the obsolete shell scripts from the scaffolding tree (≈ 15 min) + +Remove `crates/mdvs/scaffolding/hooks//{validate,search-nudge}.sh` and `{settings,hooks}.json` (the templates — the new ones come from `platform.toml` + Rust code, not from bundled files). + +`cargo build` clean. No references in code to the deleted paths. + +### Step 10 — Refresh the SKILL.md to drop the jq / shell-envelope content (≈ 30 min) + +Edit `crates/mdvs/scaffolding/skill/SKILL.md`: + +- Drop the "hook output format" subsection that taught the agent about `jq` and the JSON envelope mechanics — that's all internal to mdvs now. +- Keep the schema-evolution warning loop (Step 4 of the skill) — the agent's *response* to violations is unchanged; only the delivery mechanism is. +- Trim references to `.claude/hooks/mdvs-validate.sh` etc. — the install no longer involves shell files. + +**Verification:** Skill body shrinks slightly. Word "jq" no longer appears outside historical context. Agent still gets the warning-loop procedure intact. + +### Step 11 — Update the recipe page for the no-shell install (≈ 1 hour) + +`book/src/recipes/agentic-harnesses-and-agentic-ides.md` — replace the hook section with the new shape: + +```bash +mdvs scaffold skill > .agents/skills/mdvs/SKILL.md +mdvs scaffold snippet >> AGENTS.md +mdvs scaffold hook --platform claude-code # prints settings.json snippet +``` + +The output of `scaffold hook` is a single small JSON block users paste into their harness config. No script files, no `jq`, works on every OS. Note this in the "What's tested" section: Claude Code end-to-end; Codex / Cursor schema-correct but untested by us; Windows untested but architecturally supported (Rust binary, no shell). + +**Verification:** `mdbook build book` clean. Page is shorter than the current Unix-only version. + +### Step 12 — End-to-end test on Claude Code (the gate test, ≈ 1 hour) + +Same shape as the original Step 9: bogus-status edit in `example_kb`, expect violation through `additionalContext` + `systemMessage`. Now exercising `mdvs hook handle` end-to-end. + +### Step 13 — End-to-end test on Antigravity CLI (skill + snippet only, ≈ 30 min) + +Same as the original Step 10. + +### Step 14 — PR and merge (≈ 30 min) + +- Open PR `feat/mdvs-wire` → `main`. +- CI must pass: `cargo test`, `cargo clippy --all-targets --features testing-mocks`, `mdbook build book`, `just lint-ast`. +- Self-review the diff. +- Merge. +- Subsume [TODO-0187](TODO-0187.md): mark `status: done`, `subsumed_by: 190`. + +Post-merge: cocogitto auto-bumps. The rename of `mdvs skill` + the new `mdvs hook` subcommand are minor-version material (`0.7.x` → `0.8.0`). + +**Total estimate post-pivot: 12–15 hours of focused work, spreading across 2–3 sessions.** (Roughly the same as the original plan — the Rust impl trades off against the shell-script ship that no longer needs polishing.) + +## Verification (sketch) + +Realistic test scope: we have direct access to **Claude Code and Antigravity CLI** for end-to-end verification on macOS. Codex, OpenCode, and Cursor will ship best-effort against their documented schemas, but without an actual end-to-end smoke test on each. Windows is architecturally supported (Rust binary, no shell dependency) but we haven't smoke-tested any harness on Windows yet. The recipe page should call this out honestly. `--help` output stays concise and doesn't mention test coverage. + +- All `mdvs scaffold {skill,snippet,hook} --platform ` invocations produce output that matches the documented schema for each platform (verifiable without running the harness — sufficient for the platforms we can't test live). +- `mdvs hook handle --platform --kind ` unit tests pass for every platform + kind combination on the CI matrix (Linux, macOS, Windows). +- **End-to-end test on Claude Code (macOS)** — full loop with all three artifacts wired up. Agent edits a markdown file in a vault with `mdvs.toml`, the validate-on-write hook fires, the agent receives the markdown explanation via `additionalContext` plus the truncated pretty render via `systemMessage`, and self-corrects on the next turn. Schema-evolution path: an intentional deviation results in the agent proposing an `mdvs.toml` update rather than silently fixing the file. +- **End-to-end test on Antigravity CLI (macOS)** — skill + snippet (hook out of scope per the upstream-docs gap). Confirms the cross-harness skill works in `.agents/skills/` and the AGENTS.md snippet is picked up. +- **No end-to-end test on Codex / OpenCode / Cursor (any OS), or any harness on Windows** — configs are documented-schema-correct but not smoke-tested by us. If a user reports a wiring bug for those, we treat it as a bug report rather than promising verified support. +- Recipe page renders cleanly and no longer contains hand-written hook JSON or shell-script content. +- `mdvs skill` is removed; help text points users to `mdvs scaffold skill`. +- No `.sh` files remain under `crates/mdvs/scaffolding/`; no `jq` references in any installed config that mdvs generates. + +## Out of scope + +- Auto-detecting the harness from environment (v2). +- `mdvs scaffold all` super-command (v2). +- OpenCode TypeScript plugin emission (refuse with pointer in v1). +- Antigravity hook emission (refuse with pointer in v1 — docs incomplete). +- Pre-explain (v2) and pre-validate (v3) hook modes — tracked in [TODO-0186](TODO-0186.md) and [TODO-0188](TODO-0188.md) respectively; will ship as follow-ups when those land. +- Real-time validation feedback inside the editor (LSP-style — separate scope). diff --git a/docs/spec/todos/TODO-0191.md b/docs/spec/todos/TODO-0191.md new file mode 100644 index 0000000..8cb08d2 --- /dev/null +++ b/docs/spec/todos/TODO-0191.md @@ -0,0 +1,126 @@ +--- +id: 191 +title: Auto-rewrite array-field comparisons in `--where` (parser-based, with translation note) +status: done +priority: medium +created: 2026-06-23 +completed: 2026-06-24 +depends_on: [] +blocks: [] +related: [] +files_created: + - crates/mdvs/src/index/backend/where_translator.rs +files_updated: + - crates/mdvs/Cargo.toml + - crates/mdvs/src/index/backend/mod.rs + - crates/mdvs/src/index/backend/search.rs + - crates/mdvs/src/cmd/search.rs + - crates/mdvs/src/outcome/commands/search.rs + - crates/mdvs/src/main.rs + - crates/mdvs/scaffolding/skill/SKILL.md + - README.md +--- + +# TODO-0191: Auto-rewrite array-field comparisons in `--where` (parser-based, with translation note) + +## Summary + +When a user (or an agent) writes `--where "tags = 'rust'"` against an `Array(String)` field, mdvs's `--where` translator passes the clause through as-is (with a `data.` prefix), and Lance/DataFusion rejects it because a `List(Utf8)` column can't be compared to a `Utf8` scalar with `=`. The user has to know to write `array_has(tags, 'rust')` instead — which is documented but easy to miss, and reads as a footgun. + +The right behavior: when an array field is compared with `=`, `!=`, `IN`, or `NOT IN` against a scalar literal of the element type, automatically rewrite to the equivalent `array_has(...)` form. The schema is already known at `--where`-translation time (mdvs reads `mdvs.toml` to discover `data_children` and `float_list_fields`), so the type information for "is this field an array?" is available. + +To keep the rewrite legible and avoid the "magic" feeling, the search outcome surfaces a translation note at the top showing each original → rewritten pair. + +## Context + +Discovered 2026-06-23 when a Cursor agent invoked `mdvs search "..." --where "tags = 'rust'"` against the Refractions personal vault (matching the example in the bundled SKILL.md and `mdvs --help`'s `long_help`). The query failed at Lance: + +``` +Error: lance error: Invalid user input: Error resolving filter expression +data.tags = 'rust': Invalid user input: Received literal Utf8("rust") and +could not convert to literal of type 'List(Field { data_type: Utf8, nullable: true })' +``` + +A documentation patch landed in PR #66 (commit `a2f310d`) switching the examples to `array_has(tags, 'rust')`. That papered over the immediate symptom but doesn't fix the underlying ergonomics: users who guess the obvious form still hit the Lance error. A second hit confirmed the persistence: `--where "tags = 'smart-digital-twin'"` failed in exactly the same shape on 2026-06-24. + +## Design — parser-based translator + +The current `translate_where_to_struct` (in `crates/mdvs/src/index/backend/search.rs`) is a regex pass: two regexes (`lit` for string literals, `ident` for identifiers), one walk-and-rewrite loop, no operator awareness. Growing it for `=` / `!=` / `IN` / `NOT IN` / "don't double-rewrite inside function calls" would mean adding more regexes, more peek-ahead heuristics, and more fragility every time a new rule lands. + +Switching to a real SQL parser is the right altitude for this work: + +- **`sqlparser-rs` AST.** Returns `Expr::BinaryOp`, `Expr::InList`, `Expr::Identifier`, `Expr::Function`, etc. Pattern-matching on the AST is exact; "don't rewrite inside a function call" becomes free (structurally we know when we're inside `Expr::Function`). +- **Already in the dependency graph.** Pulled in transitively via datafusion → lance, so no new direct dependency tree weight — and using the same parser as Lance itself guarantees the rewrite output stays parseable downstream. +- **Future-proof.** Each future `--where` enhancement (LIKE on array elements? CONTAINS-ALL? regex?) becomes one match arm in the AST walk rather than another regex. + +Migration shape: + +1. **Add `sqlparser` as a direct dependency** (alongside `lance` / `lancedb`). Pin to whatever version Lance is using to avoid version skew. +2. **Replace `translate_where_to_struct`** with a `parse → walk → emit` pipeline: + - Parse the user's clause as a DataFusion-dialect SQL expression. + - Walk the `Expr` tree. At each node, apply: + - **Identifier rewrite** (unchanged from today): if the identifier names a `data` child, qualify it with `data.`; if it names an internal column, resolve via aliases/prefix; if it names an `Array(Float)` field, error out. + - **Array-equality rewrite** (NEW — see "Rewrite rules" below). + - Re-emit the transformed AST as SQL via `Expr::to_string()`. +3. **Collect rewrites** during the walk as `Vec` and return them alongside the rewritten string. The caller (`search.rs`) threads them into the search outcome. + +### Rewrite rules (4 forms) + +For each of the rules below, both literal orderings are supported: ` OP ` and ` OP `. + +| User wrote | Rewritten to | +|---|---| +| `tags = 'x'` | `array_has(data.tags, 'x')` | +| `tags != 'x'` | `NOT array_has(data.tags, 'x')` | +| `tags IN ('x', 'y')` | `array_has(data.tags, 'x') OR array_has(data.tags, 'y')` | +| `tags NOT IN ('x', 'y')` | `NOT (array_has(data.tags, 'x') OR array_has(data.tags, 'y'))` | + +**Element semantics confirmed** for `!=` and `NOT IN`: both mean "the array does not contain this element". The alternative interpretation ("the array does not equal the singleton") is what 1% of users mean — and Lance can't express it anyway without `array_length` + `array_has` clauses nobody writes. Element semantics is what the user expects, gets documented as the contract. + +**Function-call invariants**: if the array field already appears inside a function (`array_has(tags, 'x')`, `array_length(tags) > 2`, `cast(tags as text)`), leave it alone. The AST makes this trivial — we only fire the rewrite on `Expr::BinaryOp` / `Expr::InList` where the operand is a bare `Expr::Identifier`, never on identifiers nested inside `Expr::Function`. + +**Element types**: rewrite applies to `Array(String)`, `Array(Integer)`, `Array(Boolean)`, `Array(Date)`, `Array(DateTime)`. `Array(Float)` stays rejected up-front (TODO-0159, distinct bug). + +### Translation note + +Without surfacing the rewrite, users hit results that look like they came from a clause they didn't write. The note explains it concisely at the top of the search output. + +Add `where_rewrites: Vec` to `SearchOutcome`. When non-empty, render at the top as a `Block::Section` (markdown: `## Note — rewrote N array-field expressions`; pretty: italic block above the result table). Each entry shows `original → rewritten`. JSON output carries it as structured data. + +Example pretty rendering: + +``` +Note — rewrote 1 array-field expression for List-column matching: + tags = 'smart-digital-twin' → array_has(tags, 'smart-digital-twin') + +Searched "smart digital twin actor model" — 3 hits +... +``` + +The translator returns the rewrites in original-clause column order; the renderer doesn't need to know anything about array semantics, just how to format the pairs. + +## Out of scope + +- Auto-rewriting for `Array(Float)` (still rejected up front; the Lance-encoding bug from TODO-0159 isn't fixed by changing the translator). +- Auto-rewriting `LIKE` / `GLOB` / regex operators against array fields (would need different semantics — "any element matches the pattern" — and isn't requested yet). +- Schema-aware rewrites for dotted-name leaves (already work because dotted names address scalar leaves, not arrays). +- Opt-in / opt-out flag: the current behavior (fail with a Lance error) is strictly worse for anyone who doesn't already know the `array_has` incantation. The rewrite is always-on. + +## Verification + +- Existing translator tests in `crates/mdvs/src/index/backend/search.rs::tests` all still pass (regression: the parser-based translator emits the same SQL for non-array clauses). +- New tests covering each rule (= / != / IN / NOT IN) for both operand orderings, both with and without `data.` prefix: + - `array_equality_rewrites_to_array_has` + - `array_equality_literal_on_left` + - `array_inequality_rewrites_to_not_array_has` + - `array_in_list_rewrites_to_or_chain` + - `array_not_in_list_rewrites_to_not_or_chain` + - `array_inside_function_call_not_double_rewritten` — `array_has(tags, 'x')` stays unchanged + - `array_inside_array_length_not_rewritten` — `array_length(tags) > 2` stays unchanged +- New `SearchOutcome::where_rewrites` populated in tests; render assertions for pretty + markdown + JSON. +- Doc updates (SKILL.md, README, `--help`) switch examples back to `tags = 'x'` as the natural form. +- End-to-end on `example_kb`: `mdvs search "" --where "tags = 'foo'"` returns results AND prints the translation note. + +## Impact + +Removes a footgun. The previous "fix" was documentation-only — users who follow the docs work, but users who guess the obvious syntax hit a Lance-level error message that doesn't suggest the right form. Auto-rewrite means the obvious syntax just works; the translation note keeps the rewrite legible instead of magic. diff --git a/docs/spec/todos/TODO-0192.md b/docs/spec/todos/TODO-0192.md new file mode 100644 index 0000000..30915e4 --- /dev/null +++ b/docs/spec/todos/TODO-0192.md @@ -0,0 +1,57 @@ +--- +id: 192 +title: Don't persist the mock-embedder default to `mdvs.toml` +status: done +priority: high +created: 2026-06-23 +completed: 2026-06-23 +depends_on: [] +blocks: [] +related: [184] +files_updated: + - crates/mdvs/src/cmd/build/config_mutate.rs +--- + +# TODO-0192: Don't persist the mock-embedder default to `mdvs.toml` + +## Summary + +When a `cfg(test)`- or `--features testing-mocks`-enabled `mdvs` binary runs `build` (including the auto-build chain reached from `search`) against a vault whose `mdvs.toml` has no `[embedding_model]` block, `mutate_config` synthesizes a `provider = "mock"` default **and writes it to disk**. A subsequent run with the production binary (compiled without `testing-mocks`) reads that file and refuses with `embedding provider 'mock' is only available in builds compiled with --features testing-mocks`. The test-mode binary leaked its config into a real user vault. + +## Context + +Discovered 2026-06-23 against a personal vault. The user's `/usr/local/bin/mdvs` was a symlink to a `cargo build` debug binary (`cfg(test)` enabled the mock default). At some point during the session it ran `mdvs search` → auto-build → `mutate_config`, which saw `embedding_model = None` and wrote: + +```toml +[embedding_model] +provider = "mock" +name = "mock" +dim = 256 +``` + +After that, the production binary on PATH (installed via `cargo install`) reads `mdvs.toml`, sees `provider = "mock"`, and bails at `EmbedderConfig::from_config` with the "only available in builds compiled with `--features testing-mocks`" error from `crates/mdvs/src/index/embed.rs:40`. + +The intent of the cfg-gated mock default (TODO-0184) is correct: tests that flow through `init → build` shouldn't reach for HuggingFace. But the persistence is wrong — a test-mode config should never escape to the user's committed `mdvs.toml`. + +## Resolution + +In `crates/mdvs/src/cmd/build/config_mutate.rs::mutate_config`, split the `None` arm of the `match config.embedding_model` into two `cfg`-gated branches and remove `config_changed = true` from the mock branch: + +- Production (`cfg(not(any(test, feature = "testing-mocks")))`): synthesize the `model2vec` default AND set `config_changed = true`, so first-time `build` persists the real `[embedding_model]` block. +- Test/mock (`cfg(any(test, feature = "testing-mocks"))`): synthesize the mock default in-memory only. `config_changed` stays false on this branch, so `mdvs.toml` is never rewritten. The mock default exists only for the duration of the run. + +The `Some(...)` arms (existing `[embedding_model]` block, with or without `--set-model`/`--set-revision`) are unchanged — they still persist user-driven mutations. + +A regression test (`mock_embedder_default_is_not_persisted_to_mdvs_toml`) writes a minimal `mdvs.toml` without an `[embedding_model]` block, runs `mutate_config`, re-reads the file from disk, and asserts that neither `mock` nor `[embedding_model]` appears in the written file. The in-memory `config.embedding_model` is still verified to be the mock default, so the rest of the build chain still has a valid embedder to work with. Under `cfg(test)` this test exercises the mock branch directly. + +Existing tests (`second_build_skips_write_index_when_nothing_changed`, `third_build_persists_new_file_via_incremental_path`, etc.) continue to pass — they consume the in-memory `MdvsToml` after `build` and never depended on the mock provider being persisted to disk. + +## Out of scope + +- Changing the production default (still `model2vec` / `potion-base-8M`). +- Removing the `cfg`-gated mock default entirely. The TODO-0184 design (deterministic hermetic test embedder) stays. +- Auditing other call sites that may persist test-only defaults. None are known today; if more surface, file follow-ups. + +## Impact + +A test-mode binary running against a user vault stops corrupting that vault's `mdvs.toml`. Without this fix, a developer who happens to symlink `target/debug/mdvs` into PATH and then runs a quick `mdvs search` against any real corpus permanently breaks that corpus for the production binary. diff --git a/docs/spec/todos/index.md b/docs/spec/todos/index.md index 6487560..14c5949 100644 --- a/docs/spec/todos/index.md +++ b/docs/spec/todos/index.md @@ -188,6 +188,9 @@ | [0184](TODO-0184.md) | Introduce MockEmbedder; keep real-model tests in a separate local-only lane | done | high | 2026-06-05 | | [0185](TODO-0185.md) | Refresh crates/mdvs/skills/mdvs/SKILL.md for v0.7.0 surface | done | high | 2026-06-05 | | [0186](TODO-0186.md) | mdvs explain — path-scoped schema query for agent callers | todo | high | 2026-06-06 | -| [0187](TODO-0187.md) | Recipe — agent harness hook for auto-check (and optional auto-explain / pre-validate) | todo | medium | 2026-06-06 | +| [0187](TODO-0187.md) | Recipe — agent harness hook for auto-check (and optional auto-explain / pre-validate) | done | medium | 2026-06-06 | | [0188](TODO-0188.md) | mdvs check --stdin — in-memory frontmatter validation | todo | medium | 2026-06-06 | | [0189](TODO-0189.md) | Collapse main.rs output dispatch into CommandResult::render() | done | medium | 2026-06-16 | +| [0190](TODO-0190.md) | Design `mdvs scaffold` — unified agent-harness integration command surface | done | high | 2026-06-22 | +| [0191](TODO-0191.md) | Auto-rewrite array-field comparisons in `--where` (parser-based, with translation note) | done | medium | 2026-06-23 | +| [0192](TODO-0192.md) | Don't persist the mock-embedder default to `mdvs.toml` | done | high | 2026-06-23 | diff --git a/example_kb/.agents/skills/mdvs/SKILL.md b/example_kb/.agents/skills/mdvs/SKILL.md new file mode 120000 index 0000000..f9abcfb --- /dev/null +++ b/example_kb/.agents/skills/mdvs/SKILL.md @@ -0,0 +1 @@ +../../../../crates/mdvs/scaffolding/skill/SKILL.md \ No newline at end of file diff --git a/example_kb/.claude/settings.json b/example_kb/.claude/settings.json new file mode 100644 index 0000000..2635a1b --- /dev/null +++ b/example_kb/.claude/settings.json @@ -0,0 +1,25 @@ +{ + "_comment": "mdvs PostToolUse hooks generated by `mdvs scaffold hook --platform claude-code`. Merge into .claude/settings.json under the existing `hooks` key. Both hooks self-scope by walking up from the agent's cwd to find mdvs.toml; they stay silent outside an mdvs vault.", + "hooks": { + "PostToolUse": [ + { + "hooks": [ + { + "command": "mdvs hook handle --platform claude-code --kind validate", + "type": "command" + } + ], + "matcher": "Edit|Write|MultiEdit" + }, + { + "hooks": [ + { + "command": "mdvs hook handle --platform claude-code --kind search-nudge", + "type": "command" + } + ], + "matcher": "Bash" + } + ] + } +} diff --git a/example_kb/.claude/skills/mdvs/SKILL.md b/example_kb/.claude/skills/mdvs/SKILL.md new file mode 120000 index 0000000..f9abcfb --- /dev/null +++ b/example_kb/.claude/skills/mdvs/SKILL.md @@ -0,0 +1 @@ +../../../../crates/mdvs/scaffolding/skill/SKILL.md \ No newline at end of file diff --git a/example_kb/.codex/hooks.json b/example_kb/.codex/hooks.json new file mode 100644 index 0000000..5fc2124 --- /dev/null +++ b/example_kb/.codex/hooks.json @@ -0,0 +1,25 @@ +{ + "_comment": "mdvs PostToolUse hooks generated by `mdvs scaffold hook --platform codex`. Merge into .codex/hooks.json under the existing `hooks` key. Both hooks self-scope by walking up from the agent's cwd to find mdvs.toml; they stay silent outside an mdvs vault.", + "hooks": { + "PostToolUse": [ + { + "hooks": [ + { + "command": "mdvs hook handle --platform codex --kind validate", + "type": "command" + } + ], + "matcher": "Edit|Write|MultiEdit" + }, + { + "hooks": [ + { + "command": "mdvs hook handle --platform codex --kind search-nudge", + "type": "command" + } + ], + "matcher": "Bash" + } + ] + } +} diff --git a/example_kb/.cursor/hooks.json b/example_kb/.cursor/hooks.json new file mode 100644 index 0000000..79fce25 --- /dev/null +++ b/example_kb/.cursor/hooks.json @@ -0,0 +1,16 @@ +{ + "_comment": "mdvs PostToolUse hooks generated by `mdvs scaffold hook --platform cursor`. Merge into .cursor/hooks.json under the existing `hooks` key. Both hooks self-scope by walking up from the agent's cwd to find mdvs.toml; they stay silent outside an mdvs vault.", + "hooks": { + "postToolUse": [ + { + "command": "mdvs hook handle --platform cursor --kind validate", + "matcher": "Edit|Write|MultiEdit" + }, + { + "command": "mdvs hook handle --platform cursor --kind search-nudge", + "matcher": "Bash" + } + ] + }, + "version": 1 +} diff --git a/example_kb/.cursor/rules/mdvs.mdc b/example_kb/.cursor/rules/mdvs.mdc new file mode 120000 index 0000000..0bc130b --- /dev/null +++ b/example_kb/.cursor/rules/mdvs.mdc @@ -0,0 +1 @@ +../../../crates/mdvs/scaffolding/snippet/cursor-rules.mdc \ No newline at end of file diff --git a/example_kb/.cursor/skills/mdvs/SKILL.md b/example_kb/.cursor/skills/mdvs/SKILL.md new file mode 120000 index 0000000..f9abcfb --- /dev/null +++ b/example_kb/.cursor/skills/mdvs/SKILL.md @@ -0,0 +1 @@ +../../../../crates/mdvs/scaffolding/skill/SKILL.md \ No newline at end of file diff --git a/example_kb/AGENTS.md b/example_kb/AGENTS.md new file mode 100644 index 0000000..6bde67e --- /dev/null +++ b/example_kb/AGENTS.md @@ -0,0 +1,48 @@ +# Prismatiq KB — Agent guidance + +This directory is the Prismatiq research group's internal journal: project notes, experiment logs, meeting minutes, people profiles, blog posts, and protocols. Every markdown file carries structured frontmatter, validated against the schema in `mdvs.toml`. + +## Folder layout + +- `projects//` — research projects. Currently `alpha` (programmable biosensor array), `beta` (signal-processing pipeline), `archived/gamma` (discontinued). Each has an `overview.md`, plus `notes/` for experiment logs and `meetings/` for project-specific minutes. +- `people/` — team member profiles (`-.md`); `people/interns/` for short-term contributors. +- `meetings/` — cross-project meeting minutes, organised by year and quarter. +- `blog/` — `drafts/` for in-progress posts; `published///` for posts that went out. +- `reference/` — `glossary.md`, `quick-start.md`, `tools.md`, and `protocols/` for SOPs. Per-instrument protocols live under `protocols/equipment/`. +- `lab-values.md`, `scratch.md` — top-level singletons (the group's stated principles and a working scratchpad). + +## Frontmatter conventions + +The schema in `mdvs.toml` is the source of truth. A few of the load-bearing fields: + +- **`status`** — categorical. Allowed values: `active`, `archived`, `completed`, `draft`, `published`. Don't invent new values without proposing a schema update first (see the validation contract below). +- **`author`** — categorical, scoped to the current team: `Chiara Russo`, `Giulia Ferretti`, `Marco Bianchi`, `REMO`, `Sara Dell'Acqua`. New contributors get added to the schema before they can be assigned. +- **`priority`** — categorical: `high`, `medium`, `low`. +- **`tags`** — free-form array of strings. Common themes you'll see across the corpus: `biosensor`, `metamaterial`, `calibration`, `signal-processing`. +- **`date` / `joined` / `commission_date` / `last_reviewed`** — `Date` type (`YYYY-MM-DD`). +- **`synced_at`** — `DateTime` (RFC 3339 with mandatory timezone). +- **Path-scoped fields** — some fields are only allowed in specific subtrees. Equipment protocols carry `equipment_id`, `firmware_version`, `wavelength_nm`; people profiles carry `joined` and `email`. Check `mdvs.toml`'s `allowed` globs before adding a field where it doesn't normally live. + +## Working in this KB + +- **Find content with `mdvs search`**, not Grep — semantic / hybrid / SQL-filtered search across the whole vault. Filter on frontmatter (`--where "status = 'active'"`), pick a mode (`--mode semantic | fulltext | hybrid`). +- **Run `mdvs check` after edits** that touch frontmatter. The vault is 45 files; check completes in ~10 ms. +- **Inspect the schema with `mdvs info -v`** before making structural changes — the `mdvs.toml` declares which fields are required, where they're allowed, and their type / category constraints. +- **Look at folder neighbours** when unsure about local conventions — the schema is path-scoped, so different subtrees can carry different field sets. + + + +## mdvs knowledge base + +This project uses [mdvs](https://github.com/edochi/mdvs) to manage a markdown knowledge base with a schema defined in `mdvs.toml`. When working with files in this KB: + +- **Prefer `mdvs search ""` over `Grep` / `Glob` for semantic lookups.** mdvs runs semantic / hybrid / SQL-filtered search over the KB — usually a better fit than literal pattern matching for finding notes by meaning. Filter by frontmatter with `--where "field = 'value'"` and pick the mode with `--mode semantic|fulltext|hybrid`. +- **`mdvs check` validates frontmatter against the schema.** A `PostToolUse` hook may surface violations to you automatically as a markdown block under `additionalContext` in the hook output. Treat that block as **a warning, not a block** — the agent's edit already landed; the warning is for you to act on next. +- **When a violation reaches you, decide:** + 1. **Mistake** (a typo, the wrong type by accident, a dropped required field) → fix the file in the next turn. + 2. **Intentional deviation** (the KB is evolving, a category genuinely needs a new variant, a field is shifting type) → surface the deviation to the user and propose updating `mdvs.toml` to absorb the change. **Do not silently fix the file**; the user decides whether the schema or the file is wrong. + +The schema is meant to evolve with the KB. Enforcement follows the KB's shape, it does not freeze it. Whenever you propose a schema update, also explain *why* — which file(s) drove the change and what convention the user might want to standardise on. + +Full mdvs documentation: . diff --git a/example_kb/CLAUDE.md b/example_kb/CLAUDE.md new file mode 120000 index 0000000..47dc3e3 --- /dev/null +++ b/example_kb/CLAUDE.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/example_kb/mdvs.toml b/example_kb/mdvs.toml index 99dd2e1..1d0f4fe 100644 --- a/example_kb/mdvs.toml +++ b/example_kb/mdvs.toml @@ -13,7 +13,7 @@ auto_update = true [embedding_model] provider = "model2vec" -name = "minishlab/potion-base-8M" +name = "minishlab/potion-multilingual-128M" [chunking] max_chunk_size = 1024