diff --git a/CHANGELOG.md b/CHANGELOG.md index 919437067..6c649bb26 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,39 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - CI builds/tests run with `debug = 0` (no debuginfo) via `CARGO_PROFILE_*_DEBUG` env — smaller artifacts (faster cache save/restore) and faster linking; local builds unaffected. +### Fixed — FTS term-dict + FST sidecar durability, ends restart-rescan-only recovery (kernel M4, task #50) + +The full-text-search inverted index was the last plane not kill-9-lossless: +every restart rebuilt every text index by rescanning the keyspace and +reassigning term ids by `DashTable` hash-iteration first-encounter order — +not reproducible across restarts, which is why the pre-existing `.fst` +sidecar write path had its load path (`load_fst_sidecars`) deliberately left +uncalled in production (wiring it once corrupted FUZZY/PREFIX results: the +sidecar's baked-in term ids silently collided with a freshly-rescanned +dictionary's differently-assigned ids). + +Fixed by persisting the term dictionary itself alongside the FST, in one +atomic sidecar (`{shard_dir}/{index}.tfst`, magic `TFS2`, version-stamped, +`atomic_write_durable`): per TEXT field, `next_id`, `fst_high_water_mark`, +every `(term, id)` pair, and the optional FST bytes. `TermDictionary::from_pairs` +reconstructs a dictionary whose ids are taken verbatim from the sidecar +(never reassigned) and whose `next_id` continues the persisted high-water +mark. Wired into shard boot (`src/shard/event_loop.rs`) so +`TextStore::load_term_fst_sidecars` runs AFTER text index schemas are +restored but BEFORE the keyspace auto-reindex rescan — seeding the term +dicts first makes the rescan's `get_or_insert` calls resolve known terms to +their persisted ids and assign fresh, non-colliding ids only to genuinely +new terms, which is what makes loading the FST alongside it safe (FST ids +and live dict ids are the same id-space by construction). Fails closed per +index on any missing/truncated/corrupt/version-mismatched/field-count- +mismatched sidecar — falls back to today's full rescan, never partially +applies a sidecar. `FT.COMPACT` now calls the combined saver +(`save_term_fst_sidecar_for_index`) instead of the old FST-only one. New +`FT.INFO` counters `sidecar_recovered_indexes` / `text_indexes_total` +(additive across shards) surface fast-boot coverage. New default-GREEN +crash-matrix cells `cross_plane_prod_{s1,s4}_text_fts_sidecar_isolated` +verify a FUZZY query survives kill-9 identically to a from-scratch rebuild. +New fuzz target `term_fst_sidecar` covers the sidecar decoder. ### Security — clear dependency vulnerability backlog (task #51) `sdk/python` (uv.lock, 36 open Dependabot alerts incl. 1 CRITICAL) and diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index 649a595f5..2b6520a63 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -103,3 +103,8 @@ doc = false name = "redis_rdb_load" path = "fuzz_targets/redis_rdb_load.rs" doc = false + +[[bin]] +name = "term_fst_sidecar" +path = "fuzz_targets/term_fst_sidecar.rs" +doc = false diff --git a/fuzz/fuzz_targets/term_fst_sidecar.rs b/fuzz/fuzz_targets/term_fst_sidecar.rs new file mode 100644 index 000000000..c85295950 --- /dev/null +++ b/fuzz/fuzz_targets/term_fst_sidecar.rs @@ -0,0 +1,16 @@ +#![no_main] +use libfuzzer_sys::fuzz_target; + +use moon::text::index_persist::deserialize_term_fst_sidecar; + +/// Fuzz the combined term-dict + FST sidecar decoder (kernel M4, task #50). +/// +/// Exercises magic/version validation, the per-field term-count/term-len +/// framing loop, UTF-8 term validation, and FST-length truncation handling. +/// Any panic or OOB access is a bug -- malformed on-disk sidecar bytes +/// (truncated write, bit rot, downgrade/upgrade version skew) must always +/// fail closed with `Err`, never panic and never return a partially +/// populated `Vec`. +fuzz_target!(|data: &[u8]| { + let _ = deserialize_term_fst_sidecar(data); +}); diff --git a/src/command/vector_search/ft_admin.rs b/src/command/vector_search/ft_admin.rs index 119522f99..0d3895bd4 100644 --- a/src/command/vector_search/ft_admin.rs +++ b/src/command/vector_search/ft_admin.rs @@ -152,8 +152,10 @@ pub fn ft_compact( #[cfg(feature = "text-index")] if let Some(text_idx) = text_store.get_index_mut_for_db(name.as_ref(), db_index) { text_idx.build_fst(); - // Persist FST sidecar to disk so it survives server restart (FUZ-02). - text_store.save_fst_sidecar_for_index(name.as_ref()); + // Persist term-dict + FST sidecar to disk so it survives server + // restart (FUZ-02; kernel M4 task #50 -- combined saver so the + // loaded FST's ids are always backed by a matching term dict). + text_store.save_term_fst_sidecar_for_index(name.as_ref()); } // Return OK if either a vector index or text index exists with this name (db-scoped). diff --git a/src/command/vector_search/ft_info.rs b/src/command/vector_search/ft_info.rs index 2217cb731..22028fb34 100644 --- a/src/command/vector_search/ft_info.rs +++ b/src/command/vector_search/ft_info.rs @@ -253,6 +253,17 @@ pub fn ft_info( // Independent from vector_version_token — hybrid-index callers check both. items.push(Frame::BulkString(Bytes::from_static(b"text_version_token"))); items.push(Frame::Integer(text_store.version_token() as i64)); + // Kernel M4 (task #50): coverage counter mirroring + // `segments_with_exact_rerank` for the vector engine -- additive + // across shards so `sidecar_recovered_indexes < text_indexes_total` + // means at least one shard fell back to a full keyspace rescan for + // at least one index (missing/stale/corrupt `.tfst` sidecar). + items.push(Frame::BulkString(Bytes::from_static( + b"sidecar_recovered_indexes", + ))); + items.push(Frame::Integer(text_idx.recovered_from_sidecar as i64)); + items.push(Frame::BulkString(Bytes::from_static(b"text_indexes_total"))); + items.push(Frame::Integer(1)); } Frame::Array(items.into()) @@ -272,6 +283,8 @@ pub fn ft_info( /// aggregate). /// Additive per-field keys (matched by `field_name` inside `vector_fields` / /// `text_fields`): `num_docs`, `mutable_vectors`, `immutable_segments`. +/// Also additive: `sidecar_recovered_indexes` / `text_indexes_total` +/// (kernel M4, task #50 -- FT.INFO term-dict sidecar recovery coverage). /// /// Any `Frame::Error` (local or remote) is propagated unchanged (fail-loud, /// same semantics as `scatter_invalidate_range`). @@ -288,6 +301,8 @@ pub fn merge_ft_info_responses(local: Frame, remotes: &[Frame]) -> Frame { b"warm_segments_with_exact_rerank", b"unloaded_segments", b"unloaded_segments_with_exact_rerank", + b"sidecar_recovered_indexes", + b"text_indexes_total", ]; const ADDITIVE_FIELD: &[&[u8]] = &[b"num_docs", b"mutable_vectors", b"immutable_segments"]; @@ -483,6 +498,16 @@ fn ft_info_text_only( items.push(Frame::BulkString(Bytes::from_static(b"text_version_token"))); items.push(Frame::Integer(text_store.version_token() as i64)); + // Kernel M4 (task #50): coverage counters mirroring the hybrid-index + // path above (`sidecar_recovered_indexes` / `text_indexes_total`) -- + // additive across shards via `merge_ft_info_responses`. + items.push(Frame::BulkString(Bytes::from_static( + b"sidecar_recovered_indexes", + ))); + items.push(Frame::Integer(idx.recovered_from_sidecar as i64)); + items.push(Frame::BulkString(Bytes::from_static(b"text_indexes_total"))); + items.push(Frame::Integer(1)); + Frame::Array(items.into()) } diff --git a/src/shard/event_loop.rs b/src/shard/event_loop.rs index c700bbcf7..4d70c8395 100644 --- a/src/shard/event_loop.rs +++ b/src/shard/event_loop.rs @@ -1022,6 +1022,18 @@ impl super::Shard { } } }); + + // Kernel M4 (task #50): seed each restored text index's term + // dictionaries (and, where the sidecar validates cleanly, + // FST maps) from the `.tfst` combined sidecar BEFORE the + // keyspace rescan below runs any `index_document` calls. + // This MUST happen in this order -- see + // `TextStore::load_term_fst_sidecars`'s doc comment for why + // seeding after the rescan (or not at all) is exactly the + // stale-id-space corruption this closes. + crate::shard::slice::with_shard(|s| { + s.text_store.load_term_fst_sidecars(); + }); } // Auto-reindex existing HASH keys that match vector or text index prefixes. diff --git a/src/text/index_persist.rs b/src/text/index_persist.rs index 60211be99..b9c38866d 100644 --- a/src/text/index_persist.rs +++ b/src/text/index_persist.rs @@ -49,6 +49,34 @@ const DEFAULT_DB_INDEX_ON_LOAD: u8 = 0; const FST_MAGIC: &[u8; 4] = b"TFST"; const FST_VERSION: u8 = 1; +/// Combined term-dict + FST sidecar (kernel M4, task #50). +/// +/// Extends the FST-only sidecar with the term dictionary the FST's ids were +/// built against, so a loader can reconstruct BOTH pieces from the same +/// generation and never mix a stale-id-space FST with a freshly-rescanned +/// (differently-ordered) term dictionary -- see `TextStore::load_fst_sidecars`'s +/// doc comment in `src/text/store.rs` for the full corruption mechanism this +/// closes. +/// +/// ```text +/// [magic: 4B "TFS2"] [version: 1B] [field_count: 2B] +/// Per field: +/// [next_id: 4B] [fst_high_water_mark: 4B] [term_count: 4B] +/// Per term: [term_len: 2B] [term: bytes] [term_id: 4B] +/// [fst_len: 4B] [fst_bytes: fst_len] (fst_len=0 -> no FST for this field) +/// ``` +const TERM_FST_MAGIC: &[u8; 4] = b"TFS2"; +const TERM_FST_VERSION: u8 = 1; + +/// One field's persisted term-dict + optional FST bytes. +#[derive(Debug, Clone, PartialEq)] +pub struct FieldTermFstSidecar { + pub next_id: u32, + pub fst_high_water_mark: u32, + pub terms: Vec<(String, u32)>, + pub fst_bytes: Option>, +} + /// Lightweight schema-only representation of a TextIndex for persistence. /// /// Contains everything needed to reconstruct an empty TextIndex (without @@ -382,6 +410,131 @@ pub fn load_fst_sidecar(shard_dir: &Path, index_name: &[u8]) -> io::Result Vec { + let mut buf = Vec::with_capacity(256); + buf.extend_from_slice(TERM_FST_MAGIC); + buf.push(TERM_FST_VERSION); + buf.extend_from_slice(&(fields.len() as u16).to_le_bytes()); + + for field in fields { + buf.extend_from_slice(&field.next_id.to_le_bytes()); + buf.extend_from_slice(&field.fst_high_water_mark.to_le_bytes()); + buf.extend_from_slice(&(field.terms.len() as u32).to_le_bytes()); + for (term, id) in &field.terms { + buf.extend_from_slice(&(term.len() as u16).to_le_bytes()); + buf.extend_from_slice(term.as_bytes()); + buf.extend_from_slice(&id.to_le_bytes()); + } + match &field.fst_bytes { + Some(bytes) => { + buf.extend_from_slice(&(bytes.len() as u32).to_le_bytes()); + buf.extend_from_slice(bytes); + } + None => buf.extend_from_slice(&0u32.to_le_bytes()), + } + } + buf +} + +/// Deserialize the combined term-dict + FST sidecar (pure function, fuzzable). +/// +/// Fails closed on ANY structural problem (bad magic/version, truncation, +/// non-UTF8 term bytes) by returning `Err` -- callers must treat an `Err` +/// exactly like a missing sidecar (full rescan), never partially apply the +/// result. +pub fn deserialize_term_fst_sidecar(data: &[u8]) -> io::Result> { + if data.len() < 7 { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "term-fst sidecar too short", + )); + } + if &data[0..4] != TERM_FST_MAGIC { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "bad term-fst magic", + )); + } + let version = data[4]; + if version != TERM_FST_VERSION { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("unsupported term-fst version {version}"), + )); + } + let field_count = u16::from_le_bytes([data[5], data[6]]) as usize; + let mut cursor = 7; + let mut fields = Vec::with_capacity(field_count); + + for _ in 0..field_count { + let next_id = read_u32(data, &mut cursor)?; + let fst_high_water_mark = read_u32(data, &mut cursor)?; + let term_count = read_u32(data, &mut cursor)? as usize; + let mut terms = Vec::with_capacity(term_count); + for _ in 0..term_count { + let term_len = read_u16(data, &mut cursor)? as usize; + let term_bytes = read_bytes(data, &mut cursor, term_len)?; + let term = std::str::from_utf8(term_bytes) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "non-utf8 term"))? + .to_owned(); + let id = read_u32(data, &mut cursor)?; + terms.push((term, id)); + } + let fst_len = read_u32(data, &mut cursor)? as usize; + let fst_bytes = if fst_len == 0 { + None + } else { + Some(read_bytes(data, &mut cursor, fst_len)?.to_vec()) + }; + fields.push(FieldTermFstSidecar { + next_id, + fst_high_water_mark, + terms, + fst_bytes, + }); + } + + Ok(fields) +} + +/// Persist per-field term-dict + FST bytes to `{shard_dir}/{index_name}.tfst`. +/// +/// Atomic via `atomic_write_durable` (K3: temp + fsync + rename + dir-fsync), +/// same primitive as every other text/vector sidecar writer. +pub fn save_term_fst_sidecar( + shard_dir: &Path, + index_name: &[u8], + fields: &[FieldTermFstSidecar], +) -> io::Result<()> { + let name_str = String::from_utf8_lossy(index_name); + let path = shard_dir.join(format!("{name_str}.tfst")); + let data = serialize_term_fst_sidecar(fields); + crate::persistence::atomic::atomic_write_durable(&path, &data)?; + Ok(()) +} + +/// Load per-field term-dict + FST bytes from `{shard_dir}/{index_name}.tfst`. +/// +/// Returns `Ok(None)` if the file doesn't exist (D-11-style: no sidecar is +/// not an error, caller falls back to today's full-rescan behavior). Returns +/// `Err` on any structural corruption -- callers MUST treat that identically +/// to "missing" (fail closed), never load a partially-valid result. +pub fn load_term_fst_sidecar( + shard_dir: &Path, + index_name: &[u8], +) -> io::Result>> { + let name_str = String::from_utf8_lossy(index_name); + let path = shard_dir.join(format!("{name_str}.tfst")); + if !path.exists() { + return Ok(None); + } + let mut f = std::fs::File::open(&path)?; + let mut data = Vec::new(); + f.read_to_end(&mut data)?; + deserialize_term_fst_sidecar(&data).map(Some) +} + #[cfg(test)] mod tests { use super::*; @@ -632,4 +785,126 @@ mod tests { assert_eq!(result[0].key_prefixes[1], "b:"); assert_eq!(result[0].key_prefixes[2], "c:"); } + + // ── Kernel M4 (task #50): combined term-dict + FST sidecar ─────────── + + fn sample_fields() -> Vec { + vec![ + FieldTermFstSidecar { + next_id: 3, + fst_high_water_mark: 3, + terms: vec![ + ("alpha".to_owned(), 0), + ("beta".to_owned(), 1), + ("gamma".to_owned(), 2), + ], + fst_bytes: Some(b"fake_fst_bytes_field0".to_vec()), + }, + FieldTermFstSidecar { + next_id: 1, + fst_high_water_mark: 0, + terms: vec![("delta".to_owned(), 0)], + fst_bytes: None, + }, + ] + } + + #[test] + fn term_fst_sidecar_roundtrips() { + let fields = sample_fields(); + let data = serialize_term_fst_sidecar(&fields); + let decoded = deserialize_term_fst_sidecar(&data).expect("deserialize"); + assert_eq!(decoded, fields); + } + + #[test] + fn term_fst_sidecar_save_load_roundtrips_through_disk() { + let tmp = tempfile::tempdir().expect("tempdir"); + let fields = sample_fields(); + save_term_fst_sidecar(tmp.path(), b"idx", &fields).expect("save"); + let loaded = load_term_fst_sidecar(tmp.path(), b"idx") + .expect("load") + .expect("sidecar present"); + assert_eq!(loaded, fields); + } + + #[test] + fn term_fst_sidecar_missing_returns_none() { + let tmp = tempfile::tempdir().expect("tempdir"); + let loaded = load_term_fst_sidecar(tmp.path(), b"nonexistent").expect("load"); + assert!(loaded.is_none()); + } + + #[test] + fn term_fst_sidecar_empty_fields_roundtrips() { + let data = serialize_term_fst_sidecar(&[]); + let decoded = deserialize_term_fst_sidecar(&data).expect("deserialize"); + assert!(decoded.is_empty()); + } + + #[test] + fn term_fst_sidecar_bad_magic_rejected() { + let mut data = serialize_term_fst_sidecar(&sample_fields()); + data[0] = b'X'; + assert!(deserialize_term_fst_sidecar(&data).is_err()); + } + + #[test] + fn term_fst_sidecar_bad_version_rejected() { + let mut data = serialize_term_fst_sidecar(&sample_fields()); + data[4] = 0xFF; + assert!(deserialize_term_fst_sidecar(&data).is_err()); + } + + #[test] + fn term_fst_sidecar_too_short_rejected() { + let data = vec![0u8; 3]; + assert!(deserialize_term_fst_sidecar(&data).is_err()); + } + + #[test] + fn term_fst_sidecar_truncated_term_bytes_rejected() { + let data = serialize_term_fst_sidecar(&sample_fields()); + // Truncate mid-way through the term/FST payload -- any cut here + // must fail closed (Err), never panic or return a partial Vec. + let truncated = &data[..data.len() - 5]; + assert!(deserialize_term_fst_sidecar(truncated).is_err()); + } + + #[test] + fn term_fst_sidecar_non_utf8_term_rejected() { + let mut fields = sample_fields(); + // Overwrite the first field's term list with invalid UTF-8 bytes + // encoded at the right length so the byte-level framing still + // parses right up to the UTF-8 validation step. + fields[0].terms.clear(); + let mut buf = Vec::new(); + buf.extend_from_slice(&TERM_FST_MAGIC[..]); + buf.push(TERM_FST_VERSION); + buf.extend_from_slice(&1u16.to_le_bytes()); // field_count = 1 + buf.extend_from_slice(&1u32.to_le_bytes()); // next_id + buf.extend_from_slice(&0u32.to_le_bytes()); // fst_high_water_mark + buf.extend_from_slice(&1u32.to_le_bytes()); // term_count + buf.extend_from_slice(&2u16.to_le_bytes()); // term_len = 2 + buf.extend_from_slice(&[0xFF, 0xFE]); // invalid UTF-8 + buf.extend_from_slice(&0u32.to_le_bytes()); // term_id + buf.extend_from_slice(&0u32.to_le_bytes()); // fst_len = 0 + assert!(deserialize_term_fst_sidecar(&buf).is_err()); + } + + /// K3 regression guard (mirrors `test_save_leaves_no_leftover_temp_file`): + /// the combined saver must go through `atomic_write_durable`, leaving no + /// leftover `.tfst.tmp` file after a successful save. + #[test] + fn term_fst_sidecar_save_leaves_no_leftover_temp_file() { + let tmp = tempfile::tempdir().expect("tempdir"); + save_term_fst_sidecar(tmp.path(), b"idx", &sample_fields()).expect("save"); + + let entries: Vec<_> = std::fs::read_dir(tmp.path()) + .unwrap() + .map(|e| e.unwrap().file_name()) + .collect(); + assert_eq!(entries.len(), 1, "expected exactly one file: {entries:?}"); + assert_eq!(entries[0].to_string_lossy(), "idx.tfst"); + } } diff --git a/src/text/store.rs b/src/text/store.rs index 2cc29b6ca..07ee3a254 100644 --- a/src/text/store.rs +++ b/src/text/store.rs @@ -189,6 +189,14 @@ pub struct TextIndex { /// See `resident_bytes_ground_truth` (`#[cfg(test)]`) for the equivalent /// full-walk formula this field must always match. resident_bytes_extra: usize, + + /// Kernel M4 (task #50): set to `true` when this index's term + /// dictionaries (and, where present, FST maps) were reconstructed from + /// the `.tfst` sidecar on boot instead of rebuilt from scratch by the + /// keyspace rescan. Surfaced additively via `FT.INFO` so operators can + /// see when the fast-boot path was actually taken vs silently falling + /// back to a full rescan (missing/stale/corrupt sidecar). + pub recovered_from_sidecar: bool, } impl TextIndex { @@ -251,6 +259,7 @@ impl TextIndex { doc_numeric_entries: HashMap::new(), db_index: 0, resident_bytes_extra: 0, + recovered_from_sidecar: false, } } @@ -2126,7 +2135,14 @@ impl TextStore { /// Save FST sidecar for a specific index. No-op if persist_dir not set. /// /// Called after `TextIndex::build_fst()` at FT.COMPACT time (D-11). + /// + /// Deprecated by [`Self::save_term_fst_sidecar_for_index`] (kernel M4, + /// task #50), which persists the term dictionary the FST's ids were + /// built against in the same atomic write -- kept only for the existing + /// FST-only roundtrip unit tests below; production code should call the + /// combined saver instead. #[cfg(feature = "text-index")] + #[cfg(test)] pub fn save_fst_sidecar_for_index(&self, index_name: &[u8]) { if let Some(ref dir) = self.persist_dir { if let Some(idx) = self.indexes.get(index_name) { @@ -2148,7 +2164,50 @@ impl TextStore { } } - /// Load FST sidecars for all indexes. + /// Save the combined term-dict + FST sidecar for a specific index. + /// No-op if persist_dir not set. Called after `TextIndex::build_fst()` + /// at FT.COMPACT time (kernel M4, task #50 -- supersedes the FST-only + /// sidecar so the loaded FST's ids are always backed by a matching term + /// dictionary). + #[cfg(feature = "text-index")] + pub fn save_term_fst_sidecar_for_index(&self, index_name: &[u8]) { + let Some(ref dir) = self.persist_dir else { + return; + }; + let Some(idx) = self.indexes.get(index_name) else { + return; + }; + let fields: Vec = idx + .field_term_dicts + .iter() + .enumerate() + .map(|(field_idx, dict)| { + let terms: Vec<(String, u32)> = + dict.iter().map(|(t, &id)| (t.to_owned(), id)).collect(); + let fst_bytes = idx + .fst_maps + .get(field_idx) + .and_then(|m| m.as_ref()) + .map(|m| m.as_fst().as_bytes().to_vec()); + crate::text::index_persist::FieldTermFstSidecar { + next_id: dict.next_id(), + fst_high_water_mark: dict.fst_high_water_mark, + terms, + fst_bytes, + } + }) + .collect(); + if let Err(e) = crate::text::index_persist::save_term_fst_sidecar(dir, index_name, &fields) + { + tracing::warn!( + "Failed to save term-dict+FST sidecar for {}: {}", + String::from_utf8_lossy(index_name), + e + ); + } + } + + /// Load FST-only sidecars for all indexes (legacy, id-space UNSAFE). /// /// # Deliberately NOT called during startup/recovery — do not wire this in /// @@ -2170,22 +2229,13 @@ impl TextStore { /// freshly-assigned ids from the new generation — silently wrong /// search results, not even a detectable error. /// - /// This can't be fixed at the load site: the sidecar's `term_id`s are - /// only meaningful together with the exact term dictionary they were - /// built against, and that dictionary isn't persisted today. Making - /// this load safe requires persisting the term dictionary itself (so - /// ids survive a restart, or the sidecar can be fingerprint-checked - /// against the live corpus and discarded on mismatch) — that's kernel - /// M4 scope (FTS content persistence), filed as task #50. Until #50 - /// lands, this function must stay uncalled in production; it is - /// exercised only by its own roundtrip unit tests - /// (`test_fst_sidecar_roundtrip`, `test_fst_sidecar_missing_returns_empty` - /// in this module's tests) which construct a single, self-consistent - /// generation and never see the cross-generation id collision. - /// - /// If a sidecar is missing for an index, that index's fst_maps remain None - /// (fuzzy/prefix queries will fall back to HashMap brute-force, D-13). + /// Fixed by [`Self::load_term_fst_sidecars`] (kernel M4, task #50), + /// which persists AND restores the term dictionary itself before the + /// keyspace rescan runs, so ids never drift out from under a loaded + /// FST. This FST-only function is kept solely for its own pre-existing + /// roundtrip unit tests and must stay uncalled in production. #[cfg(feature = "text-index")] + #[cfg(test)] pub fn load_fst_sidecars(&mut self) { if let Some(ref dir) = self.persist_dir { let dir = dir.clone(); @@ -2224,6 +2274,111 @@ impl TextStore { } } } + + /// Load term-dict + FST sidecars for all indexes -- the SAFE loader + /// (kernel M4, task #50). + /// + /// MUST be called after `create_index` has restored the index schemas + /// from `text-indexes.meta` and BEFORE the keyspace auto-reindex rescan + /// runs any `index_document` calls -- see `src/shard/event_loop.rs`'s + /// recovery sequencing. Seeding `field_term_dicts` first makes the + /// rescan's `TermDictionary::get_or_insert` calls resolve already-known + /// terms to their PERSISTED ids (not fresh ones), and only ever assign + /// brand-new ids (continuing from the persisted `next_id`) to terms that + /// are genuinely new since the sidecar was written. That is what makes + /// loading the FST alongside it safe: the FST's baked-in ids and the + /// live term dictionary's ids are now the same id-space by construction. + /// + /// Fail-closed per index: any missing, truncated, corrupt, + /// version-mismatched, or field-count-mismatched (schema changed since + /// the sidecar was written) sidecar causes that index to be skipped + /// entirely -- both the term dict AND the FST stay at their fresh-start + /// state, identical to today's always-rescan behavior. Never partially + /// apply a sidecar (e.g. seed the dict but skip a corrupt FST, or vice + /// versa) -- that would silently reintroduce the id-space mismatch this + /// function exists to prevent. + #[cfg(feature = "text-index")] + pub fn load_term_fst_sidecars(&mut self) { + let Some(ref dir) = self.persist_dir else { + return; + }; + let dir = dir.clone(); + let names: Vec = self.indexes.keys().cloned().collect(); + for name in names { + let loaded = match crate::text::index_persist::load_term_fst_sidecar( + &dir, + name.as_ref(), + ) { + Ok(Some(fields)) => fields, + Ok(None) => continue, // no sidecar -- fresh start, today's behavior + Err(e) => { + tracing::warn!( + "Term-dict+FST sidecar for {} failed to load, falling back to full rescan: {}", + String::from_utf8_lossy(name.as_ref()), + e + ); + continue; + } + }; + + let Some(idx) = self.indexes.get_mut(name.as_ref()) else { + continue; + }; + if loaded.len() != idx.field_term_dicts.len() { + // Schema changed (field count differs) since the sidecar + // was written -- stale, fail closed. + tracing::warn!( + "Term-dict+FST sidecar for {} has {} field(s), index has {} -- stale sidecar, falling back to full rescan", + String::from_utf8_lossy(name.as_ref()), + loaded.len(), + idx.field_term_dicts.len() + ); + continue; + } + + // Build every field's TermDictionary BEFORE mutating anything on + // `idx`, so a single bad field aborts the whole index cleanly + // (all-or-nothing per index). + let mut rebuilt_dicts = Vec::with_capacity(loaded.len()); + let mut ok = true; + for field in &loaded { + match TermDictionary::from_pairs( + field.terms.clone(), + field.next_id, + field.fst_high_water_mark, + ) { + Some(dict) => rebuilt_dicts.push(dict), + None => { + ok = false; + break; + } + } + } + if !ok || rebuilt_dicts.len() != loaded.len() { + tracing::warn!( + "Term-dict+FST sidecar for {} is internally inconsistent -- falling back to full rescan", + String::from_utf8_lossy(name.as_ref()) + ); + continue; + } + + for (field_idx, (dict, field)) in rebuilt_dicts.into_iter().zip(loaded).enumerate() { + idx.field_term_dicts[field_idx] = dict; + if let Some(fst_bytes) = field.fst_bytes { + match fst::Map::new(fst_bytes) { + Ok(map) => idx.set_fst_map(field_idx, Some(map)), + Err(e) => tracing::warn!( + "FST bytes for {}[{}] failed to parse despite a valid term-dict sidecar: {}", + String::from_utf8_lossy(name.as_ref()), + field_idx, + e + ), + } + } + } + idx.recovered_from_sidecar = true; + } + } } #[cfg(test)] @@ -2596,6 +2751,218 @@ mod tests { assert!(loaded.is_empty(), "Missing sidecar should return empty Vec"); } + // ── Kernel M4 (task #50): term-dict + FST sidecar durability ───────── + + /// Build a fresh TextStore of the same shape `make_index_with_docs` + /// builds directly on TextIndex, so both the "before crash" and + /// "after restart" sides of the equivalence test share one schema. + fn make_store_with_docs(persist_dir: &std::path::Path, docs: &[(&str, &str)]) -> TextStore { + use crate::protocol::Frame; + use crate::text::types::BM25Config; + let mut store = TextStore::new(); + store.set_persist_dir(persist_dir.to_path_buf()); + let field = TextFieldDef::new(Bytes::from_static(b"body")); + let idx = TextIndex::new( + Bytes::from_static(b"test_idx"), + Vec::new(), + vec![field], + BM25Config::default(), + ); + store + .create_index(Bytes::from_static(b"test_idx"), idx) + .expect("create_index"); + let idx = store.get_index_mut(b"test_idx").expect("index exists"); + for (i, (key, text)) in docs.iter().enumerate() { + let args = vec![ + Frame::BulkString(Bytes::from_static(b"body")), + Frame::BulkString(Bytes::copy_from_slice(text.as_bytes())), + ]; + idx.index_document(i as u64, key.as_bytes(), &args); + } + store + } + + /// The core equivalence gate for task #50: an index recovered from the + /// `.tfst` sidecar (term-dict seed + rescan) must answer FUZZY and + /// PREFIX queries -- which exercise the FST, not just the HashMap + /// brute-force path -- IDENTICALLY to a from-scratch rebuilt index over + /// the same corpus. + #[test] + fn load_term_fst_sidecars_survives_restart_and_matches_rebuilt() { + let tmp = tempfile::tempdir().expect("tempdir"); + let docs: &[(&str, &str)] = &[ + ("doc:0", "machine vision"), + ("doc:1", "deep learning"), + ("doc:2", "machine learning deep"), + ]; + + // "Before crash": build the index, compact (builds FST), persist + // the combined term-dict+FST sidecar -- mirrors FT.COMPACT. + let mut before = make_store_with_docs(tmp.path(), docs); + { + let idx = before.get_index_mut(b"test_idx").expect("index exists"); + idx.build_fst(); + } + before.save_term_fst_sidecar_for_index(b"test_idx"); + + let expected_fuzzy = { + let idx = before.get_index(b"test_idx").expect("index exists"); + idx.expand_terms(0, "machn", &TermModifier::Fuzzy(1)) + }; + let expected_prefix = { + let idx = before.get_index(b"test_idx").expect("index exists"); + idx.expand_terms(0, "lear", &TermModifier::Prefix) + }; + assert!(!expected_fuzzy.is_empty(), "fixture sanity: fuzzy matches"); + assert!( + !expected_prefix.is_empty(), + "fixture sanity: prefix matches" + ); + + // "After restart": schema-only empty index (as `create_index` + // restores from `text-indexes.meta`), THEN load_term_fst_sidecars + // (seeds term dicts + FST BEFORE any doc is re-indexed), THEN the + // keyspace rescan re-indexes the same docs -- exactly the sequence + // wired in `src/shard/event_loop.rs`. + let mut after = TextStore::new(); + after.set_persist_dir(tmp.path().to_path_buf()); + let field = TextFieldDef::new(Bytes::from_static(b"body")); + let empty_idx = TextIndex::new( + Bytes::from_static(b"test_idx"), + Vec::new(), + vec![field], + crate::text::types::BM25Config::default(), + ); + after + .create_index(Bytes::from_static(b"test_idx"), empty_idx) + .expect("create_index"); + after.load_term_fst_sidecars(); + assert!( + after.get_index(b"test_idx").unwrap().recovered_from_sidecar, + "sidecar was valid -- recovered_from_sidecar must be true" + ); + { + use crate::protocol::Frame; + let idx = after.get_index_mut(b"test_idx").expect("index exists"); + for (i, (key, text)) in docs.iter().enumerate() { + let args = vec![ + Frame::BulkString(Bytes::from_static(b"body")), + Frame::BulkString(Bytes::copy_from_slice(text.as_bytes())), + ]; + idx.index_document(i as u64, key.as_bytes(), &args); + } + } + + let actual_fuzzy = { + let idx = after.get_index(b"test_idx").expect("index exists"); + idx.expand_terms(0, "machn", &TermModifier::Fuzzy(1)) + }; + let actual_prefix = { + let idx = after.get_index(b"test_idx").expect("index exists"); + idx.expand_terms(0, "lear", &TermModifier::Prefix) + }; + + let sorted = |mut v: Vec| { + v.sort_unstable(); + v + }; + assert_eq!( + sorted(actual_fuzzy), + sorted(expected_fuzzy), + "FUZZY term-id expansion after sidecar-recovered restart must match a from-scratch rebuild" + ); + assert_eq!( + sorted(actual_prefix), + sorted(expected_prefix), + "PREFIX term-id expansion after sidecar-recovered restart must match a from-scratch rebuild" + ); + + // And the search results built on top of those ids must match too. + let expected_results = { + let idx = before.get_index(b"test_idx").expect("index exists"); + let ids = idx.expand_terms(0, "machn", &TermModifier::Fuzzy(1)); + let mut r = idx.search_field_or(0, &ids, None, None, 10); + r.sort_by(|a, b| a.key.cmp(&b.key)); + r.into_iter().map(|r| r.key).collect::>() + }; + let actual_results = { + let idx = after.get_index(b"test_idx").expect("index exists"); + let ids = idx.expand_terms(0, "machn", &TermModifier::Fuzzy(1)); + let mut r = idx.search_field_or(0, &ids, None, None, 10); + r.sort_by(|a, b| a.key.cmp(&b.key)); + r.into_iter().map(|r| r.key).collect::>() + }; + assert_eq!(actual_results, expected_results); + } + + /// No `.tfst` sidecar on disk (fresh index, or FT.COMPACT was never + /// called) -- `load_term_fst_sidecars` must be a silent no-op, leaving + /// `recovered_from_sidecar` false and the term dict empty (today's + /// full-rescan behavior). + #[test] + fn load_term_fst_sidecars_missing_is_noop() { + let tmp = tempfile::tempdir().expect("tempdir"); + let mut store = TextStore::new(); + store.set_persist_dir(tmp.path().to_path_buf()); + let field = TextFieldDef::new(Bytes::from_static(b"body")); + let idx = TextIndex::new( + Bytes::from_static(b"test_idx"), + Vec::new(), + vec![field], + crate::text::types::BM25Config::default(), + ); + store + .create_index(Bytes::from_static(b"test_idx"), idx) + .expect("create_index"); + + store.load_term_fst_sidecars(); + + let idx = store.get_index(b"test_idx").expect("index exists"); + assert!(!idx.recovered_from_sidecar); + assert_eq!(idx.field_term_dicts[0].term_count(), 0); + } + + /// A sidecar written for a 1-field schema must NOT be applied to an + /// index that now has 2 fields (schema changed since the sidecar was + /// written) -- fail closed rather than silently misapplying ids to the + /// wrong field. + #[test] + fn load_term_fst_sidecars_field_count_mismatch_falls_back() { + let tmp = tempfile::tempdir().expect("tempdir"); + let docs: &[(&str, &str)] = &[("doc:0", "machine vision")]; + let mut before = make_store_with_docs(tmp.path(), docs); + { + let idx = before.get_index_mut(b"test_idx").expect("index exists"); + idx.build_fst(); + } + before.save_term_fst_sidecar_for_index(b"test_idx"); + + let mut after = TextStore::new(); + after.set_persist_dir(tmp.path().to_path_buf()); + let idx = TextIndex::new( + Bytes::from_static(b"test_idx"), + Vec::new(), + vec![ + TextFieldDef::new(Bytes::from_static(b"body")), + TextFieldDef::new(Bytes::from_static(b"body2")), + ], + crate::text::types::BM25Config::default(), + ); + after + .create_index(Bytes::from_static(b"test_idx"), idx) + .expect("create_index"); + + after.load_term_fst_sidecars(); + + let idx = after.get_index(b"test_idx").expect("index exists"); + assert!( + !idx.recovered_from_sidecar, + "field-count mismatch must fail closed, not partially apply" + ); + assert_eq!(idx.field_term_dicts[0].term_count(), 0); + assert_eq!(idx.field_term_dicts[1].term_count(), 0); + } + // ── v0.1.10 G-1: BM25 AS_OF MVCC filter ────────────────────────────── /// Doc with no recorded insert_lsn (pre-MVCC) is always visible. diff --git a/src/text/term_dict.rs b/src/text/term_dict.rs index 7d1994396..c57f0bdd8 100644 --- a/src/text/term_dict.rs +++ b/src/text/term_dict.rs @@ -62,6 +62,62 @@ impl TermDictionary { self.terms.get(term).copied() } + /// Reconstruct a `TermDictionary` from a persisted `(term, id)` pair set + /// (kernel M4 / task #50: term-dict sidecar load path). + /// + /// This is the id-space-preserving counterpart to repeated + /// `get_or_insert` calls: instead of assigning ids by first-encounter + /// order (which is NOT reproducible across a restart because the + /// keyspace rescan iterates `DashTable` hash-iteration order, not the + /// original insertion order), every id is taken verbatim from the + /// sidecar. `next_id` continues from the persisted high-water mark so + /// terms discovered fresh by the post-load rescan get NEW, non-colliding + /// ids rather than restarting from 0. + /// + /// Callers MUST validate the sidecar (magic/version/checksum) and + /// reject anything malformed BEFORE calling this -- this constructor + /// trusts its inputs (ids may be sparse or unsorted; both are fine, + /// duplicates are rejected by returning `None` since a single term + /// can't legitimately hold two persisted ids and a collision means the + /// sidecar was corrupted between writes). + #[must_use] + pub fn from_pairs( + pairs: Vec<(String, u32)>, + next_id: u32, + fst_high_water_mark: u32, + ) -> Option { + let mut terms = HashMap::with_capacity(pairs.len()); + let mut seen_ids: std::collections::HashSet = + std::collections::HashSet::with_capacity(pairs.len()); + let mut resident_bytes = 0usize; + let mut max_id_plus_one = 0u32; + for (term, id) in pairs { + if id >= next_id { + // A persisted id can never reach/exceed the persisted + // next_id counter -- that would mean the sidecar's own + // invariant was violated when it was written. + return None; + } + if !seen_ids.insert(id) { + return None; // duplicate id -- corrupt sidecar + } + resident_bytes += term.len() + std::mem::size_of::() + MAP_ENTRY_OVERHEAD; + max_id_plus_one = max_id_plus_one.max(id + 1); + if terms.insert(term, id).is_some() { + return None; // duplicate term -- corrupt sidecar + } + } + if max_id_plus_one > next_id || fst_high_water_mark > next_id { + return None; // internal inconsistency -- fail closed + } + Some(Self { + terms, + next_id, + fst_high_water_mark, + resident_bytes, + }) + } + /// Number of unique terms in the dictionary. pub fn term_count(&self) -> usize { self.terms.len() @@ -157,4 +213,62 @@ mod tests { "100k reads of resident_bytes() took {elapsed:?} -- looks like a walk, not O(1)" ); } + + /// Kernel M4 (task #50): `from_pairs` reconstructs a dictionary whose + /// ids/lookups/resident_bytes are indistinguishable from one built by + /// live `get_or_insert` calls in the same order. + #[test] + fn from_pairs_roundtrips_equivalent_to_get_or_insert() { + let mut live = TermDictionary::new(); + let id_a = live.get_or_insert("alpha"); + let id_b = live.get_or_insert("beta"); + let id_c = live.get_or_insert("gamma"); + live.fst_high_water_mark = live.next_id(); + + let pairs: Vec<(String, u32)> = live.iter().map(|(t, &id)| (t.to_owned(), id)).collect(); + let restored = TermDictionary::from_pairs(pairs, live.next_id(), live.fst_high_water_mark) + .expect("valid sidecar pairs must reconstruct"); + + assert_eq!(restored.get("alpha"), Some(id_a)); + assert_eq!(restored.get("beta"), Some(id_b)); + assert_eq!(restored.get("gamma"), Some(id_c)); + assert_eq!(restored.next_id(), live.next_id()); + assert_eq!(restored.fst_high_water_mark, live.fst_high_water_mark); + assert_eq!(restored.resident_bytes(), live.resident_bytes()); + assert_eq!( + restored.resident_bytes(), + restored.resident_bytes_ground_truth() + ); + } + + /// A term whose id is not less than `next_id` violates the sidecar's + /// own invariant -- fail closed (return `None`), never silently clamp. + #[test] + fn from_pairs_rejects_id_at_or_above_next_id() { + let pairs = vec![("x".to_owned(), 5u32)]; + assert!(TermDictionary::from_pairs(pairs, 5, 0).is_none()); + } + + /// Two different terms claiming the same persisted id is a corrupt + /// sidecar -- fail closed. + #[test] + fn from_pairs_rejects_duplicate_ids() { + let pairs = vec![("x".to_owned(), 0u32), ("y".to_owned(), 0u32)]; + assert!(TermDictionary::from_pairs(pairs, 2, 0).is_none()); + } + + /// A `fst_high_water_mark` above `next_id` is internally inconsistent + /// -- fail closed. + #[test] + fn from_pairs_rejects_hwm_above_next_id() { + let pairs = vec![("x".to_owned(), 0u32)]; + assert!(TermDictionary::from_pairs(pairs, 1, 5).is_none()); + } + + #[test] + fn from_pairs_empty_is_valid() { + let restored = TermDictionary::from_pairs(Vec::new(), 0, 0).expect("empty is valid"); + assert_eq!(restored.term_count(), 0); + assert_eq!(restored.next_id(), 0); + } } diff --git a/tests/crash_matrix_cross_plane/planes.rs b/tests/crash_matrix_cross_plane/planes.rs index bd1864451..a0dc6edaf 100644 --- a/tests/crash_matrix_cross_plane/planes.rs +++ b/tests/crash_matrix_cross_plane/planes.rs @@ -288,6 +288,88 @@ pub fn vec_search_contains(c: &mut Conn, idx: &str, k: u32, blob: &[u8], key: &s .any(|v| matches!(v, Resp::Bulk(Some(b)) if b == key.as_bytes())) } +// --------------------------------------------------------------------------- +// Text (FTS) — kernel M4, task #50: term-dict + FST sidecar durability +// --------------------------------------------------------------------------- + +pub fn text_index_name(tag: &str) -> String { + format!("{tag}textidx") +} + +pub fn text_key(tag: &str, n: u64) -> String { + format!("{{{tag}}}:txt:{n}") +} + +pub fn ft_create_text(c: &mut Conn, idx: &str, prefix: &str) { + let reply = c.cmd(&[ + b"FT.CREATE", + idx.as_bytes(), + b"ON", + b"HASH", + b"PREFIX", + b"1", + prefix.as_bytes(), + b"SCHEMA", + b"body", + b"TEXT", + ]); + assert_eq!(reply, Resp::Simple("OK".into()), "FT.CREATE {idx} failed"); +} + +pub fn hset_text(c: &mut Conn, key: &str, body: &str) { + match c.cmd(&[b"HSET", key.as_bytes(), b"body", body.as_bytes()]) { + Resp::Int(_) => {} + other => panic!("HSET {key} failed: {other:?}"), + } +} + +/// FT.COMPACT: builds + persists the term-dict + FST sidecar +/// (`TextStore::save_term_fst_sidecar_for_index`) so it survives kill-9. +pub fn ft_compact(c: &mut Conn, idx: &str) { + let reply = c.cmd(&[b"FT.COMPACT", idx.as_bytes()]); + assert_eq!(reply, Resp::Simple("OK".into()), "FT.COMPACT {idx} failed"); +} + +/// True if `key` is among the results of a FUZZY query against `idx` -- +/// FUZZY expansion exercises the FST path (`TermModifier::Fuzzy`), which is +/// exactly the path that is unsafe to serve from a stale-id-space sidecar +/// (see `TextStore::load_fst_sidecars`'s doc comment). A crash-durability +/// cell that only checked exact-term search would miss that corruption +/// class entirely. +pub fn ft_search_fuzzy_contains(c: &mut Conn, idx: &str, fuzzy_term: &str, key: &str) -> bool { + let query = format!("@body:%{fuzzy_term}%"); + let r = c.cmd(&[ + b"FT.SEARCH", + idx.as_bytes(), + query.as_bytes(), + b"DIALECT", + b"2", + ]); + let items = as_array(&r); + items[1..] + .iter() + .step_by(2) + .any(|v| matches!(v, Resp::Bulk(Some(b)) if b == key.as_bytes())) +} + +/// Additive `FT.INFO` `sidecar_recovered_indexes` counter (kernel M4, task +/// #50) — 1+ means at least one shard's boot took the fast sidecar-recovery +/// path instead of falling back to a full keyspace rescan. +pub fn ft_info_sidecar_recovered_indexes(c: &mut Conn, idx: &str) -> i64 { + let r = c.cmd(&[b"FT.INFO", idx.as_bytes()]); + let items = as_array(&r); + let mut i = 0; + while i + 1 < items.len() { + if let Resp::Bulk(Some(k)) = &items[i] { + if k == b"sidecar_recovered_indexes" { + return as_int(&items[i + 1]); + } + } + i += 2; + } + panic!("FT.INFO {idx}: sidecar_recovered_indexes key not found"); +} + // --------------------------------------------------------------------------- // WS (workspace registry — process-global, shard-0-pinned per Wave B) // --------------------------------------------------------------------------- diff --git a/tests/crash_matrix_cross_plane/scenarios.rs b/tests/crash_matrix_cross_plane/scenarios.rs index 8c0687336..726e237e8 100644 --- a/tests/crash_matrix_cross_plane/scenarios.rs +++ b/tests/crash_matrix_cross_plane/scenarios.rs @@ -303,6 +303,97 @@ pub fn vector_isolated(cfg: &Config) { drop(guard2); } +/// Text (FTS) isolated: kernel M4, task #50 tripwire. FT.CREATE, index docs, +/// FT.COMPACT (builds + persists the term-dict + FST sidecar), index MORE +/// docs post-compact (so boot must both recover the sidecar AND correctly +/// continue assigning ids to genuinely-new terms), kill-9, restart, verify a +/// FUZZY query -- which exercises the FST expansion path, not just exact +/// HashMap lookup -- still finds every doc, and that `FT.INFO` reports the +/// index as sidecar-recovered (not a silent fallback to full rescan). +/// +/// This is a default-GREEN cell: unlike the RED root-cause groups tracked +/// elsewhere in this matrix, task #50's fix is expected to hold here. A +/// regression here means either the `.tfst` sidecar stopped being written +/// at FT.COMPACT time, `load_term_fst_sidecars` stopped being wired into +/// shard boot before the rescan, or the id-continuation invariant +/// (`TermDictionary::from_pairs`'s `next_id` handoff) broke. +pub fn text_fts_sidecar_isolated(cfg: &Config) { + let dir = harness::unique_dir(&format!("txtiso-{}", cfg.label)); + let (guard, port) = harness::spawn_moon_on(&dir, cfg, &[]); + let mut c = Conn::open(port); + let tag = "txtiso"; + let idx = text_index_name(tag); + let prefix = format!("{{{tag}}}:txt:"); + ft_create_text(&mut c, &idx, &prefix); + + if cfg.no_durability_contract() { + hset_text(&mut c, &format!("{prefix}0"), "machine vision system"); + harness::crash(guard, port); + let (guard2, port2) = harness::spawn_moon_on(&dir, cfg, &[]); + let mut c2 = Conn::open(port2); + assert!(matches!(c2.cmd_s(&["PING"]), crate::resp::Resp::Simple(_))); + drop(guard2); + return; + } + + // Pre-compact corpus: every doc contains "machine", which stems to + // "machin" -- FUZZY-reachable via edit-distance-1 probe "machn". (Note: + // "machinery" stems to a DIFFERENT token ("machineri") that is NOT + // within edit-distance-1 of "machn" -- an earlier draft of this fixture + // used "deep learning machinery" and got a false-negative FUZZY miss + // that looked like a sidecar-recovery bug but was actually a test-data + // bug in the fixture, not the product.) + let pre_compact: Vec<(String, &str)> = vec![ + (text_key(tag, 0), "machine vision system"), + (text_key(tag, 1), "deep learning machine"), + (text_key(tag, 2), "machine learning pipeline"), + ]; + for (key, body) in &pre_compact { + hset_text(&mut c, key, body); + } + ft_compact(&mut c, &idx); + + // Post-compact corpus: NEW terms discovered only by the rescan after a + // sidecar-recovered boot -- proves `next_id` continuation doesn't + // collide with the loaded sidecar's id-space. + let post_compact: Vec<(String, &str)> = vec![ + (text_key(tag, 3), "machine translation service"), + (text_key(tag, 4), "robotic machine arm"), + ]; + for (key, body) in &post_compact { + hset_text(&mut c, key, body); + } + + let all_docs: Vec<(String, &str)> = pre_compact + .into_iter() + .chain(post_compact) + .collect::>(); + harness::wait_for_aof_bytes_any_shard( + &dir, + all_docs.last().expect("non-empty corpus").0.as_bytes(), + ); + harness::crash(guard, port); + + let (guard2, port2) = harness::spawn_moon_on(&dir, cfg, &[]); + let mut c2 = Conn::open(port2); + for (key, _) in &all_docs { + assert!( + ft_search_fuzzy_contains(&mut c2, &idx, "machn", key), + "cell {}: FUZZY 'machn' must find {key} after sidecar-recovered restart \ + (term-dict+FST sidecar durability, kernel M4 task #50)", + cfg.label + ); + } + assert!( + ft_info_sidecar_recovered_indexes(&mut c2, &idx) >= 1, + "cell {}: FT.INFO sidecar_recovered_indexes must be >= 1 after a clean \ + restart with a valid .tfst sidecar on disk -- 0 means boot silently \ + fell back to a full keyspace rescan", + cfg.label + ); + drop(guard2); +} + /// WS isolated: WS CREATE, synced, kill, verify WS LIST. pub fn ws_isolated(cfg: &Config) { let dir = harness::unique_dir(&format!("wsiso-{}", cfg.label)); diff --git a/tests/crash_matrix_cross_plane/tests_prod.rs b/tests/crash_matrix_cross_plane/tests_prod.rs index f5ae5f23e..92c7fec5c 100644 --- a/tests/crash_matrix_cross_plane/tests_prod.rs +++ b/tests/crash_matrix_cross_plane/tests_prod.rs @@ -47,6 +47,14 @@ fn cross_plane_prod_s1_graph_isolated() { scenarios::graph_isolated(&Config::PROD_S1); } +/// Kernel M4, task #50: term-dict + FST sidecar durability -- default-GREEN +/// tripwire, see `scenarios::text_fts_sidecar_isolated`'s doc comment. +#[test] +#[ignore] // Requires built release binary; run explicitly. +fn cross_plane_prod_s1_text_fts_sidecar_isolated() { + scenarios::text_fts_sidecar_isolated(&Config::PROD_S1); +} + #[test] #[ignore] fn cross_plane_prod_s1_vector_isolated() { @@ -140,6 +148,14 @@ fn cross_plane_prod_s4_vector_isolated() { scenarios::vector_isolated(&Config::PROD_S4); } +/// Kernel M4, task #50 -- shards=4 leg (multi-shard: verifies the sidecar +/// is keyed/loaded per-shard, not once globally). +#[test] +#[ignore] // Requires built release binary; run explicitly. +fn cross_plane_prod_s4_text_fts_sidecar_isolated() { + scenarios::text_fts_sidecar_isolated(&Config::PROD_S4); +} + #[test] #[ignore] fn cross_plane_prod_s4_ws_isolated() {