feat(persistence): FTS term-dict + FST sidecar durability across kill-9 (kernel M4, task #50)

CHANGELOG.md

@@ -29,6 +29,39 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - CI builds/tests run with `debug = 0` (no debuginfo) via
   `CARGO_PROFILE_*_DEBUG` env — smaller artifacts (faster cache
   save/restore) and faster linking; local builds unaffected.
+### Fixed — FTS term-dict + FST sidecar durability, ends restart-rescan-only recovery (kernel M4, task #50)
+
+The full-text-search inverted index was the last plane not kill-9-lossless:
+every restart rebuilt every text index by rescanning the keyspace and
+reassigning term ids by `DashTable` hash-iteration first-encounter order —
+not reproducible across restarts, which is why the pre-existing `.fst`
+sidecar write path had its load path (`load_fst_sidecars`) deliberately left
+uncalled in production (wiring it once corrupted FUZZY/PREFIX results: the
+sidecar's baked-in term ids silently collided with a freshly-rescanned
+dictionary's differently-assigned ids).
+
+Fixed by persisting the term dictionary itself alongside the FST, in one
+atomic sidecar (`{shard_dir}/{index}.tfst`, magic `TFS2`, version-stamped,
+`atomic_write_durable`): per TEXT field, `next_id`, `fst_high_water_mark`,
+every `(term, id)` pair, and the optional FST bytes. `TermDictionary::from_pairs`
+reconstructs a dictionary whose ids are taken verbatim from the sidecar
+(never reassigned) and whose `next_id` continues the persisted high-water
+mark. Wired into shard boot (`src/shard/event_loop.rs`) so
+`TextStore::load_term_fst_sidecars` runs AFTER text index schemas are
+restored but BEFORE the keyspace auto-reindex rescan — seeding the term
+dicts first makes the rescan's `get_or_insert` calls resolve known terms to
+their persisted ids and assign fresh, non-colliding ids only to genuinely
+new terms, which is what makes loading the FST alongside it safe (FST ids
+and live dict ids are the same id-space by construction). Fails closed per
+index on any missing/truncated/corrupt/version-mismatched/field-count-
+mismatched sidecar — falls back to today's full rescan, never partially
+applies a sidecar. `FT.COMPACT` now calls the combined saver
+(`save_term_fst_sidecar_for_index`) instead of the old FST-only one. New
+`FT.INFO` counters `sidecar_recovered_indexes` / `text_indexes_total`
+(additive across shards) surface fast-boot coverage. New default-GREEN
+crash-matrix cells `cross_plane_prod_{s1,s4}_text_fts_sidecar_isolated`
+verify a FUZZY query survives kill-9 identically to a from-scratch rebuild.
+New fuzz target `term_fst_sidecar` covers the sidecar decoder.
 ### Security — clear dependency vulnerability backlog (task #51)
 
 `sdk/python` (uv.lock, 36 open Dependabot alerts incl. 1 CRITICAL) and

fuzz/Cargo.toml

@@ -103,3 +103,8 @@ doc = false
 name = "redis_rdb_load"
 path = "fuzz_targets/redis_rdb_load.rs"
 doc = false
+
+[[bin]]
+name = "term_fst_sidecar"
+path = "fuzz_targets/term_fst_sidecar.rs"
+doc = false

fuzz/fuzz_targets/term_fst_sidecar.rs

@@ -0,0 +1,16 @@
+#![no_main]
+use libfuzzer_sys::fuzz_target;
+
+use moon::text::index_persist::deserialize_term_fst_sidecar;
+
+/// Fuzz the combined term-dict + FST sidecar decoder (kernel M4, task #50).
+///
+/// Exercises magic/version validation, the per-field term-count/term-len
+/// framing loop, UTF-8 term validation, and FST-length truncation handling.
+/// Any panic or OOB access is a bug -- malformed on-disk sidecar bytes
+/// (truncated write, bit rot, downgrade/upgrade version skew) must always
+/// fail closed with `Err`, never panic and never return a partially
+/// populated `Vec`.
+fuzz_target!(|data: &[u8]| {
+    let _ = deserialize_term_fst_sidecar(data);
+});

src/command/vector_search/ft_admin.rs

@@ -152,8 +152,10 @@ pub fn ft_compact(
     #[cfg(feature = "text-index")]
     if let Some(text_idx) = text_store.get_index_mut_for_db(name.as_ref(), db_index) {
         text_idx.build_fst();
-        // Persist FST sidecar to disk so it survives server restart (FUZ-02).
-        text_store.save_fst_sidecar_for_index(name.as_ref());
+        // Persist term-dict + FST sidecar to disk so it survives server
+        // restart (FUZ-02; kernel M4 task #50 -- combined saver so the
+        // loaded FST's ids are always backed by a matching term dict).
+        text_store.save_term_fst_sidecar_for_index(name.as_ref());
     }
 
     // Return OK if either a vector index or text index exists with this name (db-scoped).

src/command/vector_search/ft_info.rs

@@ -253,6 +253,17 @@ pub fn ft_info(
         // Independent from vector_version_token — hybrid-index callers check both.
         items.push(Frame::BulkString(Bytes::from_static(b"text_version_token")));
         items.push(Frame::Integer(text_store.version_token() as i64));
+        // Kernel M4 (task #50): coverage counter mirroring
+        // `segments_with_exact_rerank` for the vector engine -- additive
+        // across shards so `sidecar_recovered_indexes < text_indexes_total`
+        // means at least one shard fell back to a full keyspace rescan for
+        // at least one index (missing/stale/corrupt `.tfst` sidecar).
+        items.push(Frame::BulkString(Bytes::from_static(
+            b"sidecar_recovered_indexes",
+        )));
+        items.push(Frame::Integer(text_idx.recovered_from_sidecar as i64));
+        items.push(Frame::BulkString(Bytes::from_static(b"text_indexes_total")));
+        items.push(Frame::Integer(1));
     }
 
     Frame::Array(items.into())
@@ -272,6 +283,8 @@ pub fn ft_info(
 /// aggregate).
 /// Additive per-field keys (matched by `field_name` inside `vector_fields` /
 /// `text_fields`): `num_docs`, `mutable_vectors`, `immutable_segments`.
+/// Also additive: `sidecar_recovered_indexes` / `text_indexes_total`
+/// (kernel M4, task #50 -- FT.INFO term-dict sidecar recovery coverage).
 ///
 /// Any `Frame::Error` (local or remote) is propagated unchanged (fail-loud,
 /// same semantics as `scatter_invalidate_range`).
@@ -288,6 +301,8 @@ pub fn merge_ft_info_responses(local: Frame, remotes: &[Frame]) -> Frame {
         b"warm_segments_with_exact_rerank",
         b"unloaded_segments",
         b"unloaded_segments_with_exact_rerank",
+        b"sidecar_recovered_indexes",
+        b"text_indexes_total",
     ];
     const ADDITIVE_FIELD: &[&[u8]] = &[b"num_docs", b"mutable_vectors", b"immutable_segments"];
 
@@ -483,6 +498,16 @@ fn ft_info_text_only(
     items.push(Frame::BulkString(Bytes::from_static(b"text_version_token")));
     items.push(Frame::Integer(text_store.version_token() as i64));
 
+    // Kernel M4 (task #50): coverage counters mirroring the hybrid-index
+    // path above (`sidecar_recovered_indexes` / `text_indexes_total`) --
+    // additive across shards via `merge_ft_info_responses`.
+    items.push(Frame::BulkString(Bytes::from_static(
+        b"sidecar_recovered_indexes",
+    )));
+    items.push(Frame::Integer(idx.recovered_from_sidecar as i64));
+    items.push(Frame::BulkString(Bytes::from_static(b"text_indexes_total")));
+    items.push(Frame::Integer(1));
+
     Frame::Array(items.into())
 }
 

src/shard/event_loop.rs

@@ -1022,6 +1022,18 @@ impl super::Shard {
                         }
                     }
                 });
+
+                // Kernel M4 (task #50): seed each restored text index's term
+                // dictionaries (and, where the sidecar validates cleanly,
+                // FST maps) from the `.tfst` combined sidecar BEFORE the
+                // keyspace rescan below runs any `index_document` calls.
+                // This MUST happen in this order -- see
+                // `TextStore::load_term_fst_sidecars`'s doc comment for why
+                // seeding after the rescan (or not at all) is exactly the
+                // stale-id-space corruption this closes.
+                crate::shard::slice::with_shard(|s| {
+                    s.text_store.load_term_fst_sidecars();
+                });
             }
 
             // Auto-reindex existing HASH keys that match vector or text index prefixes.