perf(registry): throttle the auto-embed ticker's width probe (#677 item 2)

src/main.rs

@@ -1175,7 +1175,7 @@ fn run_flush_tick(
         let workers = state.embed_parallel();
         tokio::task::block_in_place(|| {
             parallel_map(flushed, workers, |name| {
-                match state.refresh_embeddings(&name, Deadline::unbounded()) {
+                match state.auto_refresh_embeddings(&name, Deadline::unbounded()) {
                     None | Some(Ok((0, _))) => {}
                     Some(Ok((embedded, _))) => {
                         info!(context = %name, embedded, "auto-embedded glosses");
@@ -1199,7 +1199,7 @@ fn run_flush_tick(
             let workers = state.embed_parallel();
             tokio::task::block_in_place(|| {
                 parallel_map(stale, workers, |name| {
-                    match state.refresh_passage_embeddings(&name, Deadline::unbounded()) {
+                    match state.auto_refresh_passage_embeddings(&name, Deadline::unbounded()) {
                         None => {}
                         Some(Ok(outcome)) if outcome.embedded == 0 => {}
                         Some(Ok(outcome)) => {

src/registry.rs

@@ -2300,6 +2300,14 @@ struct StateInner {
     /// fallback path. Valid for the whole process because the provider
     /// (and so the model) is fixed at boot.
     cue_cache: Mutex<CueCache>,
+    /// The output width the provider most recently produced, and when
+    /// — shared across every context because the width is a property
+    /// of the PROVIDER, not of any one context's data. Lets a refresh
+    /// skip its width probe (`embeddings.rs`) when a recent-enough
+    /// observation already agrees with what is carried, instead of
+    /// paying a provider round trip on every no-op pass. See
+    /// [`AppState::provider_width_recently_confirmed`].
+    observed_embed_width: Mutex<Option<(std::time::Instant, usize)>>,
     /// The exact-match retrieval cache (issue #150): an identical
     /// request against an identical corpus state answers from the
     /// stored response. Unlike `cue_cache`, validity is
@@ -2738,6 +2746,14 @@ impl AppState {
         match self.timed_embed(embedder, texts, EmbedPurpose::Index, deadline) {
             Ok(vectors) => {
                 self.0.metrics.record_embed_refresh(true);
+                // Every refresh embed call — probe included — lands
+                // here, so this is where the process-wide width
+                // observation `provider_width_recently_confirmed`
+                // reads gets refreshed, regardless of which context or
+                // table asked.
+                if let Some(width) = vectors.first().map(Vec::len) {
+                    *self.0.observed_embed_width.lock() = Some((std::time::Instant::now(), width));
+                }
                 Ok(vectors)
             }
             Err(error) => {
@@ -2747,6 +2763,36 @@ impl AppState {
         }
     }
 
+    /// Whether a refresh may skip its own width probe: the provider's
+    /// most recently observed output width, from ANY context's embed
+    /// call, is both fresh (`WIDTH_OBSERVATION_TRUST`) and equal to
+    /// `carried` — the width this refresh's own sidecar already holds.
+    /// Deliberately one-sided: a stale-or-absent observation, or one
+    /// that disagrees with `carried`, answers `false` and lets the
+    /// caller fall back to spending an actual provider round trip —
+    /// an unconfirmed skip could hide a real width change, but an
+    /// unconfirmed probe only ever costs one extra call.
+    fn provider_width_recently_confirmed(&self, carried: usize) -> bool {
+        self.0
+            .observed_embed_width
+            .lock()
+            .is_some_and(|(observed_at, width)| {
+                width == carried && width_observation_fresh(&observed_at)
+            })
+    }
+
+    /// Test-only: rewinds the remembered provider width observation so
+    /// `WIDTH_OBSERVATION_TRUST` can elapse without the test sleeping
+    /// through it — the same shape as `age_load_failures` below.
+    #[cfg(test)]
+    pub fn age_width_observation(&self, by: std::time::Duration) {
+        if let Some((observed_at, _)) = &mut *self.0.observed_embed_width.lock() {
+            *observed_at = observed_at
+                .checked_sub(by)
+                .expect("test ages within the Instant range");
+        }
+    }
+
     /// The query side of every embedding lookup: process cache first,
     /// provider (as [`EmbedPurpose::Query`]) on a miss. No lock is held
     /// across the provider call.
@@ -3182,6 +3228,24 @@ fn still_quarantined(failed_at: &std::time::Instant) -> bool {
     failed_at.elapsed() < LOAD_FAILURE_RETRY
 }
 
+/// How long a provider's most recently observed output width is
+/// trusted for skipping a refresh's own width probe (issue #677 item
+/// 2). Long enough that a busy, gloss-stable context stops paying a
+/// provider round trip on every 5s flush tick; short enough that a
+/// genuine backend swap is still caught well within a minute even for
+/// a context whose own writes never trigger a real embed.
+const WIDTH_OBSERVATION_TRUST: std::time::Duration = std::time::Duration::from_secs(60);
+
+/// Whether a remembered width observation is still within its trust
+/// window. Same shape as `still_quarantined` above — the `<` boundary
+/// is not a gap a behavioral test can close (see that function's doc)
+/// — kept as its own predicate so both share the one place the
+/// boundary is decided.
+#[mutants::skip]
+fn width_observation_fresh(observed_at: &std::time::Instant) -> bool {
+    observed_at.elapsed() < WIDTH_OBSERVATION_TRUST
+}
+
 /// Loads the image behind a cold slot and replays whatever the WAL
 /// holds above the image's watermark; hot slots pass through. On
 /// success the slot is hot, the stats are fresh, and `wal_seq`

src/registry/boot.rs

@@ -204,6 +204,7 @@ impl AppState {
                 .unwrap_or(DEFAULT_SEMANTIC_FLOOR)
                 .clamp(0.0, 1.0),
             cue_cache: Mutex::new(CueCache::default()),
+            observed_embed_width: Mutex::new(None),
             retrieval_cache: Mutex::new(retrieval_cache::RetrievalCache::new(
                 retrieval_cache_bytes,
             )),

src/registry/embeddings.rs

@@ -209,10 +209,43 @@ impl AppState {
     /// operator calls this after ingesting, so embedding spend stays
     /// intentional. Returns (newly embedded, total vectors), or `None`
     /// for an unknown context.
+    ///
+    /// Always pays for its own width probe when one is needed — see
+    /// [`AppState::auto_refresh_embeddings`] for the throttled variant
+    /// the auto-embed ticker uses instead. An explicit call (this one)
+    /// is rare and deliberate, often an operator diagnosing a stale
+    /// result, so it must reliably detect and heal a width change in
+    /// ONE call — the contract `tests/http_api/width_probe.rs` pins.
     pub fn refresh_embeddings(
         &self,
         name: &str,
         deadline: Deadline,
+    ) -> Option<Result<(usize, usize), String>> {
+        self.refresh_embeddings_inner(name, deadline, false)
+    }
+
+    /// The auto-embed ticker's variant of [`AppState::refresh_embeddings`]
+    /// (issue #677 item 2): identical, except its width probe is
+    /// skipped when a recent embed from ANY context already confirmed
+    /// the provider's current width (`provider_width_recently_confirmed`
+    /// — the width is the provider's property, not this context's). A
+    /// busy, gloss-stable context would otherwise pay one provider
+    /// round trip per flush tick forever; an explicit caller still gets
+    /// the unthrottled [`AppState::refresh_embeddings`] instead, so a
+    /// deliberate refresh always heals a width change in one call.
+    pub(crate) fn auto_refresh_embeddings(
+        &self,
+        name: &str,
+        deadline: Deadline,
+    ) -> Option<Result<(usize, usize), String>> {
+        self.refresh_embeddings_inner(name, deadline, true)
+    }
+
+    fn refresh_embeddings_inner(
+        &self,
+        name: &str,
+        deadline: Deadline,
+        throttle_probe: bool,
     ) -> Option<Result<(usize, usize), String>> {
         let Some(embedder) = self.0.embedder.clone() else {
             return Some(Err(
@@ -338,12 +371,28 @@ impl AppState {
         let mut fresh_width = width(&embedded_concepts).or_else(|| width(&embedded_labels));
         // Unchanged hashes embed nothing, which would leave the width
         // change of exactly this scenario — backend swap, no gloss
-        // edits — undetectable forever. One probe embedding per no-op
-        // refresh keeps that from hiding.
+        // edits — undetectable forever. A probe embedding keeps that
+        // from hiding, UNLESS this is the throttled ticker path AND a
+        // recent embed from any context (this one or another) already
+        // confirmed the provider is still producing `carried`'s width
+        // — the width is the provider's property, not this context's,
+        // so that confirmation is just as good as one bought here. An
+        // explicit caller (`throttle_probe: false`) always probes, so
+        // it reliably heals a width change in this one call. Both
+        // tables must be confirmed (whichever carry a width), matching
+        // the mismatch check just below: a probe confined to labels
+        // alone would miss a change confined to concepts, so a skip
+        // confined to labels alone must not happen either.
+        let width_confirmed = |carried: Option<usize>| {
+            carried.is_none_or(|w| self.provider_width_recently_confirmed(w))
+        };
         if failure.is_none()
             && !fresh_model
             && (carried_concepts_width.is_some() || carried_labels_width.is_some())
             && fresh_width.is_none()
+            && !(throttle_probe
+                && width_confirmed(carried_concepts_width)
+                && width_confirmed(carried_labels_width))
             && let Some((_, gloss)) = concepts.first().or_else(|| labels.first())
         {
             match self.timed_embed_for_refresh(embedder.as_ref(), &[gloss.as_str()], deadline) {
@@ -631,10 +680,36 @@ impl AppState {
     /// across the whole backfill. A provider failure partway persists
     /// what did land and reports the error — the next refresh continues
     /// from there instead of re-buying the same vectors.
+    ///
+    /// Always pays for its own width probe when one is needed — see
+    /// [`AppState::auto_refresh_passage_embeddings`] for the throttled
+    /// variant the auto-embed ticker uses instead, and
+    /// [`AppState::refresh_embeddings`]'s doc for why the split exists.
     pub fn refresh_passage_embeddings(
         &self,
         name: &str,
         deadline: Deadline,
+    ) -> Option<Result<PassageRefreshOutcome, String>> {
+        self.refresh_passage_embeddings_inner(name, deadline, false)
+    }
+
+    /// The auto-embed ticker's variant of
+    /// [`AppState::refresh_passage_embeddings`] — see
+    /// [`AppState::auto_refresh_embeddings`]'s doc for the throttle
+    /// this mirrors (issue #677 item 2).
+    pub(crate) fn auto_refresh_passage_embeddings(
+        &self,
+        name: &str,
+        deadline: Deadline,
+    ) -> Option<Result<PassageRefreshOutcome, String>> {
+        self.refresh_passage_embeddings_inner(name, deadline, true)
+    }
+
+    fn refresh_passage_embeddings_inner(
+        &self,
+        name: &str,
+        deadline: Deadline,
+        throttle_probe: bool,
     ) -> Option<Result<PassageRefreshOutcome, String>> {
         let Some(embedder) = self.0.embedder.clone() else {
             return Some(Err(
@@ -827,11 +902,17 @@ impl AppState {
             }
             // Unchanged hashes embed nothing, which would leave the width
             // change of exactly this scenario — backend swap, no passage
-            // edits — undetectable. One probe embedding per no-op refresh
-            // keeps it from hiding, matching the concept refresh.
+            // edits — undetectable. A probe embedding keeps it from
+            // hiding, matching the concept refresh — including the same
+            // ticker-only skip when a recent embed from any context
+            // already confirmed the provider is still producing
+            // `carried_width` (see `provider_width_recently_confirmed`'s
+            // doc: the width is the provider's property, not this
+            // context's). An explicit caller always probes.
             if failure.is_none()
                 && !fresh_model
-                && carried_width.is_some()
+                && carried_width
+                    .is_some_and(|w| !(throttle_probe && self.provider_width_recently_confirmed(w)))
                 && fresh_width.is_none()
                 && let Some(probe) = records
                     .iter()