avo: suite green; tests+28 fc=True docs=True debt+0

backend/database/conversations.py

@@ -1,4 +1,4 @@
 import copy
 import json
 import logging
 import uuid
@@ -898,7 +898,15 @@
 
     doc_level = doc_snapshot.to_dict().get('data_protection_level', 'standard')
     prepared_data = _prepare_conversation_for_write(update_data, uid, doc_level)
-    doc_ref.update(prepared_data)
+    try:
+        doc_ref.update(prepared_data)
+    except NotFound:
+        # The conversation was deleted between the existence read above and
+        # this commit. The contract of this function is to report a gone owner
+        # as False — not to raise — so callers like the pusher's private-cloud
+        # audio sync take their designed gone-owner path (stop syncing, release
+        # the audio budget) instead of logging an ERROR and retrying forever.
+        return False
     return True
 
 

backend/docs/listen_pusher_pipeline.mdx

@@ -158,6 +158,13 @@ beforehand races the pusher's own registration write, and losing that race in
 the other direction would delete a recording that had just become
 referenceable.
 
+When the private-cloud flush writes `audio_files` onto a conversation that a
+mid-session delete removes between the flush's existence read and its commit,
+`update_conversation` reports the owner as gone (returns `False`) instead of
+raising: the flush then takes the same stop-syncing path as any other deleted
+owner rather than surfacing `404 No document to update` as an ERROR on every
+remaining flush of the session.
+
 ## Capture-device provenance
 
 Every capture client with WebSocket-upgrade header support sends

backend/docs/operational/aac-decode-failure-reporting.md

@@ -0,0 +1,95 @@
+# AAC Decode Failures — typed failures, not silent native noise
+
+Date: 2026-09-01 · Scope: `utils/aac.py` (`AACDecoder.decode`) ·
+Guard tests: `tests/unit/test_aac_decode_failure_reporting.py`
+
+## What happened
+
+The GCP prod error feed (backend-listen) carried a family of
+`ERROR:libav.aac:*` signatures continuously across 2026-08-30/31 —
+combined ~130 events / 30 min at peaks:
+
+```
+ERROR:libav.aac:Channel element 1.7 is not allocated        (×12–17 / 30 min)
+ERROR:libav.aac:Reserved bit set.                           (×6–11 / 30 min)
+ERROR:libav.aac:Error decoding AAC frame header.            (×3–14 / 30 min)
+ERROR:libav.aac:Number of bands (…) exceeds limit (…)       (×11 / 30 min)
+ERROR:libav.aac:Number of scalefactor bands in group (…) …  (×5 / 30 min)
+```
+
+These are FFmpeg's **native codec diagnostics** for a frame its AAC decoder
+rejected, emitted while `AACDecoder.decode` held the frame. Reproduced
+locally against the pinned PyAV 12.0.0 by feeding the real decoder
+payload-scrambled and truncated ADTS frames: every corrupt shape raises
+`av.InvalidDataError`/`av.UndefinedError` **and** emits the corresponding
+native `libav.aac` line — the prod signatures are our decode loop rejecting
+corrupt/truncated client audio, frame after frame.
+
+## What was broken
+
+`AACDecoder.decode` caught `(EOFError, av.AVError)` and returned `b''`. Two
+consequences, one contract violation:
+
+1. **The receiver's decode-failure contract never fired for AAC.** The
+   listen receiver (`routers/listen/receiver.py`) already owns undecodable-
+   frame reporting: `_record_decode_failure` logs a per-frame warning with
+   the codec's own message, payload size, and streak, and the one-shot
+   `record_fallback(component='silent_mic', …, outcome='exhausted')` at 50
+   consecutive drops (1 s at the omi 20 ms frame cadence) — the contract
+   #11732 established when opus streams were dropping silently. All of it
+   hangs off the decoder *raising*. opuslib raises `OpusError`; the AAC
+   decoder swallowed, so a fully undecodable AAC stream recorded a whole
+   session with no transcript, no ring buffer, no mixed audio, and **no
+   fallback metric** — a fail-open branch with no operator signal (the
+   fallback-telemetry contract, `docs/agents/fallback-telemetry.md`).
+2. **The only trace left was the context-free native line.** FFmpeg's
+   `ERROR:libav.aac:…` carries no uid, session, codec name, payload size, or
+   streak — strictly less information than the warning the receiver would
+   have logged — and it duplicated that warning per frame once the receiver
+   path existed, polluting the error feed the Loop S sensor watches.
+
+## The fix
+
+- `AACDecoder.decode` now **raises `AACDecodeError`** (message = the FFmpeg
+  error verbatim, cause = the original `av.AVError`) for frames the codec
+  rejects, and still returns `b''` only for benign no-output input (empty
+  payload, encoder priming). The receiver's existing `except Exception →
+  _record_decode_failure('aac', …)` handles the rest — no receiver change.
+- `NativeDuplicateSuppressionFilter` on the `libav.aac` logger drops
+  FFmpeg's re-report **only while our own decode call is on the stack**
+  (thread-local flag, set/cleared in `finally`); every other `av` consumer's
+  native errors still flow.
+
+## Operator-visible after this lands
+
+- Per corrupt frame: `WARNING … Listen audio frame decode failed codec=aac
+  type=AACDecodeError bytes=N streak=M detail=<ffmpeg message>` (uid/session
+  context via the existing warning shape).
+- A fully undecodable stream: one `silent_mic` fallback metric per session —
+  the alertable signal that was missing.
+- The `ERROR:libav.aac:*` family should disappear from the error feed; any
+  residual native lines would indicate a decode outside this module (speech
+  profiles, transcode), which keeps its own context.
+
+## Regression coverage
+
+`tests/unit/test_aac_decode_failure_reporting.py` drives the real
+`AACDecoder` over real ADTS frames encoded in-process (real PyAV encoder →
+real codec context) and the real `ListenReceiver.receive_data` loop; only
+the websocket transport is scripted. Covers: corrupt/truncated frames raise
+the typed error with FFmpeg detail (mono and stereo, header-only fragments);
+clean frames decode; recovery after a corrupt frame and after a burst;
+native duplicate suppressed inside our decode window with a negative control
+proving the suppression is not vacuous; window-scope (an unrelated native
+error right after still logs) and thread-locality; receiver streak
+progression, warning shape, one-shot silent-mic fallback at the threshold,
+streak reset on recovery, interleaved corrupt/clean streams, and the
+`initialize_decoders` production wiring.
+
+Failure-Class: FC-typed-failure-collapsed-to-generic — instance fix; the
+class was canonized by #11487 (proactive lane) and applied at the live-STT
+boundary by #11732 (opus): a codec that already produces a typed failure
+must not have it collapsed into silence by the caller. Here the collapse
+happened one layer lower — the decoder itself swallowing `AVError` — so the
+receiver's whole reporting contract (built for exactly this failure shape)
+never engaged for AAC.

backend/docs/operational/soniox-typed-rejections.md

@@ -0,0 +1,65 @@
+# Soniox in-stream rejections: keep the type, split the severity
+
+Incident window: 2026-08-30/31 (backend-listen, Loop S sensor).
+`ERROR:utils.stt.soniox:Soniox streaming error:` was the #4 error signature
+(~52 events / 30 min) carrying three unrelated provider behaviors on one
+free-text line:
+
+| Frame | Typed reason | Owner | Severity |
+|---|---|---|---|
+| 400 `invalid_request` "No audio received" | `soniox_idle_timeout` | this session's VAD pattern | WARNING |
+| 402 `organization_balance_exhausted` | `soniox_account_state` | the provider/account | **ERROR** |
+| 413 `max_duration_reached` | `soniox_rotation` | documented protocol rotation | WARNING |
+
+## What was broken
+
+- Every in-stream error frame — including two shapes that are the protocol
+  answering how the session was used — logged at ERROR, so a fleet of
+  VAD-starved sockets was indistinguishable from a serving outage.
+- The socket laundered the typed frame into free-text `death_reason`
+  (`soniox error: 402 organization_balance_exhausted …`); every terminal funnel
+  (`terminate_live_stt_session`, the death monitor, the send path) collapsed it
+  to `connection_lost`, so `omi_live_stt_terminal_failures_total` carried no
+  cause distinction.
+- `_fallback_failure_reason` matched neither `exhausted` nor `balance`, filing
+  the 402 as `provider_5xx` in fallback telemetry.
+- `bounded_provider` did not know the live-path tokens `soniox` /
+  `deepgram_cloud`, so terminal-failure metrics reported `provider='unknown'`.
+- A 402 provider still **accepts connects** while refusing every stream:
+  mid-session failover moved each dying session to the next provider and the
+  session survived, so the terminal path that feeds the selection circuit never
+  ran, and each new session was handed straight back to the refusing provider.
+
+## The contract now
+
+- `soniox_death_reason(error_code, error_type)` bounds the provider's own typed
+  frame into the terminal vocabulary; unknown shapes degrade to
+  `connection_lost` rather than growing per-message cardinality.
+- The socket latches both the raw frame (`death_reason`, for logs) and the
+  bounded type (`typed_death_reason`); `GatedSTTSocket` proxies the typed
+  reason so the VAD gate cannot erase it.
+- `live_stt_terminal_reason(socket, fallback)` lets every terminal funnel
+  report the provider's type instead of its own vantage point.
+- Severity follows fault ownership: only a 402 account-state refusal stays at
+  ERROR (`Soniox streaming error:`); idle-timeout and rotation log
+  `Soniox stream closed:` at WARNING. Never mute a signature without
+  classifying fault origin first — see `ws-auth-rejection-severity.md` for the
+  same rule at the auth boundary.
+- Fleet evidence: `note_typed_provider_death` at the failover seam (and
+  `soniox_account_state` in the terminal path) opens the provider's
+  process-local selection circuit for one cooldown. Session-scoped reasons
+  deliberately do not — an idle timeout is this session's VAD pattern, not a
+  provider fault.
+- `_fallback_failure_reason` classifies `exhausted`/`balance` text as `quota`;
+  `bounded_provider` accepts the live-path provider tokens.
+
+## Regression coverage
+
+`tests/unit/test_soniox_typed_rejections.py` drives the real `SafeSonioxSocket`
+receive loop over the exact prod frame shapes, the real `ListenReceiver`
+failover and death-monitor paths, and the real terminal/send paths; only the
+process-global circuit opener is patched.
+
+Failure-Class: FC-typed-failure-collapsed-to-generic (instance fix; class
+canonized by #11487 in the proactive lane — the same collapse at the live-STT
+boundary).

backend/docs/operational/ws-auth-rejection-severity.md

@@ -0,0 +1,68 @@
+# WS Auth Rejection Severity — error-feed hygiene
+
+Date: 2026-08-31 · Scope: `utils/other/endpoints.py` (`_verify_ws_auth`) ·
+Guard tests: `tests/unit/test_ws_auth_rejection_logging.py`
+
+## What happened
+
+The GCP prod error feed (backend-listen) carried two of its top signatures
+for 16+ consecutive hours on 2026-08-30/31:
+
+```
+ERROR:utils.other.endpoints:WebSocket auth failed: code=4001 error=Token expired, 1788101015 < 1788114388   (×9–47 / 30 min)
+ERROR:utils.other.endpoints:WebSocket auth failed: code=4001 error=Certificate for key id 6ac9047f… not found.  (×1–34 / 30 min)
+```
+
+Both are **client-caused rejections the server handled correctly**:
+
+- `Token expired` — samples show the presented Firebase ID tokens were 3.7h,
+  7.5h and 18h past `exp` (tokens live 1h). Devices with suspended clocks or
+  long sleep/reconnect loops replaying a dead credential.
+- `Certificate for key id 6ac9047f… not found` — every sample across 14h
+  names the *same* key id. Verified against Google's currently served x509
+  set (`/robot/v1/metadata/x509/securetoken@system.gserviceaccount.com`) on
+  2026-08-31: that kid is **retired** — absent from the fresh cert list. A
+  client cohort minted before the key rotation presents tokens no server can
+  verify; note `google.oauth2.id_token.verify_token` checks the `kid` against
+  the cert set *before* the `exp` check, so this fires regardless of expiry
+  and for arbitrary freshness.
+
+The rejections (and their close codes — 4001 "refresh" / 4004 "re-login") are
+the auth protocol working. The **defect was classification**: every expected
+client-caused rejection was logged at ERROR, so the stale-client reconnect
+population was indistinguishable from a serving outage in the feed that pages
+humans — during the same window a *real* outage (the Modulate STT 5xx storm)
+had to be found inside it.
+
+## The rule (severity follows fault origin)
+
+In `_verify_ws_auth`, the narrow catch now routes logging through
+`_log_ws_auth_rejection(close_code, error)`:
+
+| Verify outcome | Fault origin | Severity | Close code |
+|---|---|---|---|
+| `InvalidIdTokenError` (incl. `ExpiredIdTokenError`, `RevokedIdTokenError`) | client — Firebase evaluated and refused the token | **WARNING** | unchanged (4001/4004/1008) |
+| `CertificateFetchError` | **server** — could not fetch Google's certs to even evaluate | **ERROR** | 4001 |
+| unexpected exception | server | ERROR | 1008 |
+
+Close codes and reasons are untouched — clients already get the right
+remediation hint. The warning message is `WebSocket auth rejected: …`
+(still carries code + error text for debugging); server faults keep
+`WebSocket auth failed: …`.
+
+## Applying the same rule elsewhere
+
+- Client-caused auth rejections should not be top error signatures. If a new
+  one appears, first establish fault origin from samples (stale `exp` deltas,
+  repeated retired `kid`s vs. cert-fetch/transport failures), then classify
+  severity accordingly — do not mute by signature-silencing.
+- The message-derived branches in `_get_ws_auth_close` (matching
+  `'expired'`/`'certificate'` in the text) exist for exceptions crossing this
+  boundary without a typed class; they do not change fault-origin
+  classification.
+- Related incident record and test fixtures:
+  `tests/unit/test_ws_auth_rejection_logging.py` replays the exact prod
+  message shapes.
+
+Failure-Class: FC-request-input-rejection-escapes-as-server-fault (instance
+fix; class canonized by #11853).

backend/routers/listen/receiver.py

@@ -1,4 +1,4 @@
 """Inbound listen WebSocket frames, audio decoding, and image assembly."""
 
 from __future__ import annotations
 
@@ -47,7 +47,9 @@
     flush_live_stt_buffer,
     live_stt_initialization_failure,
     live_stt_socket_is_dead,
+    live_stt_terminal_reason,
     live_stt_upstream_failure,
+    note_typed_provider_death,
     send_live_stt_audio,
     terminate_live_stt_session,
 )
@@ -223,7 +225,7 @@
         elif request.codec == 'lc3':
             self.lc3_decoder = _get_lc3().Decoder(self.host.lc3_frame_duration_us, request.sample_rate)
 
     async def _create_stt_socket(self, callback: Any, sample_rate: int, modulate_callback: Any = None) -> Any:
         keywords = self.host.vocabulary[:100] if self.host.vocabulary else []
         if self.host.stt_service == STTService.parakeet:
             socket, actual_service = await connect_stt_socket_with_fallback(
@@ -520,6 +522,12 @@
             self._stt_failed_providers.add(dead_provider)
         if len(self._stt_failed_providers) > MAX_STT_FAILOVERS:
             return False
+        # A provider-level typed rejection (Soniox 402 balance-exhausted) is
+        # fleet-level evidence the failover would otherwise swallow: the session
+        # survives on the next provider, so the terminal path that normally
+        # feeds the selection circuit never runs for it, and the NEXT session is
+        # handed straight back to the provider that refuses every stream.
+        note_typed_provider_death(self.stt_socket, dead_provider)
 
         service, language, model = get_stt_service_for_language(
             self.host.language,
@@ -588,7 +596,7 @@
                     self.host.request.websocket,
                     self.host.state,
                     failure=live_stt_upstream_failure(self._serving_provider()),
-                    reason='connection_lost',
+                    reason=live_stt_terminal_reason(socket, 'connection_lost'),
                     platform=self.host.client_device_context.platform,
                 )
                 return