From 18a8c4c08834069e57890572e3dcfb9b39ab385a Mon Sep 17 00:00:00 2001 From: Sina Setayesh Date: Sat, 29 Aug 2026 18:16:03 +1000 Subject: [PATCH 1/6] fix(bin): name what actually ended a watcher cycle "watcher: FAILED - cycle ended without an actionable reason" gives the operator nothing to act on. A watcher that exited cleanly, one that died leaving its lock behind, one another watcher replaced, and one that is still live but no longer beating all reach that sentence, and all need different responses. In the incident that prompted this it twice read as a false alarm when it was not, and cost a twelve-hour supervision gap. The typed prefix is unchanged, so existing consumers keep matching, and neither the verdict nor the exit status moves. Only the evidence improves: watcher: FAILED - cycle ended without an actionable reason: watcher pid=19771 exited 1 without delivering a wake (last beacon 3s ago, lock now: unheld) An arm that merely ATTACHED holds no handle on its watcher's exit status, so where the evidence is a real exit code it is read back from the lifecycle ledger the OWNING arm already wrote. The lookup is bound to the same watcher pid, the same recorded process identity, and a close no earlier than this arm attached, so neither a recycled pid nor an older cycle can be read as this one. It stays diagnostic: a missing or unreadable row degrades to the disposition text and never changes the verdict. Two details that are easy to get wrong and are covered by tests: - Liveness inside the evidence is identity-qualified rather than bare. A recycled pid is a live pid, and treating it as the watcher would name an unrelated process to the operator while suppressing the recorded exit code that is the only remaining evidence in exactly that case. - The ledger probe is rebuilt through the same transforms the ledger applied on the way in. lock_snapshot cleans each part and cycle_log_append cleans the whole composite again, and cut(1) over a pair is not cut(1) over each half, so a singly-truncated probe stops matching its own row once an identity is long enough - which a deep worktree path reaches. The pull warning and the turn-end block keep their verdicts, exit statuses and alarms exactly as they were; only their wording now separates a live identity-matched holder that has not beaten from nothing running at all, so one fault is not read as the other. Both read the holder through a wait-free lock check, because a guard whose job is to block a turn must not wait to phrase itself. Both regressions were checked against the code with their own fix removed. The mutant for the evidence clause reproduces the reported symptom exactly - a fresh beacon, a failed cycle, and nothing to act on - and the long-identity mutant loses the recorded exit while keeping the verdict, which is the precise shape of that defect. --- bin/fm-guard.sh | 7 ++ bin/fm-turnend-guard.sh | 16 +++- bin/fm-wake-lib.sh | 18 +++++ bin/fm-watch-arm.sh | 97 +++++++++++++++++++++- docs/turnend-guard.md | 1 + docs/verification/supervision.md | 42 ++++++++++ docs/watcher-continuity.md | 12 +++ tests/fm-guard-stale-banner.test.sh | 64 +++++++++++++++ tests/fm-turnend-guard.test.sh | 43 ++++++++++ tests/fm-watcher-lock.test.sh | 121 ++++++++++++++++++++++++++++ 10 files changed, 413 insertions(+), 8 deletions(-) diff --git a/bin/fm-guard.sh b/bin/fm-guard.sh index 21d6da3ed81..513d65899ed 100755 --- a/bin/fm-guard.sh +++ b/bin/fm-guard.sh @@ -200,6 +200,13 @@ if [ "$watcher_healthy" = false ]; then printf '● WATCHER DOWN - SUPERVISION IS OFF\n' if [ "$watcher_down_reason" = no-watcher ]; then watcher_cause=$(printf 'no live watcher process holds this home lock (last beat: %s)' "$beacon_desc") + elif live_holder=$(fm_watcher_live_holder_pid "$STATE" "$WATCH" "$FM_HOME"); then + # A live identity-matched holder with a stale beacon is a different fault + # from nothing running at all, and it is what a suspended host leaves + # behind. Naming it keeps the operator from reading a resume as a dead + # watcher; the verdict itself stays unchanged and still alarms. + watcher_cause=$(printf 'watcher pid %s holds this home lock but has not beaten (last beat: %s, grace %ss)' \ + "$live_holder" "$beacon_desc" "$GRACE") else watcher_cause=$(printf 'no watcher has a fresh beacon (last beat: %s, grace %ss)' "$beacon_desc" "$GRACE") fi diff --git a/bin/fm-turnend-guard.sh b/bin/fm-turnend-guard.sh index 4ab1f4728b9..8c89c8fba5f 100755 --- a/bin/fm-turnend-guard.sh +++ b/bin/fm-turnend-guard.sh @@ -199,7 +199,7 @@ if [ "$FM_SUP_WATCHER_FRESH" = true ] && fm_afk_daemon_owns_supervision "$STATE" fi block_stop() { - local afk x_mode reason rule + local afk x_mode reason rule live_holder holder_phrase afk=0 [ -e "$STATE/.afk" ] && afk=1 x_mode=0 @@ -207,15 +207,23 @@ block_stop() { reason=$("$SCRIPT_DIR/fm-supervision-instructions.sh" --afk "$afk" --x-mode "$x_mode" --repair-line 2>/dev/null \ || printf '%s\n' 'tasks in flight, no live watcher - repair missing watcher supervision according to the session-start operating block before ending the turn') rule='━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━' + # The block itself is unchanged: this only names WHICH fault is blocking, so a + # host that merely resumed is not reported as a watcher that died. The read is + # wait-free, because a guard that blocks a turn must not wait to do it. + if live_holder=$(fm_watcher_live_holder_pid "$STATE" "$WATCH" "$FM_HOME"); then + holder_phrase="watcher pid $live_holder holds this home lock but has not beaten" + else + holder_phrase='no live watcher holds this home lock' + fi { printf '●%s\n' "$rule" printf '● TURN WOULD END BLIND - SUPERVISION IS OFF\n' if [ "$FM_SUP_IN_FLIGHT" -gt 0 ]; then - printf '● %s task(s) in flight, but no live watcher holds this home lock (last beat: %s).\n' "$FM_SUP_IN_FLIGHT" "$FM_SUP_BEACON_DESC" + printf '● %s task(s) in flight, but %s (last beat: %s).\n' "$FM_SUP_IN_FLIGHT" "$holder_phrase" "$FM_SUP_BEACON_DESC" elif [ "$FM_SUP_SOURCES" -gt 0 ]; then - printf '● %s process-event source(s) registered, but no live watcher holds this home lock (last beat: %s).\n' "$FM_SUP_SOURCES" "$FM_SUP_BEACON_DESC" + printf '● %s process-event source(s) registered, but %s (last beat: %s).\n' "$FM_SUP_SOURCES" "$holder_phrase" "$FM_SUP_BEACON_DESC" else - printf '● X-mode relay polling needs supervision, but no live watcher holds this home lock (last beat: %s).\n' "$FM_SUP_BEACON_DESC" + printf '● X-mode relay polling needs supervision, but %s (last beat: %s).\n' "$holder_phrase" "$FM_SUP_BEACON_DESC" fi if [ "$CLAUDE_MODE" -eq 1 ]; then printf '● The Stop-owned auto-arm did not claim this home either, so recovery is NOT already under way.\n' diff --git a/bin/fm-wake-lib.sh b/bin/fm-wake-lib.sh index 7964dab4595..67fc7a48408 100755 --- a/bin/fm-wake-lib.sh +++ b/bin/fm-wake-lib.sh @@ -126,6 +126,24 @@ fm_watcher_lock_matches_pid() { FM_WATCHER_HEALTHY_PID= FM_WATCHER_HEALTHY_IDENTITY= +# fm_watcher_live_holder_pid [home] +# Echo the pid of a LIVE, identity-matched holder of this home's watcher lock, or +# fail when there is none. This answers "is a watcher process still there", NOT +# "is it still cycling" - a holder whose beacon is stale still qualifies, which is +# exactly the distinction the callers below need. +# +# Deliberately wait-free: it reads the lock and re-proves the recorded identity, +# and never sleeps or watches the beacon. It is used only to make an operator +# message name the right fault, so it must not add latency to a guard whose job +# is to block a turn, and it never changes a verdict. +fm_watcher_live_holder_pid() { + local state=$1 watch_path=$2 home=${3:-$FM_HOME} pid + pid=$(cat "$state/.watch.lock/pid" 2>/dev/null || true) + fm_pid_alive "$pid" || return 1 + fm_watcher_lock_matches_pid "$state" "$watch_path" "$pid" "$home" || return 1 + printf '%s' "$pid" +} + fm_watcher_healthy() { local state=$1 watch_path=$2 grace=${3:-${FM_GUARD_GRACE:-300}} home=${4:-$FM_HOME} lockdir beat pid identity age FM_WATCHER_HEALTHY_PID= diff --git a/bin/fm-watch-arm.sh b/bin/fm-watch-arm.sh index d134f519402..ceb5c00031e 100755 --- a/bin/fm-watch-arm.sh +++ b/bin/fm-watch-arm.sh @@ -29,9 +29,11 @@ # watcher: attached pid= (beacon s) - a live+fresh successor holds the lock; # this arm attaches and follows it # watcher: FAILED - no live watcher with a fresh beacon - could not confirm one -# watcher: FAILED - cycle ended without an actionable reason +# watcher: FAILED - cycle ended without an actionable reason: # - a clean cycle ended with no wake and no -# verified healthy successor +# verified healthy successor; the evidence +# clause names what ended it (see +# unexplained_cycle_evidence below) # It NEVER reports started/attached/healthy off a stale beacon or a dead/reused pid: a # stale-beacon or dead-pid holder either self-heals (the fresh child steals the # dead lock per the singleton self-eviction/steal path and is confirmed) or this @@ -269,8 +271,95 @@ wait_for_healthy_successor() { done } +# An arm that only ATTACHED to a watcher holds no handle on that watcher's exit +# status, so a cycle that ends without a delivered wake is silent from here even +# though the arm that OWNED the child classified it in the lifecycle ledger. Read +# that classification back rather than reporting nothing: this is the difference +# between "the cycle ended" and "the watcher exited 1". +# +# The row is bound to this exact cycle - same watcher pid AND the same recorded +# process identity in its lock snapshot, closed no earlier than this arm attached +# - so neither a recycled pid nor an older cycle of the same watcher can be read +# as this one. The ledger stays diagnostic: a missing or unreadable row degrades +# to the disposition text below and never changes the verdict. +# +# The wanted field is rebuilt through the SAME transforms the ledger applied on +# the way in - lock_snapshot cleans each part, then cycle_log_append cleans the +# whole composite again - because cut(1) applied once to a pair is not the same +# as applied to each half. Comparing a singly-truncated probe against a doubly +# truncated row silently stops matching once an identity is long enough, which is +# reachable with a deep watcher path. +owner_recorded_exit() { + local pid=$1 identity=$2 want + [ -f "$CYCLE_LOG" ] || return 1 + want=$(cycle_clean_field "$(printf 'pid:%s|identity:%s' \ + "$(cycle_clean_field "${pid:-none}")" "$(cycle_clean_field "${identity:-none}")")") + awk -F'\t' -v pid="watcher_pid=$pid" -v want="lock_before=$want" \ + -v since="$cycle_started_at" ' + $2 == pid && $3 == "origin=started" && $10 == want { + ended = $5; sub(/^ended_at=/, "", ended) + if (ended + 0 < since + 0) next + code = $6; sub(/^exit_code=/, "", code) + sig = $7; sub(/^signal=/, "", sig) + if (code == "unknown") next + found = (sig == "none") ? "exited " code : "was killed by " sig + } + END { if (found != "") print found } + ' "$CYCLE_LOG" 2>/dev/null | grep . || return 1 +} + +# A recycled pid is a live pid, so bare liveness cannot stand in for "the watcher +# is still running": it would name an unrelated process to the operator and, by +# claiming the watcher never exited, suppress the recorded exit code that is the +# only remaining evidence in exactly that case. Re-prove the identity the cycle +# recorded, the way every other holder check in this codebase does, and compare +# it through the same cleaning the ledger uses so length cannot decide the answer. +cycle_watcher_still_live() { + local pid=$1 identity=$2 current + fm_pid_alive "$pid" || return 1 + [ -n "$identity" ] || return 0 + current=$(cycle_clean_field "$(fm_pid_identity "$pid" 2>/dev/null || true)") + [ -n "$current" ] && [ "$current" = "$identity" ] +} + +# Name what actually ended a cycle that produced no reason line. "Cycle ended +# without an actionable reason" alone is unactionable: a watcher that exited +# cleanly, one that died leaving its lock, one another watcher replaced, and one +# that is still live but no longer beating all reach here and need different +# responses. The typed prefix is unchanged so existing consumers still match it, +# and the verdict and exit status are untouched - only the operator's evidence +# improves. +unexplained_cycle_evidence() { + local pid=$cycle_watcher_pid holder age owner_exit clean_identity + age=$(fm_path_age "$BEAT") + case "$pid" in ''|*[!0-9]*) printf 'watcher identity was never established'; return 0 ;; esac + holder=$(cat "$WATCH_LOCK/pid" 2>/dev/null || true) + clean_identity=$(cycle_clean_field "$cycle_watcher_identity") + if ! cycle_watcher_still_live "$pid" "$clean_identity" \ + && owner_exit=$(owner_recorded_exit "$pid" "$clean_identity"); then + printf 'watcher pid=%s %s without delivering a wake (last beacon %ss ago, lock now: %s)' \ + "$pid" "$owner_exit" "$age" "${holder:-unheld}" + return 0 + fi + if cycle_watcher_still_live "$pid" "$clean_identity"; then + if [ "$holder" = "$pid" ]; then + printf 'watcher pid=%s is still live and holds this home lock, but its beacon has not advanced for %ss' "$pid" "$age" + else + printf 'watcher pid=%s is still live but no longer holds this home lock (lock now: %s)' "$pid" "${holder:-unheld}" + fi + return 0 + fi + if [ -z "$holder" ]; then + printf 'watcher pid=%s exited and released this home lock without recording a delivered wake (last beacon %ss ago)' "$pid" "$age" + elif [ "$holder" = "$pid" ]; then + printf 'watcher pid=%s died leaving its own lock behind (last beacon %ss ago)' "$pid" "$age" + else + printf 'watcher pid=%s exited and this home lock moved to pid %s (last beacon %ss ago)' "$pid" "$holder" "$age" + fi +} + fail_unexplained_cycle() { - echo "watcher: FAILED - cycle ended without an actionable reason" + echo "watcher: FAILED - cycle ended without an actionable reason: $(unexplained_cycle_evidence)" return 1 } @@ -530,7 +619,7 @@ owned_child_finished() { cycle_log_append "$rc" "$signal" "$reason_type" none print_watch_output "$child_out" if ! grep -q '^watcher: FAILED' "$child_out" 2>/dev/null; then - echo "watcher: FAILED - watcher cycle exited $rc without an actionable reason" + echo "watcher: FAILED - watcher cycle exited $rc without an actionable reason: $(unexplained_cycle_evidence)" fi rm -f "$child_out" 2>/dev/null || true child= diff --git a/docs/turnend-guard.md b/docs/turnend-guard.md index 41b97b6adf0..75b72f8f16f 100644 --- a/docs/turnend-guard.md +++ b/docs/turnend-guard.md @@ -168,6 +168,7 @@ That warning uses `bin/fm-supervision-instructions.sh --repair-line`, so it alwa ## Regression coverage `tests/fm-turnend-guard.test.sh` covers the predicate, main and secondmate primary scope, child-worktree exclusion, `FM_HOME` and `FM_STATE_OVERRIDE` precedence, the live-lock and fresh-beacon guard predicate, the cooperative `--claude` open-generation claim wait, monotonic failed-epoch progression, bounded attended fail-open, post-alarm continuation suppression, positive recovery reset, generation and legacy claim cases that must block or clear instead of allowing a blind stop, away-mode daemon ownership between watcher cycles and over a watcher lock left behind by an exited watcher, plus its dead, pid-reused, absent, stale-beacon, and away-mode-off negatives, Pi logical-run latching, missing-`jq` behavior, all five primary registrations, Grok native and legacy selection, typed field precedence, malformed input, and exactly-one-path safety. +It also covers the blocking banner's separation of a live unbeaten holder from an absent watcher, with both legs still blocking. `tests/fm-guard-stale-banner.test.sh` covers the pull-guard predicate, including the persistent-model fresh-leftover-beacon negative control, the auto-arm model's healthy fresh-beacon-without-a-watcher case and stale-beacon alarm, and the extension model's live-watcher path, ownership-qualified fresh hand-off, held-lock failures, independently broken ownership signals, stale-beacon alarm, queued-wake warning, and Pi and pi-signed harness routing. It also covers true-reason banner wording and reason-keyed episode dedup surviving a beacon mtime change. `tests/fm-cursor-primary.test.sh` covers the Cursor park end to end over real processes with no harness installed: each tracked Claude-shaped entrypoint standing down on a Cursor payload, both follow-up sources, the bounded repair nag and its reset, the nested loop bounds, supersession, away-mode and lock-ownership inertness, Pi-host stand-down without Cursor identity and continued parking when `PI_CODING_AGENT` leaks alongside `CURSOR_AGENT` or `CURSOR_INVOKED_AS`, child-worktree exclusion, and that the adapter never exits 2. diff --git a/docs/verification/supervision.md b/docs/verification/supervision.md index 41ed77fad30..857d77dc122 100644 --- a/docs/verification/supervision.md +++ b/docs/verification/supervision.md @@ -505,6 +505,48 @@ tests/fm-claude-stop-autoarm.test.sh tests/fm-turnend-guard.test.sh ``` +### Cycle-end evidence and holder-naming banners, 2026-08-29 + +```text +macOS 27.0 (26A5421a) arm64 +GNU bash 5.3.9(1)-release +ShellCheck 0.11.0 (pinned) +base: origin/main f66be0f +``` + +Both regression cases were checked against the code with their own fix removed, because a case that cannot fail proves nothing. + +The cycle-end evidence, mutant = the evidence clause reverted to the bare upstream sentence: + +```text +control -> ok - an attached arm reports the exit status its owning arm recorded +mutant -> not ok - attached arm did not report the owner-recorded exit: + watcher: attached pid=96760 (beacon 0s) + watcher: FAILED - cycle ended without an actionable reason +``` + +That mutant output is the reported symptom itself: a fresh beacon, a failed cycle, and nothing to act on. + +The ledger probe at a long identity, mutant = the singly-truncated probe restored: + +```text +control -> ok - the owner-recorded exit is still found when the identity is long +mutant -> not ok - a long identity lost the owner-recorded exit: + watcher: FAILED - cycle ended without an actionable reason: + watcher pid=15635 exited and released this home lock without recording a delivered wake +``` + +The mutant does not lose the verdict, only the exit code - it falls back to the generic disposition text in exactly the case where the recorded exit is the only evidence left. +The fixture asserts its own identity length before relying on it, so it cannot pass vacuously on a short path. + +### Beacon freshness under host suspend - attempted and deferred + +The second half of the reported problem, that `state/.last-watcher-beat` can read fresh while supervision is already dead and stale while a watcher is merely suspended, was attempted on this branch and deliberately deferred rather than shipped. +The approach - separating a suspended watcher from a wedged one by watching the beacon for a bounded window instead of inferring from its age - is sound and reproduces the fault, but bounding those windows introduced five regressions of its own across four review rounds, each fix bounding one window and opening another. +Two of them broke guarantees the change itself had stated: callers that declared no budget stopped keeping their previous timing, and a receipt could claim a full observation that was never spent, which let a caller skip the one observation the design promised. +The pattern is the finding, and it is recorded here rather than lost: adding timed windows to a supervision path that had none, measured on a clock that can move underneath them in both directions, is the same class of fault the work set out to fix. +No part of that machinery is present in this change; the evidence above covers only what ships. + ## Wedge-alarm channels The two real notification channels were bounded manually on 2026-07-10 on macOS 26.5.2 with Herdr 0.7.3. diff --git a/docs/watcher-continuity.md b/docs/watcher-continuity.md index 537a236d842..613364678d2 100644 --- a/docs/watcher-continuity.md +++ b/docs/watcher-continuity.md @@ -91,6 +91,16 @@ An attached arm follows verified identity-matched successors and resolves the sa Before releasing its singleton lock after printing an actionable reason, the watcher records that reason with its PID and process identity in `state/.watch-deliveries.log`. A matching PID and identity lets an attached arm report the delivered reason and exit zero even after its durable wake was handled and acknowledged, while an unrelated queue producer or a recycled PID cannot satisfy the match. Only a cycle with no matching delivery record emits `watcher: FAILED - cycle ended without an actionable reason` and exits nonzero. +That line now carries an evidence clause naming what actually ended the cycle, because the bare sentence is unactionable: a watcher that exited cleanly, one that died leaving its lock behind, one another watcher replaced, and one that is still live but no longer beating all reach it and all need different responses. +The typed prefix is unchanged, so anything already matching on it keeps matching, and neither the verdict nor the exit status moves - only the operator's evidence improves. +An arm that merely ATTACHED holds no handle on its watcher's exit status, so where the evidence is a real exit code it is read back from the lifecycle ledger the OWNING arm already wrote. +That lookup is bound to the same watcher PID, the same recorded process identity, and a close no earlier than this arm attached, so neither a recycled PID nor an older cycle of the same watcher can be read as this one. +It stays diagnostic: a missing or unreadable row degrades to the disposition text and never changes the verdict. +Liveness inside that evidence is identity-qualified rather than bare, because a recycled PID is a live PID and would otherwise name an unrelated process to the operator while suppressing the recorded exit code that is the only remaining evidence in exactly that case. +The ledger probe is rebuilt through the same transforms the ledger applied on the way in, since a single truncation of a composite is not the same as truncating each half, and a singly-truncated probe silently stops matching its own row once an identity is long enough - which a deep worktree path reaches. + +`bin/fm-guard.sh`'s pull warning and `bin/fm-turnend-guard.sh`'s turn-end block keep their verdicts, their exit statuses, and their alarms exactly as they were; only their banner wording now separates a live identity-matched holder that has not beaten from nothing running at all, so an operator does not read one fault as the other. +Both read the holder through a deliberately wait-free lock check that never sleeps and never watches the beacon, because a guard whose job is to block a turn must not wait in order to phrase itself. The arm layer appends one tab-separated record per observed cycle to `state/.watch-cycle-exits.log`. Each record includes arm and watcher PIDs, start and end timestamps, exit code and signal, classified reason, beacon age, lock identity before and after close, and successor disposition. @@ -106,6 +116,8 @@ Only the watcher process touches `state/.last-watcher-beat`; no helper process c The same suite covers ordinary same-process session replacement for `/new`, `/resume`, `/fork`, and reload, same-instance shutdown-plus-start, automatic re-arm before any model turn, a fresh extension-module rebind carrying all in-flight actionable closes exactly once, stale prior-generation callbacks, repeated transitions with exactly one live cycle, disappearance of the shutting-down refusal after a valid replacement activates, and terminal quit still refusing late rearm. `tests/fm-watch-arm.test.sh` covers durable queue replay, real remote parent-replies ingestion into the authoritative status log, decision-only OPEN DECISIONS recovery, interrupted handling replay, generation-bound acknowledgement, a persistent live successor after recovery, a watcher close inside the handling window that must leave the printed acknowledgement valid, and the self-healing moved-generation acknowledgement that consumes its handled rows and names its remedy. `tests/fm-watch-recovery-loop.test.sh` covers the once-per-generation announcement bound with the real Pi extension against a refused handling handshake, and a handling successor that must surface a real crew event instead of going blind. +`tests/fm-watcher-lock.test.sh` covers the cycle-end evidence on two paired fixtures: an attached arm whose watcher exits nonzero must report the exit its owning arm recorded rather than the bare unactionable sentence, and the same two-arm fixture run from a deliberately deep path must still find that row once the identity is long enough for the ledger's two truncations to disagree. +`tests/fm-guard-stale-banner.test.sh` and `tests/fm-turnend-guard.test.sh` each pair a live unbeaten holder against nothing running at all, with both legs still warning and still blocking respectively, so the wording can change without the verdict moving. `tests/fm-watcher-lock.test.sh` covers verified-successor attach, recovery publication before stale-lock removal, the typed self-eviction failure, bounded and successor-linked lifecycle rows, and a SIGSTOP counterfactual that distinguishes a live PID from a stale beacon before classifying termination. `tests/fm-subagent-pretool-check.test.sh` proves Claude retains only the non-status Bash seatbelts. `tests/fm-claude-stop-autoarm.test.sh` covers the auto-arm's scope, stale and live session owners, unchanged AFK and need boundaries, single-flight, bounded failure retries, benign live-watcher cycle ends, one-notice failure episodes, and exit-2 translation. diff --git a/tests/fm-guard-stale-banner.test.sh b/tests/fm-guard-stale-banner.test.sh index 4171301f6c6..f5d9bf84132 100755 --- a/tests/fm-guard-stale-banner.test.sh +++ b/tests/fm-guard-stale-banner.test.sh @@ -419,6 +419,69 @@ test_persistent_no_watcher_episode_survives_beacon_touch() { pass "fm-guard stale banner: a no-watcher episode survives a beacon mtime change" } +# The third banner cause, and the one a suspended host leaves behind: a live, +# identity-matched holder of THIS home's watcher lock whose beacon has gone stale +# (docs/watcher-continuity.md). +# The verdict deliberately still alarms and the guard still exits 0 because it +# warns and never blocks; only the wording separates this from nothing running at +# all. +# The two negative controls run the same fixture with the holder dead and with +# the lock genuinely unheld, so all three causes are proven distinct rather than +# merely present. +test_persistent_stale_beacon_banner_separates_all_three_causes() { + local dir home out status pid + dir=$(make_guard_case persistent-stale-live-holder) + home=$(case_home "$dir") + sleep 60 & + pid=$! + record_live_watcher "$dir" "$pid" || fail "could not record the live watcher for the stale-beacon holder" + touch -t 200001010000 "$home/state/.last-watcher-beat" + out=$(run_guard_case "$dir") + status=$? + kill "$pid" 2>/dev/null || true + wait "$pid" 2>/dev/null || true + [ "$status" -eq 0 ] \ + || fail "the watcher-down banner must warn without blocking, got exit status $status: $out" + [ "$(count_text "$out" "WATCHER DOWN - SUPERVISION IS OFF")" -eq 1 ] \ + || fail "a live holder with a stale beacon must still alarm exactly once: $out" + assert_contains "$out" "watcher pid $pid holds this home lock but has not beaten" \ + "the stale-beacon banner must name the live holder pid that has not beaten" + assert_not_contains "$out" "no watcher has a fresh beacon" \ + "a live holder must not be reported as nothing having a fresh beacon" + assert_not_contains "$out" "no live watcher process holds this home lock" \ + "a live holder must not be reported as a missing watcher process" + + # Same fixture, holder dead: the pre-existing stale-beacon wording, unchanged. + dir=$(make_guard_case persistent-stale-dead-holder) + home=$(case_home "$dir") + sleep 60 & + pid=$! + record_live_watcher "$dir" "$pid" || fail "could not record the watcher lock for the dead holder" + kill "$pid" 2>/dev/null || true + wait "$pid" 2>/dev/null || true + touch -t 200001010000 "$home/state/.last-watcher-beat" + out=$(run_guard_case "$dir") + [ "$(count_text "$out" "WATCHER DOWN - SUPERVISION IS OFF")" -eq 1 ] \ + || fail "a stale beacon with no live holder must still alarm exactly once: $out" + assert_contains "$out" "no watcher has a fresh beacon" \ + "a stale beacon with no live holder must keep the stale-beacon wording" + assert_not_contains "$out" "holds this home lock but has not beaten" \ + "a dead holder must not be reported as a live holder that has not beaten" + + # Genuinely unheld lock with a fresh beacon: the no-watcher wording, unchanged. + dir=$(make_guard_case persistent-stale-unheld-lock) + home=$(case_home "$dir") + touch "$home/state/.last-watcher-beat" + out=$(run_guard_case "$dir") + [ "$(count_text "$out" "WATCHER DOWN - SUPERVISION IS OFF")" -eq 1 ] \ + || fail "a genuinely unheld lock must still alarm exactly once: $out" + assert_contains "$out" "no live watcher process holds this home lock" \ + "a genuinely unheld lock must keep the no-watcher wording" + assert_not_contains "$out" "holds this home lock but has not beaten" \ + "an unheld lock must not be reported as a live holder that has not beaten" + pass "fm-guard stale banner: the banner separates a live unbeaten holder from both other causes" +} + # The send-time false alarm this suite exists to pin: on a Pi primary the watcher # process is torn down and respawned by the extension on every actionable wake, so # a guarded command that lands in a hand-off sees a fresh beacon and an unheld lock @@ -697,6 +760,7 @@ test_autoarm_stale_beacon_alarms_with_correct_reason test_autoarm_stale_episode_is_stable test_persistent_no_watcher_banner_names_missing_process test_persistent_no_watcher_episode_survives_beacon_touch +test_persistent_stale_beacon_banner_separates_all_three_causes test_fresh_beacon_without_live_watcher_stays_alarm test_x_mode_without_live_watcher_stays_alarm test_healthy_recovery_rearms_next_stale_episode diff --git a/tests/fm-turnend-guard.test.sh b/tests/fm-turnend-guard.test.sh index f19e12adb70..cc2d5f03e9f 100755 --- a/tests/fm-turnend-guard.test.sh +++ b/tests/fm-turnend-guard.test.sh @@ -353,6 +353,48 @@ test_hook_blocks_with_live_lock_and_stale_beacon() { pass "fm-turnend-guard: blocks on a live watcher lock with an ancient beacon" } +# The blocking banner names the actual fault. A live, identity-matched holder of +# this home's watcher lock whose beacon has gone stale (what a suspended host +# leaves behind) is not a missing watcher process, and sending the operator +# looking for one hides the pid they must inspect or stop. +# The verdict is unchanged: both legs still block with exit 2, so the alarm is +# proven not to have been softened to make the wording true. +test_hook_block_banner_names_a_live_unbeaten_holder() { + local dir pid identity out status + dir=$(make_primary_dir "$TMP_ROOT/hook-banner-live-unbeaten") + : > "$dir/state/task1.meta" + sleep 60 & + pid=$! + identity=$(watcher_identity "$dir" "$pid") || { + kill "$pid" 2>/dev/null || true + wait "$pid" 2>/dev/null || true + fail "could not identify the live unbeaten watcher holder" + } + record_watcher_lock "$dir" "$pid" "$identity" + touch -t 202001010000 "$dir/state/.last-watcher-beat" + out=$(run_hook "$dir" false); status=$? + kill "$pid" 2>/dev/null || true + wait "$pid" 2>/dev/null || true + expect_code 2 "$status" "a live unbeaten holder must still block the turn" + assert_contains "$out" "watcher pid $pid holds this home lock but has not beaten" \ + "the block banner must name the live holder pid that has not beaten" + assert_not_contains "$out" "no live watcher holds this home lock" \ + "a live holder must not be reported as a missing watcher process" + + # Negative control on the same fixture shape: no holder at all keeps the + # absent-watcher wording, and still blocks. + dir=$(make_primary_dir "$TMP_ROOT/hook-banner-no-holder") + : > "$dir/state/task1.meta" + touch -t 202001010000 "$dir/state/.last-watcher-beat" + out=$(run_hook "$dir" false); status=$? + expect_code 2 "$status" "an absent watcher must still block the turn" + assert_contains "$out" "no live watcher holds this home lock" \ + "an unheld lock must keep the absent-watcher wording" + assert_not_contains "$out" "holds this home lock but has not beaten" \ + "an unheld lock must not be reported as a live holder that has not beaten" + pass "fm-turnend-guard: the block banner separates a live unbeaten holder from an absent one" +} + test_hook_blocks_when_unhealthy_in_primary() { local dir out status dir=$(make_primary_dir "$TMP_ROOT/hook-block") @@ -1915,6 +1957,7 @@ test_hook_blocks_when_dead_lock_has_fresh_beacon test_hook_silent_with_live_lock_and_fresh_beacon test_hook_non_claude_health_ignores_claude_budget_contention test_hook_blocks_with_live_lock_and_stale_beacon +test_hook_block_banner_names_a_live_unbeaten_holder test_hook_blocks_when_unhealthy_in_primary test_hook_blocks_from_fm_home_state test_hook_x_mode_reason_sources_cadence diff --git a/tests/fm-watcher-lock.test.sh b/tests/fm-watcher-lock.test.sh index 77fd4fbcca3..77dc0621743 100755 --- a/tests/fm-watcher-lock.test.sh +++ b/tests/fm-watcher-lock.test.sh @@ -836,6 +836,125 @@ test_arm_fails_loud_when_no_fresh_watcher_confirmable() { pass "arm reports FAILED and exits non-zero when no fresh watcher can be confirmed" } + +# An attached arm holds no handle on its watcher's exit status, so a watcher that +# ends nonzero - killed, or bailing out of its own startup - used to reach the +# operator as a bare "cycle ended without an actionable reason" with a perfectly +# fresh beacon and nothing to act on. The owning arm classified that exit in the +# lifecycle ledger, and the attached arm must report it. +test_attached_arm_reports_the_owner_recorded_exit() { + local dir state fakebin ownerout attachout ownerpid attachpid wpid status i + dir=$(make_case attached-owner-exit) + state="$dir/state" + fakebin="$dir/fakebin" + ownerout="$dir/owner-arm.out" + attachout="$dir/attached-arm.out" + mark_pr_check_migration_complete "$state" + PATH="$fakebin:$PATH" FM_HOME="$dir" FM_STATE_OVERRIDE="$state" FM_POLL=1 FM_SIGNAL_GRACE=1 FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH_ARM" > "$ownerout" & + ownerpid=$! + i=0 + while [ "$i" -lt 100 ]; do + grep -qF 'watcher: started pid=' "$ownerout" 2>/dev/null && break + sleep 0.1 + i=$((i + 1)) + done + wpid=$(cat "$state/.watch.lock/pid" 2>/dev/null || true) + grep -qF "watcher: started pid=$wpid" "$ownerout" || fail "owning arm did not start a watcher" + + PATH="$fakebin:$PATH" FM_HOME="$dir" FM_STATE_OVERRIDE="$state" FM_ARM_ATTACH_POLL=0.1 \ + FM_ARM_CONFIRM_TIMEOUT=5 "$WATCH_ARM" > "$attachout" & + attachpid=$! + i=0 + while [ "$i" -lt 100 ]; do + grep -qF "watcher: attached pid=$wpid" "$attachout" 2>/dev/null && break + sleep 0.1 + i=$((i + 1)) + done + grep -qF "watcher: attached pid=$wpid" "$attachout" || fail "second arm did not attach to the running watcher" + + # A watcher that ends without delivering, while its beacon is still well inside + # grace: the shape the suspend fix deliberately does not cover. + kill -TERM "$wpid" 2>/dev/null || fail "could not terminate the watcher" + wait_for_exit "$ownerpid" 200 >/dev/null 2>&1 || true + wait_for_exit "$attachpid" "$ARM_FAIL_EXIT_POLLS" + status=$? + [ "$status" -ne 0 ] && [ "$status" -ne 124 ] || fail "attached arm did not fail after a nonzero watcher exit (status $status)" + grep -qF "cycle ended without an actionable reason: watcher pid=$wpid exited 1 without delivering a wake" "$attachout" \ + || fail "attached arm did not report the owner-recorded exit: $(cat "$attachout")" + pass "an attached arm reports the exit status its owning arm recorded" +} + +# The same ledger lookup at a LONG identity. The ledger writes its lock snapshot +# through cycle_clean_field TWICE - once per part in lock_snapshot, then once +# over the whole "pid:|identity:" composite in cycle_log_append - and +# cut(1) applied to a pair is not cut(1) applied to each half. A probe truncated +# only once therefore stops matching its own row past roughly a 497-character +# identity, and the operator silently drops from the recorded exit code back to +# the generic disposition text in exactly the case where the exit code is the +# only evidence left. +# +# A deep watcher path is how that length is reached in practice: this home runs +# out of pooled worktrees whose paths are already long, and the identity carries +# the full command. This case runs the SAME two-arm fixture as the one above from +# a deliberately deep copy, so it differs from it by path length alone. +test_owner_recorded_exit_survives_a_long_identity() { + local dir state fakebin deep bin ownerout attachout ownerpid attachpid wpid status i identity + dir=$(make_case long-identity) + state="$dir/state" + fakebin="$dir/fakebin" + mark_pr_check_migration_complete "$state" + + # ~470 characters of path, so lstart plus the command clears the 497-character + # point where the two truncations start to disagree. + deep="$dir" + for i in 1 2 3 4 5 6 7 8 9 10 11 12; do + deep="$deep/nested-worktree-path-segment-$i-padding" + done + bin="$deep/bin" + mkdir -p "$bin" || fail "could not build the deep fixture path" + cp "$ROOT"/bin/*.sh "$bin/" 2>/dev/null || fail "could not stage the deep bin" + [ -x "$bin/fm-watch-arm.sh" ] || fail "deep bin is missing the arm" + + PATH="$fakebin:$PATH" FM_HOME="$dir" FM_STATE_OVERRIDE="$state" FM_POLL=1 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$bin/fm-watch-arm.sh" > "$dir/owner.out" 2>&1 & + ownerpid=$! + ownerout="$dir/owner.out" + i=0 + while [ "$i" -lt 150 ]; do + grep -qF 'watcher: started pid=' "$ownerout" 2>/dev/null && break + sleep 0.1 + i=$((i + 1)) + done + wpid=$(cat "$state/.watch.lock/pid" 2>/dev/null || true) + grep -qF "watcher: started pid=$wpid" "$ownerout" || fail "deep-path owning arm did not start a watcher: $(cat "$ownerout")" + + # The fixture is only meaningful if it actually produced a long identity. + identity=$(cat "$state/.watch.lock/pid-identity" 2>/dev/null || true) + [ "${#identity}" -gt 497 ] \ + || fail "fixture did not reach a long identity (${#identity} chars); the truncation case would pass vacuously" + + PATH="$fakebin:$PATH" FM_HOME="$dir" FM_STATE_OVERRIDE="$state" FM_ARM_ATTACH_POLL=0.1 \ + FM_ARM_CONFIRM_TIMEOUT=5 "$bin/fm-watch-arm.sh" > "$dir/attached.out" 2>&1 & + attachpid=$! + attachout="$dir/attached.out" + i=0 + while [ "$i" -lt 150 ]; do + grep -qF "watcher: attached pid=$wpid" "$attachout" 2>/dev/null && break + sleep 0.1 + i=$((i + 1)) + done + grep -qF "watcher: attached pid=$wpid" "$attachout" || fail "deep-path second arm did not attach: $(cat "$attachout")" + + kill -TERM "$wpid" 2>/dev/null || fail "could not terminate the deep-path watcher" + wait_for_exit "$ownerpid" 200 >/dev/null 2>&1 || true + wait_for_exit "$attachpid" "$ARM_FAIL_EXIT_POLLS" + status=$? + [ "$status" -ne 0 ] && [ "$status" -ne 124 ] || fail "deep-path attached arm did not fail after a nonzero exit (status $status)" + grep -qF "watcher pid=$wpid exited 1 without delivering a wake" "$attachout" \ + || fail "a long identity lost the owner-recorded exit: $(cat "$attachout")" + pass "the owner-recorded exit is still found when the identity is long" +} + test_cycle_exit_ledger_links_successor_and_stays_bounded() { local dir state fakebin armout check_file first_arm successor_arm successor_pid i size iteration dir=$(make_case cycle-ledger) @@ -1130,4 +1249,6 @@ test_arm_propagates_immediate_wake_before_confirmation test_arm_waits_for_peer_beacon_after_child_stands_down test_arm_fails_loud_when_no_fresh_watcher_confirmable test_cycle_exit_ledger_links_successor_and_stays_bounded +test_attached_arm_reports_the_owner_recorded_exit +test_owner_recorded_exit_survives_a_long_identity test_stopped_watcher_is_live_but_stale_then_exit_is_classified From 8025bfaa7b318aa41e755e9e5ccf708bff9e896a Mon Sep 17 00:00:00 2001 From: Sina Setayesh Date: Sat, 29 Aug 2026 18:49:36 +1000 Subject: [PATCH 2/6] no-mistakes(review): filter arm-owned ledger rows and keep unprovable liveness unknown --- bin/fm-watch-arm.sh | 40 ++++++-- docs/verification/supervision.md | 26 +++++ docs/watcher-continuity.md | 6 ++ tests/fm-watcher-lock.test.sh | 169 +++++++++++++++++++++++++++++++ 4 files changed, 234 insertions(+), 7 deletions(-) diff --git a/bin/fm-watch-arm.sh b/bin/fm-watch-arm.sh index ceb5c00031e..6cc03ae4580 100755 --- a/bin/fm-watch-arm.sh +++ b/bin/fm-watch-arm.sh @@ -289,6 +289,15 @@ wait_for_healthy_successor() { # as applied to each half. Comparing a singly-truncated probe against a doubly # truncated row silently stops matching once an identity is long enough, which is # reachable with a deep watcher path. +# +# The reason column decides whether a row describes the WATCHER's own end at all. +# A row is written under the watcher's pid whenever a cycle closes, including the +# ones where the ARM was the thing that ended - an interrupted arm TERMs its child +# and then records its OWN signal against that pid. Reading such a row back as the +# watcher's fate names a signal the watcher never received. Only the two +# classifications owned_child_finished draws from the child's own wait status are +# eligible; this is an allow-list on purpose, so a reason added later is ignored +# here by default instead of silently becoming a watcher exit. owner_recorded_exit() { local pid=$1 identity=$2 want [ -f "$CYCLE_LOG" ] || return 1 @@ -296,7 +305,8 @@ owner_recorded_exit() { "$(cycle_clean_field "${pid:-none}")" "$(cycle_clean_field "${identity:-none}")")") awk -F'\t' -v pid="watcher_pid=$pid" -v want="lock_before=$want" \ -v since="$cycle_started_at" ' - $2 == pid && $3 == "origin=started" && $10 == want { + $2 == pid && $3 == "origin=started" && $10 == want \ + && ($8 == "reason=nonzero-exit" || $8 == "reason=signal-exit") { ended = $5; sub(/^ended_at=/, "", ended) if (ended + 0 < since + 0) next code = $6; sub(/^exit_code=/, "", code) @@ -314,12 +324,21 @@ owner_recorded_exit() { # only remaining evidence in exactly that case. Re-prove the identity the cycle # recorded, the way every other holder check in this codebase does, and compare # it through the same cleaning the ledger uses so length cannot decide the answer. +# +# Three answers, not two. 0: the recorded identity is re-proven on a live pid. +# 1: the watcher is provably gone - its pid is free, or that pid now belongs to a +# different process. 2: nothing is proven either way, because this cycle recorded +# no identity for the watcher or the live pid's identity cannot be read right now. +# Collapsing 2 into either neighbour invents a fact: "still live" names a process +# that may not be the watcher, and "exited" asserts an end that was never +# observed. An unknown has to stay unknown all the way to the operator. cycle_watcher_still_live() { local pid=$1 identity=$2 current fm_pid_alive "$pid" || return 1 - [ -n "$identity" ] || return 0 + [ -n "$identity" ] || return 2 current=$(cycle_clean_field "$(fm_pid_identity "$pid" 2>/dev/null || true)") - [ -n "$current" ] && [ "$current" = "$identity" ] + [ -n "$current" ] || return 2 + [ "$current" = "$identity" ] } # Name what actually ended a cycle that produced no reason line. "Cycle ended @@ -330,18 +349,25 @@ cycle_watcher_still_live() { # and the verdict and exit status are untouched - only the operator's evidence # improves. unexplained_cycle_evidence() { - local pid=$cycle_watcher_pid holder age owner_exit clean_identity + local pid=$cycle_watcher_pid holder age owner_exit clean_identity live=0 age=$(fm_path_age "$BEAT") case "$pid" in ''|*[!0-9]*) printf 'watcher identity was never established'; return 0 ;; esac holder=$(cat "$WATCH_LOCK/pid" 2>/dev/null || true) clean_identity=$(cycle_clean_field "$cycle_watcher_identity") - if ! cycle_watcher_still_live "$pid" "$clean_identity" \ - && owner_exit=$(owner_recorded_exit "$pid" "$clean_identity"); then + # One probe, read once: two calls could disagree about a single fact if the + # process changes state between them, and each one forks ps off this host. + cycle_watcher_still_live "$pid" "$clean_identity" || live=$? + if [ "$live" -eq 2 ]; then + printf 'watcher pid=%s could not be identified - that pid is in use but the identity recorded for this cycle could not be re-proven against it, so neither a live watcher nor an exit is established (last beacon %ss ago, lock now: %s)' \ + "$pid" "$age" "${holder:-unheld}" + return 0 + fi + if [ "$live" -ne 0 ] && owner_exit=$(owner_recorded_exit "$pid" "$clean_identity"); then printf 'watcher pid=%s %s without delivering a wake (last beacon %ss ago, lock now: %s)' \ "$pid" "$owner_exit" "$age" "${holder:-unheld}" return 0 fi - if cycle_watcher_still_live "$pid" "$clean_identity"; then + if [ "$live" -eq 0 ]; then if [ "$holder" = "$pid" ]; then printf 'watcher pid=%s is still live and holds this home lock, but its beacon has not advanced for %ss' "$pid" "$age" else diff --git a/docs/verification/supervision.md b/docs/verification/supervision.md index 857d77dc122..8f038a67573 100644 --- a/docs/verification/supervision.md +++ b/docs/verification/supervision.md @@ -539,6 +539,32 @@ mutant -> not ok - a long identity lost the owner-recorded exit: The mutant does not lose the verdict, only the exit code - it falls back to the generic disposition text in exactly the case where the recorded exit is the only evidence left. The fixture asserts its own identity length before relying on it, so it cannot pass vacuously on a short path. +The ledger reason filter, mutant = the allow-list removed so any row under the watcher's PID is eligible: + +```text +control -> ok - an arm-interrupted row is not reported as the watcher's own exit +mutant -> not ok - attached arm reported its own arm's signal as the watcher's exit: + watcher: attached pid=25977 (beacon 1s) + watcher: FAILED - cycle ended without an actionable reason: + watcher pid=25977 was killed by HUP without delivering a wake +``` + +The watcher in that fixture was never sent SIGHUP; its own arm was, and the arm terminated it. +The mutant names a signal source that does not exist, which is the same unactionable evidence in a more confident voice. + +The three-answer liveness probe, mutant = an unreadable identity collapsed back into "not live": + +```text +control -> ok - an unreadable identity reads as unknown, and a real death still reads as a death +mutant -> not ok - an unreadable identity was reported as a death: + watcher: attached pid=27845 (beacon 1s) + watcher: FAILED - cycle ended without an actionable reason: + watcher pid=27845 died leaving its own lock behind +``` + +The fixture asserts the watcher is still alive at that moment, so the mutant's sentence is provably false rather than merely unproven. +Its second leg kills that same watcher and requires the death wording to still fire, so the fix cannot pass by simply never saying it. + ### Beacon freshness under host suspend - attempted and deferred The second half of the reported problem, that `state/.last-watcher-beat` can read fresh while supervision is already dead and stale while a watcher is merely suspended, was attempted on this branch and deliberately deferred rather than shipped. diff --git a/docs/watcher-continuity.md b/docs/watcher-continuity.md index 613364678d2..ee5c3bd27aa 100644 --- a/docs/watcher-continuity.md +++ b/docs/watcher-continuity.md @@ -96,7 +96,11 @@ The typed prefix is unchanged, so anything already matching on it keeps matching An arm that merely ATTACHED holds no handle on its watcher's exit status, so where the evidence is a real exit code it is read back from the lifecycle ledger the OWNING arm already wrote. That lookup is bound to the same watcher PID, the same recorded process identity, and a close no earlier than this arm attached, so neither a recycled PID nor an older cycle of the same watcher can be read as this one. It stays diagnostic: a missing or unreadable row degrades to the disposition text and never changes the verdict. +Only rows classifying the WATCHER's own termination are eligible, on an explicit allow-list, because a row is written under the watcher's PID whenever a cycle closes - including the closes the ARM caused, where an interrupted arm terminates its child and then records its OWN signal there. +Reading such a row back as the watcher's fate would name a signal the watcher never received, so every other reason falls through to the disposition text instead. Liveness inside that evidence is identity-qualified rather than bare, because a recycled PID is a live PID and would otherwise name an unrelated process to the operator while suppressing the recorded exit code that is the only remaining evidence in exactly that case. +That probe answers three ways rather than two: the recorded identity is re-proven on a live PID, the watcher is provably gone, or nothing is established either way because no identity was recorded for the cycle or the live PID's identity cannot be read right now. +The third answer is reported as an unknown rather than resolved into either neighbour, since naming a live watcher that cannot be identified and asserting an exit that was never observed are the same invented certainty. The ledger probe is rebuilt through the same transforms the ledger applied on the way in, since a single truncation of a composite is not the same as truncating each half, and a singly-truncated probe silently stops matching its own row once an identity is long enough - which a deep worktree path reaches. `bin/fm-guard.sh`'s pull warning and `bin/fm-turnend-guard.sh`'s turn-end block keep their verdicts, their exit statuses, and their alarms exactly as they were; only their banner wording now separates a live identity-matched holder that has not beaten from nothing running at all, so an operator does not read one fault as the other. @@ -117,6 +121,8 @@ The same suite covers ordinary same-process session replacement for `/new`, `/re `tests/fm-watch-arm.test.sh` covers durable queue replay, real remote parent-replies ingestion into the authoritative status log, decision-only OPEN DECISIONS recovery, interrupted handling replay, generation-bound acknowledgement, a persistent live successor after recovery, a watcher close inside the handling window that must leave the printed acknowledgement valid, and the self-healing moved-generation acknowledgement that consumes its handled rows and names its remedy. `tests/fm-watch-recovery-loop.test.sh` covers the once-per-generation announcement bound with the real Pi extension against a refused handling handshake, and a handling successor that must surface a real crew event instead of going blind. `tests/fm-watcher-lock.test.sh` covers the cycle-end evidence on two paired fixtures: an attached arm whose watcher exits nonzero must report the exit its owning arm recorded rather than the bare unactionable sentence, and the same two-arm fixture run from a deliberately deep path must still find that row once the identity is long enough for the ledger's two truncations to disagree. +Its paired leg signals the OWNING arm instead of the watcher, so the ledger carries an arm-interrupted row under the watcher's PID, and asserts the attached arm does not report that signal as the watcher's own exit. +A further paired fixture makes one attached arm's identity read fail while its watcher keeps running and requires the evidence to name that unknown, while the other leg of the same fixture, which can read identities, must still call a watcher that really died with its lock left behind a death. `tests/fm-guard-stale-banner.test.sh` and `tests/fm-turnend-guard.test.sh` each pair a live unbeaten holder against nothing running at all, with both legs still warning and still blocking respectively, so the wording can change without the verdict moving. `tests/fm-watcher-lock.test.sh` covers verified-successor attach, recovery publication before stale-lock removal, the typed self-eviction failure, bounded and successor-linked lifecycle rows, and a SIGSTOP counterfactual that distinguishes a live PID from a stale beacon before classifying termination. `tests/fm-subagent-pretool-check.test.sh` proves Claude retains only the non-status Bash seatbelts. diff --git a/tests/fm-watcher-lock.test.sh b/tests/fm-watcher-lock.test.sh index 77dc0621743..cb5fb2ea505 100755 --- a/tests/fm-watcher-lock.test.sh +++ b/tests/fm-watcher-lock.test.sh @@ -955,6 +955,173 @@ test_owner_recorded_exit_survives_a_long_identity() { pass "the owner-recorded exit is still found when the identity is long" } +# The paired leg of the case above, differing by a single fact: which process is +# signalled. The ledger writes a row under the WATCHER's pid whenever a cycle +# closes, including the closes the ARM caused - an interrupted arm TERMs its child +# and then records its OWN signal in that row. Read back unfiltered, that row told +# the operator "watcher pid=W was killed by HUP" about a watcher that was TERMed +# by its own arm, sending them after a SIGHUP source that never existed. +test_attached_arm_does_not_blame_the_watcher_for_its_arms_signal() { + local dir state fakebin ledger ownerout attachout ownerpid attachpid wpid status i + dir=$(make_case attached-owner-interrupted) + state="$dir/state" + fakebin="$dir/fakebin" + ledger="$state/.watch-cycle-exits.log" + ownerout="$dir/owner-arm.out" + attachout="$dir/attached-arm.out" + mark_pr_check_migration_complete "$state" + PATH="$fakebin:$PATH" FM_HOME="$dir" FM_STATE_OVERRIDE="$state" FM_POLL=1 FM_SIGNAL_GRACE=1 FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH_ARM" > "$ownerout" 2>&1 & + ownerpid=$! + i=0 + while [ "$i" -lt 100 ]; do + grep -qF 'watcher: started pid=' "$ownerout" 2>/dev/null && break + sleep 0.1 + i=$((i + 1)) + done + wpid=$(cat "$state/.watch.lock/pid" 2>/dev/null || true) + grep -qF "watcher: started pid=$wpid" "$ownerout" \ + || { kill -TERM "$ownerpid" 2>/dev/null; fail "owning arm did not start a watcher"; } + + PATH="$fakebin:$PATH" FM_HOME="$dir" FM_STATE_OVERRIDE="$state" FM_ARM_ATTACH_POLL=0.1 \ + FM_ARM_CONFIRM_TIMEOUT=5 "$WATCH_ARM" > "$attachout" 2>&1 & + attachpid=$! + i=0 + while [ "$i" -lt 100 ]; do + grep -qF "watcher: attached pid=$wpid" "$attachout" 2>/dev/null && break + sleep 0.1 + i=$((i + 1)) + done + grep -qF "watcher: attached pid=$wpid" "$attachout" \ + || { kill -TERM "$ownerpid" "$attachpid" "$wpid" 2>/dev/null; fail "second arm did not attach to the running watcher"; } + + # Interrupt the OWNING ARM, not the watcher. Its handler TERMs the watcher and + # then records 129/HUP - the ARM's fate - against the watcher's pid. + kill -HUP "$ownerpid" 2>/dev/null \ + || { kill -TERM "$ownerpid" "$attachpid" "$wpid" 2>/dev/null; fail "could not interrupt the owning arm"; } + wait_for_exit "$ownerpid" 200 >/dev/null 2>&1 || true + i=0 + while [ "$i" -lt 100 ]; do + grep -q "watcher_pid=$wpid origin=started.*reason=arm-interrupted" "$ledger" 2>/dev/null && break + sleep 0.1 + i=$((i + 1)) + done + grep -q "watcher_pid=$wpid origin=started.*reason=arm-interrupted" "$ledger" 2>/dev/null \ + || { kill -TERM "$attachpid" "$wpid" 2>/dev/null; fail "fixture wrote no arm-interrupted row under the watcher pid: $(cat "$ledger" 2>/dev/null)"; } + + wait_for_exit "$attachpid" "$ARM_FAIL_EXIT_POLLS" + status=$? + kill -TERM "$wpid" 2>/dev/null || true + [ "$status" -ne 0 ] && [ "$status" -ne 124 ] \ + || fail "attached arm did not fail after its watcher ended (status $status)" + grep -qF 'cycle ended without an actionable reason:' "$attachout" \ + || fail "attached arm printed no evidence clause: $(cat "$attachout")" + if grep -qF 'killed by HUP' "$attachout"; then + fail "attached arm reported its own arm's signal as the watcher's exit: $(cat "$attachout")" + fi + pass "an arm-interrupted row is not reported as the watcher's own exit" +} + +# A liveness probe has three answers, not two. When a live pid cannot be matched +# to the identity this cycle recorded - because the identity read itself fails - +# nothing is established either way. Collapsing that into "not live" made the arm +# tell the operator the watcher "died leaving its own lock behind" about a watcher +# that was running the whole time, which is the same invented certainty this +# change exists to remove. Both legs share one fixture and one watcher and differ +# by a single fact: whether the failing arm could read that identity. +test_unprovable_liveness_is_not_reported_as_a_death() { + local dir state fakebin noproc real_ps sentinel ownerout blindout deadout + local ownerpid blindpid deadpid wpid status i + dir=$(make_case unprovable-liveness) + state="$dir/state" + fakebin="$dir/fakebin" + noproc="$dir/no-proc" + sentinel="$dir/identity-unreadable" + ownerout="$dir/owner-arm.out" + blindout="$dir/blind-arm.out" + deadout="$dir/dead-arm.out" + mkdir -p "$noproc" + mark_pr_check_migration_complete "$state" + + # Route every identity read through ps on both Linux and macOS, then make that + # read fail for one arm only, on demand. Nothing else in the fixture carries + # FM_TEST_BLIND_IDENTITY, so the watcher and its owning arm keep seeing a real ps. + real_ps=$(command -v ps) || fail "no ps on PATH to wrap" + cat > "$fakebin/ps" < "$ownerout" 2>&1 & + ownerpid=$! + i=0 + while [ "$i" -lt 150 ]; do + grep -qF 'watcher: started pid=' "$ownerout" 2>/dev/null && break + sleep 0.1 + i=$((i + 1)) + done + wpid=$(cat "$state/.watch.lock/pid" 2>/dev/null || true) + grep -qF "watcher: started pid=$wpid" "$ownerout" \ + || { kill -TERM "$ownerpid" 2>/dev/null; fail "owning arm did not start a watcher: $(cat "$ownerout")"; } + + PATH="$fakebin:$PATH" FM_HOME="$dir" FM_STATE_OVERRIDE="$state" FM_PROC_ROOT_OVERRIDE="$noproc" \ + FM_TEST_BLIND_IDENTITY="$sentinel" FM_ARM_ATTACH_POLL=0.1 FM_ARM_CONFIRM_TIMEOUT=2 \ + "$WATCH_ARM" > "$blindout" 2>&1 & + blindpid=$! + PATH="$fakebin:$PATH" FM_HOME="$dir" FM_STATE_OVERRIDE="$state" FM_PROC_ROOT_OVERRIDE="$noproc" \ + FM_ARM_ATTACH_POLL=0.1 FM_ARM_CONFIRM_TIMEOUT=2 "$WATCH_ARM" > "$deadout" 2>&1 & + deadpid=$! + i=0 + while [ "$i" -lt 150 ]; do + grep -qF "watcher: attached pid=$wpid" "$blindout" 2>/dev/null \ + && grep -qF "watcher: attached pid=$wpid" "$deadout" 2>/dev/null && break + sleep 0.1 + i=$((i + 1)) + done + if ! grep -qF "watcher: attached pid=$wpid" "$blindout" 2>/dev/null \ + || ! grep -qF "watcher: attached pid=$wpid" "$deadout" 2>/dev/null; then + kill -TERM "$ownerpid" "$blindpid" "$deadpid" "$wpid" 2>/dev/null + fail "both arms did not attach to the running watcher" + fi + + # Leg one: the identity read fails while the watcher is untouched and running. + : > "$sentinel" + wait_for_exit "$blindpid" "$ARM_FAIL_EXIT_POLLS" + status=$? + kill -0 "$wpid" 2>/dev/null \ + || { rm -f "$sentinel"; kill -TERM "$ownerpid" "$deadpid" "$wpid" 2>/dev/null; fail "the watcher was supposed to outlive the blinded arm"; } + rm -f "$sentinel" + if [ "$status" -eq 0 ] || [ "$status" -eq 124 ]; then + kill -TERM "$ownerpid" "$deadpid" "$wpid" 2>/dev/null + fail "blinded arm did not fail loudly (status $status)" + fi + if grep -qF "died leaving its own lock behind" "$blindout"; then + kill -TERM "$ownerpid" "$deadpid" "$wpid" 2>/dev/null + fail "an unreadable identity was reported as a death: $(cat "$blindout")" + fi + grep -qF "watcher pid=$wpid could not be identified" "$blindout" \ + || { kill -TERM "$ownerpid" "$deadpid" "$wpid" 2>/dev/null; fail "blinded arm did not name the unknown: $(cat "$blindout")"; } + + # Leg two: the same fixture, the same sentence at stake, one fact different - + # this arm can read identities, and the watcher really is gone with its lock + # still on disk. The wording that must not be invented above must still fire here. + kill -KILL "$ownerpid" 2>/dev/null || true + wait_for_exit "$ownerpid" 100 >/dev/null 2>&1 || true + kill -KILL "$wpid" 2>/dev/null || true + wait_for_exit "$deadpid" "$ARM_FAIL_EXIT_POLLS" + status=$? + [ "$status" -ne 0 ] && [ "$status" -ne 124 ] \ + || fail "arm did not fail loudly after its watcher was killed (status $status)" + grep -qF "watcher pid=$wpid died leaving its own lock behind" "$deadout" \ + || fail "a real death was not named as one: $(cat "$deadout")" + pass "an unreadable identity reads as unknown, and a real death still reads as a death" +} + test_cycle_exit_ledger_links_successor_and_stays_bounded() { local dir state fakebin armout check_file first_arm successor_arm successor_pid i size iteration dir=$(make_case cycle-ledger) @@ -1250,5 +1417,7 @@ test_arm_waits_for_peer_beacon_after_child_stands_down test_arm_fails_loud_when_no_fresh_watcher_confirmable test_cycle_exit_ledger_links_successor_and_stays_bounded test_attached_arm_reports_the_owner_recorded_exit +test_attached_arm_does_not_blame_the_watcher_for_its_arms_signal test_owner_recorded_exit_survives_a_long_identity +test_unprovable_liveness_is_not_reported_as_a_death test_stopped_watcher_is_live_but_stale_then_exit_is_classified From 7ea8d5aaf75a033227b1b16a7cb09c4a9ad91e86 Mon Sep 17 00:00:00 2001 From: Sina Setayesh Date: Sat, 29 Aug 2026 19:25:11 +1000 Subject: [PATCH 3/6] no-mistakes(test): record the unproven empty-identity liveness leg --- docs/verification/supervision.md | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/docs/verification/supervision.md b/docs/verification/supervision.md index 8f038a67573..be991386c54 100644 --- a/docs/verification/supervision.md +++ b/docs/verification/supervision.md @@ -514,7 +514,8 @@ ShellCheck 0.11.0 (pinned) base: origin/main f66be0f ``` -Both regression cases were checked against the code with their own fix removed, because a case that cannot fail proves nothing. +Each regression case below was checked against the code with its own fix removed, because a case that cannot fail proves nothing. +One behaviour in this change has no such case; it is named in full under "The uncovered leg" below rather than left for a reader to find by opening the test files. The cycle-end evidence, mutant = the evidence clause reverted to the bare upstream sentence: @@ -565,6 +566,24 @@ mutant -> not ok - an unreadable identity was reported as a death: The fixture asserts the watcher is still alive at that moment, so the mutant's sentence is provably false rather than merely unproven. Its second leg kills that same watcher and requires the death wording to still fire, so the fix cannot pass by simply never saying it. +#### The uncovered leg: an empty recorded identity + +That liveness probe has a second leg, and no test in this repository fails without it. +This is the one behaviour in this change that ships unproven, and it is stated here so the coverage of the rest is not read as covering it too. + +When the identity recorded for the cycle is empty rather than merely unreadable, `cycle_watcher_still_live` answers unknown instead of falling back to bare pid liveness. +Reverting that leg alone - `[ -n "$identity" ] || return 2` back to `return 0` - leaves the entire `tests/fm-watcher-lock.test.sh` suite green at 35 ok, 0 not ok, exit 0. +The mutant was confirmed to be valid bash first, so that green is a real pass rather than a broken script. + +It could not be staged because the fixed and unfixed code only diverge on a pid the kernel has already recycled. +An empty recorded identity is captured in exactly one place: when an owning arm forks its watcher and `fm_pid_identity` cannot read the new child, at `cycle_begin "$child" started ...` in `bin/fm-watch-arm.sh`. +Shadowing `ps` so that read fails does reproduce the empty identity - driven through the real scripts, that arm prints `watcher pid=N exited and released this home lock without recording a delivered wake` - but an owning arm reaches the evidence path only after `wait` has reaped that child, so `fm_pid_alive` is already false and both versions take the same branch above the leg. +The attached-arm paths cannot reach it at all, because `fm_watcher_lock_matches_pid` rejects an empty recorded identity before a cycle can begin from one. +Forcing the divergence therefore needs that freed pid to be reused by a live process inside that window, which is not deterministically stageable without test-only machinery that would make the fixture unlike a real run. + +The leg is kept exactly as written rather than deleted. +Deleting it would restore the bare-liveness fallback this change's own rationale rules out, which trades an unproven behaviour for a known defect. + ### Beacon freshness under host suspend - attempted and deferred The second half of the reported problem, that `state/.last-watcher-beat` can read fresh while supervision is already dead and stale while a watcher is merely suspended, was attempted on this branch and deliberately deferred rather than shipped. From 49769c30f9ca1a94a104b99bc70f82afbbda914c Mon Sep 17 00:00:00 2001 From: Sina Setayesh Date: Sat, 29 Aug 2026 19:38:10 +1000 Subject: [PATCH 4/6] no-mistakes(document): fix stale pull-guard banner prose and dedupe suite inventory --- docs/turnend-guard.md | 7 +++++-- docs/watcher-continuity.md | 7 +++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/docs/turnend-guard.md b/docs/turnend-guard.md index 75b72f8f16f..78e315aab24 100644 --- a/docs/turnend-guard.md +++ b/docs/turnend-guard.md @@ -42,7 +42,10 @@ That ownership proof is `fm_pi_extension_owns_supervision` in `bin/fm-wake-lib.s Requiring the turn-end guard extension as well as the watch extension is deliberate, because a home without that structural backstop has no benign hand-off to tolerate. Without that proof an unheld lock alarms exactly as it did before, so an unloaded, version-drifted, or exited Pi session is loud immediately, and a cycle the extension never restores is loud once the beacon passes grace. Under every persistent-watcher harness a live identity-matched watcher with a fresh beacon is still required, so the pull guard keeps the same strict semantics there. -Its banner names the true failing condition, either a missing live watcher process or a genuinely stale beacon with its real age, and keys the once-per-episode dedup on that condition rather than the beacon mtime. +Its banner names the true failing condition: a missing live watcher process, a live identity-matched holder of this home's lock that has not beaten, or no watcher with a fresh beacon at all, each with the real beacon age. +The turn-end block draws the same live-holder distinction in its own banner, because a host that merely resumed from suspend must not read as a watcher that died. +Both name that holder through `fm_watcher_live_holder_pid` in `bin/fm-wake-lib.sh`, a deliberately wait-free lock read that re-proves the recorded identity and never sleeps or watches the beacon, because a guard whose job is to block a turn must not wait in order to phrase itself. +Naming the holder changes no verdict, exit status, or alarm, and the once-per-episode dedup still keys on the verdict reason - a missing watcher versus a stale beacon - rather than on that wording or the beacon mtime. While `state/.afk` exists the away-mode daemon (`bin/fm-supervise-daemon.sh`) owns supervision and runs the watcher one-shot: the watcher exits on every wake and the daemon starts its replacement, so a turn boundary regularly lands in a hand-off where no watcher process holds the lock and nothing is wrong. The turn-end guard therefore accepts `fm_afk_daemon_owns_supervision` from `bin/fm-wake-lib.sh` as proof of supervision on that path: away mode must be active, and this home's `state/.supervise-daemon.lock` must name a live pid whose current process identity still matches the identity the daemon recorded for itself. @@ -170,7 +173,7 @@ That warning uses `bin/fm-supervision-instructions.sh --repair-line`, so it alwa `tests/fm-turnend-guard.test.sh` covers the predicate, main and secondmate primary scope, child-worktree exclusion, `FM_HOME` and `FM_STATE_OVERRIDE` precedence, the live-lock and fresh-beacon guard predicate, the cooperative `--claude` open-generation claim wait, monotonic failed-epoch progression, bounded attended fail-open, post-alarm continuation suppression, positive recovery reset, generation and legacy claim cases that must block or clear instead of allowing a blind stop, away-mode daemon ownership between watcher cycles and over a watcher lock left behind by an exited watcher, plus its dead, pid-reused, absent, stale-beacon, and away-mode-off negatives, Pi logical-run latching, missing-`jq` behavior, all five primary registrations, Grok native and legacy selection, typed field precedence, malformed input, and exactly-one-path safety. It also covers the blocking banner's separation of a live unbeaten holder from an absent watcher, with both legs still blocking. `tests/fm-guard-stale-banner.test.sh` covers the pull-guard predicate, including the persistent-model fresh-leftover-beacon negative control, the auto-arm model's healthy fresh-beacon-without-a-watcher case and stale-beacon alarm, and the extension model's live-watcher path, ownership-qualified fresh hand-off, held-lock failures, independently broken ownership signals, stale-beacon alarm, queued-wake warning, and Pi and pi-signed harness routing. -It also covers true-reason banner wording and reason-keyed episode dedup surviving a beacon mtime change. +It also covers true-reason banner wording and reason-keyed episode dedup surviving a beacon mtime change, including a live unbeaten holder named apart from both a dead holder and a genuinely unheld lock, with every leg still alarming exactly once. `tests/fm-cursor-primary.test.sh` covers the Cursor park end to end over real processes with no harness installed: each tracked Claude-shaped entrypoint standing down on a Cursor payload, both follow-up sources, the bounded repair nag and its reset, the nested loop bounds, supersession, away-mode and lock-ownership inertness, Pi-host stand-down without Cursor identity and continued parking when `PI_CODING_AGENT` leaks alongside `CURSOR_AGENT` or `CURSOR_INVOKED_AS`, child-worktree exclusion, and that the adapter never exits 2. `FM_CURSOR_PRIMARY_LIVE_E2E=1 tests/fm-cursor-primary-live-e2e.test.sh` is the opt-in guard that proves the same behavior against the installed cursor-agent and fails naming the harness and version. `tests/fm-kimi-harness.test.sh` covers the separate Kimi crew hook's format preservation, idempotence, refusal cases, token guard, spawn registration, and teardown cleanup. diff --git a/docs/watcher-continuity.md b/docs/watcher-continuity.md index ee5c3bd27aa..616c822acbe 100644 --- a/docs/watcher-continuity.md +++ b/docs/watcher-continuity.md @@ -103,8 +103,7 @@ That probe answers three ways rather than two: the recorded identity is re-prove The third answer is reported as an unknown rather than resolved into either neighbour, since naming a live watcher that cannot be identified and asserting an exit that was never observed are the same invented certainty. The ledger probe is rebuilt through the same transforms the ledger applied on the way in, since a single truncation of a composite is not the same as truncating each half, and a singly-truncated probe silently stops matching its own row once an identity is long enough - which a deep worktree path reaches. -`bin/fm-guard.sh`'s pull warning and `bin/fm-turnend-guard.sh`'s turn-end block keep their verdicts, their exit statuses, and their alarms exactly as they were; only their banner wording now separates a live identity-matched holder that has not beaten from nothing running at all, so an operator does not read one fault as the other. -Both read the holder through a deliberately wait-free lock check that never sleeps and never watches the beacon, because a guard whose job is to block a turn must not wait in order to phrase itself. +`bin/fm-guard.sh`'s pull warning and `bin/fm-turnend-guard.sh`'s turn-end block now draw the same live-holder distinction in their banners with no verdict, exit status, or alarm moving; [`turnend-guard.md`](turnend-guard.md#guard-predicates) owns that wording and the wait-free holder read it uses. The arm layer appends one tab-separated record per observed cycle to `state/.watch-cycle-exits.log`. Each record includes arm and watcher PIDs, start and end timestamps, exit code and signal, classified reason, beacon age, lock identity before and after close, and successor disposition. @@ -120,11 +119,11 @@ Only the watcher process touches `state/.last-watcher-beat`; no helper process c The same suite covers ordinary same-process session replacement for `/new`, `/resume`, `/fork`, and reload, same-instance shutdown-plus-start, automatic re-arm before any model turn, a fresh extension-module rebind carrying all in-flight actionable closes exactly once, stale prior-generation callbacks, repeated transitions with exactly one live cycle, disappearance of the shutting-down refusal after a valid replacement activates, and terminal quit still refusing late rearm. `tests/fm-watch-arm.test.sh` covers durable queue replay, real remote parent-replies ingestion into the authoritative status log, decision-only OPEN DECISIONS recovery, interrupted handling replay, generation-bound acknowledgement, a persistent live successor after recovery, a watcher close inside the handling window that must leave the printed acknowledgement valid, and the self-healing moved-generation acknowledgement that consumes its handled rows and names its remedy. `tests/fm-watch-recovery-loop.test.sh` covers the once-per-generation announcement bound with the real Pi extension against a refused handling handshake, and a handling successor that must surface a real crew event instead of going blind. -`tests/fm-watcher-lock.test.sh` covers the cycle-end evidence on two paired fixtures: an attached arm whose watcher exits nonzero must report the exit its owning arm recorded rather than the bare unactionable sentence, and the same two-arm fixture run from a deliberately deep path must still find that row once the identity is long enough for the ledger's two truncations to disagree. +`tests/fm-watcher-lock.test.sh` covers verified-successor attach, recovery publication before stale-lock removal, the typed self-eviction failure, bounded and successor-linked lifecycle rows, and a SIGSTOP counterfactual that distinguishes a live PID from a stale beacon before classifying termination. +It also covers the cycle-end evidence on two paired fixtures: an attached arm whose watcher exits nonzero must report the exit its owning arm recorded rather than the bare unactionable sentence, and the same two-arm fixture run from a deliberately deep path must still find that row once the identity is long enough for the ledger's two truncations to disagree. Its paired leg signals the OWNING arm instead of the watcher, so the ledger carries an arm-interrupted row under the watcher's PID, and asserts the attached arm does not report that signal as the watcher's own exit. A further paired fixture makes one attached arm's identity read fail while its watcher keeps running and requires the evidence to name that unknown, while the other leg of the same fixture, which can read identities, must still call a watcher that really died with its lock left behind a death. `tests/fm-guard-stale-banner.test.sh` and `tests/fm-turnend-guard.test.sh` each pair a live unbeaten holder against nothing running at all, with both legs still warning and still blocking respectively, so the wording can change without the verdict moving. -`tests/fm-watcher-lock.test.sh` covers verified-successor attach, recovery publication before stale-lock removal, the typed self-eviction failure, bounded and successor-linked lifecycle rows, and a SIGSTOP counterfactual that distinguishes a live PID from a stale beacon before classifying termination. `tests/fm-subagent-pretool-check.test.sh` proves Claude retains only the non-status Bash seatbelts. `tests/fm-claude-stop-autoarm.test.sh` covers the auto-arm's scope, stale and live session owners, unchanged AFK and need boundaries, single-flight, bounded failure retries, benign live-watcher cycle ends, one-notice failure episodes, and exit-2 translation. It also covers generation-claim single-flight, stuck-claim supersession, superseded-owner silence, notice-marker refusal and retry, ownership-atomic episode reset, and the legacy upgrade shim; [`turnend-guard.md`](turnend-guard.md) owns those behavior contracts. From cc54f29431e1928fa8396856ec4031a7244d524a Mon Sep 17 00:00:00 2001 From: Sina Setayesh Date: Sat, 29 Aug 2026 19:40:29 +1000 Subject: [PATCH 5/6] no-mistakes(document): state the deferred beacon limit and drop internal round vocabulary --- docs/verification/supervision.md | 2 +- docs/watcher-continuity.md | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/verification/supervision.md b/docs/verification/supervision.md index be991386c54..23ace4c678a 100644 --- a/docs/verification/supervision.md +++ b/docs/verification/supervision.md @@ -587,7 +587,7 @@ Deleting it would restore the bare-liveness fallback this change's own rationale ### Beacon freshness under host suspend - attempted and deferred The second half of the reported problem, that `state/.last-watcher-beat` can read fresh while supervision is already dead and stale while a watcher is merely suspended, was attempted on this branch and deliberately deferred rather than shipped. -The approach - separating a suspended watcher from a wedged one by watching the beacon for a bounded window instead of inferring from its age - is sound and reproduces the fault, but bounding those windows introduced five regressions of its own across four review rounds, each fix bounding one window and opening another. +The approach - separating a suspended watcher from a wedged one by watching the beacon for a bounded window instead of inferring from its age - is sound and reproduces the fault, but bounding those windows introduced five regressions of its own, each fix bounding one window and opening another. Two of them broke guarantees the change itself had stated: callers that declared no budget stopped keeping their previous timing, and a receipt could claim a full observation that was never spent, which let a caller skip the one observation the design promised. The pattern is the finding, and it is recorded here rather than lost: adding timed windows to a supervision path that had none, measured on a clock that can move underneath them in both directions, is the same class of fault the work set out to fix. No part of that machinery is present in this change; the evidence above covers only what ships. diff --git a/docs/watcher-continuity.md b/docs/watcher-continuity.md index 616c822acbe..a723155fbe0 100644 --- a/docs/watcher-continuity.md +++ b/docs/watcher-continuity.md @@ -136,5 +136,7 @@ The goal is continuity without a Pi or OpenCode model-memory re-arm step. No zero-latency guarantee is claimed because lock verification, watcher startup, and bounded retry delays remain deliberate safety work. OpenCode support targets persistent TUI sessions rather than headless `opencode run`. Claude depends on the Stop `asyncRewake` rewake, Cursor depends on its awaited stop-hook park, Grok retains native background-completion notifications, and Codex retains bounded foreground checkpoints. +The beacon in `state/.last-watcher-beat` remains an age reading rather than a liveness proof, so it can read fresh while supervision is already dead and stale while a watcher is merely suspended. +[`verification/supervision.md`](verification/supervision.md#beacon-freshness-under-host-suspend---attempted-and-deferred) records why that half is deferred rather than fixed here. [`verification/supervision.md`](verification/supervision.md#watcher-continuity) records the current five-harness live evidence, the 2026-07-24 Stop-owned Claude auto-arm results, and exact opt-in commands. From d97be756bf05060fdec76fd2a843ea10344415be Mon Sep 17 00:00:00 2001 From: Sina Setayesh Date: Thu, 3 Sep 2026 08:58:27 +1000 Subject: [PATCH 6/6] no-mistakes: apply CI fixes --- tests/fm-watcher-lock.test.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/fm-watcher-lock.test.sh b/tests/fm-watcher-lock.test.sh index cb5fb2ea505..44716478368 100755 --- a/tests/fm-watcher-lock.test.sh +++ b/tests/fm-watcher-lock.test.sh @@ -849,7 +849,6 @@ test_attached_arm_reports_the_owner_recorded_exit() { fakebin="$dir/fakebin" ownerout="$dir/owner-arm.out" attachout="$dir/attached-arm.out" - mark_pr_check_migration_complete "$state" PATH="$fakebin:$PATH" FM_HOME="$dir" FM_STATE_OVERRIDE="$state" FM_POLL=1 FM_SIGNAL_GRACE=1 FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH_ARM" > "$ownerout" & ownerpid=$! i=0 @@ -902,7 +901,6 @@ test_owner_recorded_exit_survives_a_long_identity() { dir=$(make_case long-identity) state="$dir/state" fakebin="$dir/fakebin" - mark_pr_check_migration_complete "$state" # ~470 characters of path, so lstart plus the command clears the 497-character # point where the two truncations start to disagree. @@ -969,7 +967,6 @@ test_attached_arm_does_not_blame_the_watcher_for_its_arms_signal() { ledger="$state/.watch-cycle-exits.log" ownerout="$dir/owner-arm.out" attachout="$dir/attached-arm.out" - mark_pr_check_migration_complete "$state" PATH="$fakebin:$PATH" FM_HOME="$dir" FM_STATE_OVERRIDE="$state" FM_POLL=1 FM_SIGNAL_GRACE=1 FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH_ARM" > "$ownerout" 2>&1 & ownerpid=$! i=0 @@ -1040,7 +1037,6 @@ test_unprovable_liveness_is_not_reported_as_a_death() { blindout="$dir/blind-arm.out" deadout="$dir/dead-arm.out" mkdir -p "$noproc" - mark_pr_check_migration_complete "$state" # Route every identity read through ps on both Linux and macOS, then make that # read fail for one arm only, on demand. Nothing else in the fixture carries