Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions bin/fm-guard.sh
Original file line number Diff line number Diff line change
Expand Up @@ -200,6 +200,13 @@ if [ "$watcher_healthy" = false ]; then
printf '● WATCHER DOWN - SUPERVISION IS OFF\n'
if [ "$watcher_down_reason" = no-watcher ]; then
watcher_cause=$(printf 'no live watcher process holds this home lock (last beat: %s)' "$beacon_desc")
elif live_holder=$(fm_watcher_live_holder_pid "$STATE" "$WATCH" "$FM_HOME"); then
# A live identity-matched holder with a stale beacon is a different fault
# from nothing running at all, and it is what a suspended host leaves
# behind. Naming it keeps the operator from reading a resume as a dead
# watcher; the verdict itself stays unchanged and still alarms.
watcher_cause=$(printf 'watcher pid %s holds this home lock but has not beaten (last beat: %s, grace %ss)' \
"$live_holder" "$beacon_desc" "$GRACE")
else
watcher_cause=$(printf 'no watcher has a fresh beacon (last beat: %s, grace %ss)' "$beacon_desc" "$GRACE")
fi
Expand Down
16 changes: 12 additions & 4 deletions bin/fm-turnend-guard.sh
Original file line number Diff line number Diff line change
Expand Up @@ -199,23 +199,31 @@ if [ "$FM_SUP_WATCHER_FRESH" = true ] && fm_afk_daemon_owns_supervision "$STATE"
fi

block_stop() {
local afk x_mode reason rule
local afk x_mode reason rule live_holder holder_phrase
afk=0
[ -e "$STATE/.afk" ] && afk=1
x_mode=0
[ -f "$CONFIG/x-mode.env" ] && x_mode=1
reason=$("$SCRIPT_DIR/fm-supervision-instructions.sh" --afk "$afk" --x-mode "$x_mode" --repair-line 2>/dev/null \
|| printf '%s\n' 'tasks in flight, no live watcher - repair missing watcher supervision according to the session-start operating block before ending the turn')
rule='━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━'
# The block itself is unchanged: this only names WHICH fault is blocking, so a
# host that merely resumed is not reported as a watcher that died. The read is
# wait-free, because a guard that blocks a turn must not wait to do it.
if live_holder=$(fm_watcher_live_holder_pid "$STATE" "$WATCH" "$FM_HOME"); then
holder_phrase="watcher pid $live_holder holds this home lock but has not beaten"
else
holder_phrase='no live watcher holds this home lock'
fi
{
printf '●%s\n' "$rule"
printf '● TURN WOULD END BLIND - SUPERVISION IS OFF\n'
if [ "$FM_SUP_IN_FLIGHT" -gt 0 ]; then
printf '● %s task(s) in flight, but no live watcher holds this home lock (last beat: %s).\n' "$FM_SUP_IN_FLIGHT" "$FM_SUP_BEACON_DESC"
printf '● %s task(s) in flight, but %s (last beat: %s).\n' "$FM_SUP_IN_FLIGHT" "$holder_phrase" "$FM_SUP_BEACON_DESC"
elif [ "$FM_SUP_SOURCES" -gt 0 ]; then
printf '● %s process-event source(s) registered, but no live watcher holds this home lock (last beat: %s).\n' "$FM_SUP_SOURCES" "$FM_SUP_BEACON_DESC"
printf '● %s process-event source(s) registered, but %s (last beat: %s).\n' "$FM_SUP_SOURCES" "$holder_phrase" "$FM_SUP_BEACON_DESC"
else
printf '● X-mode relay polling needs supervision, but no live watcher holds this home lock (last beat: %s).\n' "$FM_SUP_BEACON_DESC"
printf '● X-mode relay polling needs supervision, but %s (last beat: %s).\n' "$holder_phrase" "$FM_SUP_BEACON_DESC"
fi
if [ "$CLAUDE_MODE" -eq 1 ]; then
printf '● The Stop-owned auto-arm did not claim this home either, so recovery is NOT already under way.\n'
Expand Down
18 changes: 18 additions & 0 deletions bin/fm-wake-lib.sh
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,24 @@ fm_watcher_lock_matches_pid() {

FM_WATCHER_HEALTHY_PID=
FM_WATCHER_HEALTHY_IDENTITY=
# fm_watcher_live_holder_pid <state> <watch-path> [home]
# Echo the pid of a LIVE, identity-matched holder of this home's watcher lock, or
# fail when there is none. This answers "is a watcher process still there", NOT
# "is it still cycling" - a holder whose beacon is stale still qualifies, which is
# exactly the distinction the callers below need.
#
# Deliberately wait-free: it reads the lock and re-proves the recorded identity,
# and never sleeps or watches the beacon. It is used only to make an operator
# message name the right fault, so it must not add latency to a guard whose job
# is to block a turn, and it never changes a verdict.
fm_watcher_live_holder_pid() {
local state=$1 watch_path=$2 home=${3:-$FM_HOME} pid
pid=$(cat "$state/.watch.lock/pid" 2>/dev/null || true)
fm_pid_alive "$pid" || return 1
fm_watcher_lock_matches_pid "$state" "$watch_path" "$pid" "$home" || return 1
printf '%s' "$pid"
}

fm_watcher_healthy() {
local state=$1 watch_path=$2 grace=${3:-${FM_GUARD_GRACE:-300}} home=${4:-$FM_HOME} lockdir beat pid identity age
FM_WATCHER_HEALTHY_PID=
Expand Down
123 changes: 119 additions & 4 deletions bin/fm-watch-arm.sh
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,11 @@
# watcher: attached pid=<N> (beacon <age>s) - a live+fresh successor holds the lock;
# this arm attaches and follows it
# watcher: FAILED - no live watcher with a fresh beacon - could not confirm one
# watcher: FAILED - cycle ended without an actionable reason
# watcher: FAILED - cycle ended without an actionable reason: <evidence>
# - a clean cycle ended with no wake and no
# verified healthy successor
# verified healthy successor; the evidence
# clause names what ended it (see
# unexplained_cycle_evidence below)
# It NEVER reports started/attached/healthy off a stale beacon or a dead/reused pid: a
# stale-beacon or dead-pid holder either self-heals (the fresh child steals the
# dead lock per the singleton self-eviction/steal path and is confirmed) or this
Expand Down Expand Up @@ -269,8 +271,121 @@ wait_for_healthy_successor() {
done
}

# An arm that only ATTACHED to a watcher holds no handle on that watcher's exit
# status, so a cycle that ends without a delivered wake is silent from here even
# though the arm that OWNED the child classified it in the lifecycle ledger. Read
# that classification back rather than reporting nothing: this is the difference
# between "the cycle ended" and "the watcher exited 1".
#
# The row is bound to this exact cycle - same watcher pid AND the same recorded
# process identity in its lock snapshot, closed no earlier than this arm attached
# - so neither a recycled pid nor an older cycle of the same watcher can be read
# as this one. The ledger stays diagnostic: a missing or unreadable row degrades
# to the disposition text below and never changes the verdict.
#
# The wanted field is rebuilt through the SAME transforms the ledger applied on
# the way in - lock_snapshot cleans each part, then cycle_log_append cleans the
# whole composite again - because cut(1) applied once to a pair is not the same
# as applied to each half. Comparing a singly-truncated probe against a doubly
# truncated row silently stops matching once an identity is long enough, which is
# reachable with a deep watcher path.
#
# The reason column decides whether a row describes the WATCHER's own end at all.
# A row is written under the watcher's pid whenever a cycle closes, including the
# ones where the ARM was the thing that ended - an interrupted arm TERMs its child
# and then records its OWN signal against that pid. Reading such a row back as the
# watcher's fate names a signal the watcher never received. Only the two
# classifications owned_child_finished draws from the child's own wait status are
# eligible; this is an allow-list on purpose, so a reason added later is ignored
# here by default instead of silently becoming a watcher exit.
owner_recorded_exit() {
local pid=$1 identity=$2 want
[ -f "$CYCLE_LOG" ] || return 1
want=$(cycle_clean_field "$(printf 'pid:%s|identity:%s' \
"$(cycle_clean_field "${pid:-none}")" "$(cycle_clean_field "${identity:-none}")")")
awk -F'\t' -v pid="watcher_pid=$pid" -v want="lock_before=$want" \
-v since="$cycle_started_at" '
$2 == pid && $3 == "origin=started" && $10 == want \
&& ($8 == "reason=nonzero-exit" || $8 == "reason=signal-exit") {
ended = $5; sub(/^ended_at=/, "", ended)
if (ended + 0 < since + 0) next
code = $6; sub(/^exit_code=/, "", code)
sig = $7; sub(/^signal=/, "", sig)
if (code == "unknown") next
found = (sig == "none") ? "exited " code : "was killed by " sig
}
END { if (found != "") print found }
' "$CYCLE_LOG" 2>/dev/null | grep . || return 1
}

# A recycled pid is a live pid, so bare liveness cannot stand in for "the watcher
# is still running": it would name an unrelated process to the operator and, by
# claiming the watcher never exited, suppress the recorded exit code that is the
# only remaining evidence in exactly that case. Re-prove the identity the cycle
# recorded, the way every other holder check in this codebase does, and compare
# it through the same cleaning the ledger uses so length cannot decide the answer.
#
# Three answers, not two. 0: the recorded identity is re-proven on a live pid.
# 1: the watcher is provably gone - its pid is free, or that pid now belongs to a
# different process. 2: nothing is proven either way, because this cycle recorded
# no identity for the watcher or the live pid's identity cannot be read right now.
# Collapsing 2 into either neighbour invents a fact: "still live" names a process
# that may not be the watcher, and "exited" asserts an end that was never
# observed. An unknown has to stay unknown all the way to the operator.
cycle_watcher_still_live() {
local pid=$1 identity=$2 current
fm_pid_alive "$pid" || return 1
[ -n "$identity" ] || return 2
current=$(cycle_clean_field "$(fm_pid_identity "$pid" 2>/dev/null || true)")
[ -n "$current" ] || return 2
[ "$current" = "$identity" ]
}

# Name what actually ended a cycle that produced no reason line. "Cycle ended
# without an actionable reason" alone is unactionable: a watcher that exited
# cleanly, one that died leaving its lock, one another watcher replaced, and one
# that is still live but no longer beating all reach here and need different
# responses. The typed prefix is unchanged so existing consumers still match it,
# and the verdict and exit status are untouched - only the operator's evidence
# improves.
unexplained_cycle_evidence() {
local pid=$cycle_watcher_pid holder age owner_exit clean_identity live=0
age=$(fm_path_age "$BEAT")
case "$pid" in ''|*[!0-9]*) printf 'watcher identity was never established'; return 0 ;; esac
holder=$(cat "$WATCH_LOCK/pid" 2>/dev/null || true)
clean_identity=$(cycle_clean_field "$cycle_watcher_identity")
# One probe, read once: two calls could disagree about a single fact if the
# process changes state between them, and each one forks ps off this host.
cycle_watcher_still_live "$pid" "$clean_identity" || live=$?
if [ "$live" -eq 2 ]; then
printf 'watcher pid=%s could not be identified - that pid is in use but the identity recorded for this cycle could not be re-proven against it, so neither a live watcher nor an exit is established (last beacon %ss ago, lock now: %s)' \
"$pid" "$age" "${holder:-unheld}"
return 0
fi
if [ "$live" -ne 0 ] && owner_exit=$(owner_recorded_exit "$pid" "$clean_identity"); then
printf 'watcher pid=%s %s without delivering a wake (last beacon %ss ago, lock now: %s)' \
"$pid" "$owner_exit" "$age" "${holder:-unheld}"
return 0
fi
if [ "$live" -eq 0 ]; then
if [ "$holder" = "$pid" ]; then
printf 'watcher pid=%s is still live and holds this home lock, but its beacon has not advanced for %ss' "$pid" "$age"
else
printf 'watcher pid=%s is still live but no longer holds this home lock (lock now: %s)' "$pid" "${holder:-unheld}"
fi
return 0
fi
if [ -z "$holder" ]; then
printf 'watcher pid=%s exited and released this home lock without recording a delivered wake (last beacon %ss ago)' "$pid" "$age"
elif [ "$holder" = "$pid" ]; then
printf 'watcher pid=%s died leaving its own lock behind (last beacon %ss ago)' "$pid" "$age"
else
printf 'watcher pid=%s exited and this home lock moved to pid %s (last beacon %ss ago)' "$pid" "$holder" "$age"
fi
}

fail_unexplained_cycle() {
echo "watcher: FAILED - cycle ended without an actionable reason"
echo "watcher: FAILED - cycle ended without an actionable reason: $(unexplained_cycle_evidence)"
return 1
}

Expand Down Expand Up @@ -530,7 +645,7 @@ owned_child_finished() {
cycle_log_append "$rc" "$signal" "$reason_type" none
print_watch_output "$child_out"
if ! grep -q '^watcher: FAILED' "$child_out" 2>/dev/null; then
echo "watcher: FAILED - watcher cycle exited $rc without an actionable reason"
echo "watcher: FAILED - watcher cycle exited $rc without an actionable reason: $(unexplained_cycle_evidence)"
fi
rm -f "$child_out" 2>/dev/null || true
child=
Expand Down
8 changes: 6 additions & 2 deletions docs/turnend-guard.md
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,10 @@ That ownership proof is `fm_pi_extension_owns_supervision` in `bin/fm-wake-lib.s
Requiring the turn-end guard extension as well as the watch extension is deliberate, because a home without that structural backstop has no benign hand-off to tolerate.
Without that proof an unheld lock alarms exactly as it did before, so an unloaded, version-drifted, or exited Pi session is loud immediately, and a cycle the extension never restores is loud once the beacon passes grace.
Under every persistent-watcher harness a live identity-matched watcher with a fresh beacon is still required, so the pull guard keeps the same strict semantics there.
Its banner names the true failing condition, either a missing live watcher process or a genuinely stale beacon with its real age, and keys the once-per-episode dedup on that condition rather than the beacon mtime.
Its banner names the true failing condition: a missing live watcher process, a live identity-matched holder of this home's lock that has not beaten, or no watcher with a fresh beacon at all, each with the real beacon age.
The turn-end block draws the same live-holder distinction in its own banner, because a host that merely resumed from suspend must not read as a watcher that died.
Both name that holder through `fm_watcher_live_holder_pid` in `bin/fm-wake-lib.sh`, a deliberately wait-free lock read that re-proves the recorded identity and never sleeps or watches the beacon, because a guard whose job is to block a turn must not wait in order to phrase itself.
Naming the holder changes no verdict, exit status, or alarm, and the once-per-episode dedup still keys on the verdict reason - a missing watcher versus a stale beacon - rather than on that wording or the beacon mtime.

While `state/.afk` exists the away-mode daemon (`bin/fm-supervise-daemon.sh`) owns supervision and runs the watcher one-shot: the watcher exits on every wake and the daemon starts its replacement, so a turn boundary regularly lands in a hand-off where no watcher process holds the lock and nothing is wrong.
The turn-end guard therefore accepts `fm_afk_daemon_owns_supervision` from `bin/fm-wake-lib.sh` as proof of supervision on that path: away mode must be active, and this home's `state/.supervise-daemon.lock` must name a live pid whose current process identity still matches the identity the daemon recorded for itself.
Expand Down Expand Up @@ -168,8 +171,9 @@ That warning uses `bin/fm-supervision-instructions.sh --repair-line`, so it alwa
## Regression coverage

`tests/fm-turnend-guard.test.sh` covers the predicate, main and secondmate primary scope, child-worktree exclusion, `FM_HOME` and `FM_STATE_OVERRIDE` precedence, the live-lock and fresh-beacon guard predicate, the cooperative `--claude` open-generation claim wait, monotonic failed-epoch progression, bounded attended fail-open, post-alarm continuation suppression, positive recovery reset, generation and legacy claim cases that must block or clear instead of allowing a blind stop, away-mode daemon ownership between watcher cycles and over a watcher lock left behind by an exited watcher, plus its dead, pid-reused, absent, stale-beacon, and away-mode-off negatives, Pi logical-run latching, missing-`jq` behavior, all five primary registrations, Grok native and legacy selection, typed field precedence, malformed input, and exactly-one-path safety.
It also covers the blocking banner's separation of a live unbeaten holder from an absent watcher, with both legs still blocking.
`tests/fm-guard-stale-banner.test.sh` covers the pull-guard predicate, including the persistent-model fresh-leftover-beacon negative control, the auto-arm model's healthy fresh-beacon-without-a-watcher case and stale-beacon alarm, and the extension model's live-watcher path, ownership-qualified fresh hand-off, held-lock failures, independently broken ownership signals, stale-beacon alarm, queued-wake warning, and Pi and pi-signed harness routing.
It also covers true-reason banner wording and reason-keyed episode dedup surviving a beacon mtime change.
It also covers true-reason banner wording and reason-keyed episode dedup surviving a beacon mtime change, including a live unbeaten holder named apart from both a dead holder and a genuinely unheld lock, with every leg still alarming exactly once.
`tests/fm-cursor-primary.test.sh` covers the Cursor park end to end over real processes with no harness installed: each tracked Claude-shaped entrypoint standing down on a Cursor payload, both follow-up sources, the bounded repair nag and its reset, the nested loop bounds, supersession, away-mode and lock-ownership inertness, Pi-host stand-down without Cursor identity and continued parking when `PI_CODING_AGENT` leaks alongside `CURSOR_AGENT` or `CURSOR_INVOKED_AS`, child-worktree exclusion, and that the adapter never exits 2.
`FM_CURSOR_PRIMARY_LIVE_E2E=1 tests/fm-cursor-primary-live-e2e.test.sh` is the opt-in guard that proves the same behavior against the installed cursor-agent and fails naming the harness and version.
`tests/fm-kimi-harness.test.sh` covers the separate Kimi crew hook's format preservation, idempotence, refusal cases, token guard, spawn registration, and teardown cleanup.
Expand Down
Loading