diff --git a/AGENTS.md b/AGENTS.md index 6648c806301..d77af3f74a6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -94,6 +94,7 @@ state/ runtime records and signals; gitignored .status appended by crewmates: ": " wake-event lines, not current-state truth .turn-ended touched by turn-end hooks .grok-turnend-token firstmate-owned grok hook registry token for the task; removed by teardown + .worktree-returned best-effort pool-return marker in the rerun tenant-safety protocol owned by bin/fm-teardown.sh's header; removed by teardown .kimi-turnend-token firstmate-owned Kimi hook registry token for the task; removed by teardown .muse-session muse busy-source binding (sessions root plus task worktree) written by fm-spawn; removed by teardown .cursor-session cursor busy-source binding (projects root, task worktree, prior conversations) written by fm-spawn; removed by teardown diff --git a/bin/backends/herdr.sh b/bin/backends/herdr.sh index 8728b356cc0..d3391e63a44 100644 --- a/bin/backends/herdr.sh +++ b/bin/backends/herdr.sh @@ -2594,14 +2594,22 @@ fm_backend_herdr_send_key() { # # the composer-state guard/fallback reads around submit and injection). Workaround: # always request a generous fetch far above any realistic viewport height, then # trim to the caller's requested bound ourselves with `tail`. +# +# Fetches --format ansi and strips locally instead of asking for text: on herdr +# 0.8.0 a text-format `pane read --source recent` against an idle alt-screen +# agent triggers the wheel-event history harvest (src/server/alt_screen_read.rs) +# - the pane VISIBLY scrolls up for seconds and snaps back on every poll +# (herdrdev/herdr#2669, closed as dup of #2387). The ansi path skips the +# harvest entirely and returns herdr's retained rows instantly, which matches +# the pre-0.8.0 text behavior this capture was written against. fm_backend_herdr_capture() { # fm_backend_herdr_target_ready "$1" || return 1 local lines=${2:-200} fetch out case "$lines" in ''|*[!0-9]*) lines=200 ;; esac fetch=$lines case "$fetch" in ''|*[!0-9]*) fetch=200 ;; *) [ "$fetch" -ge 200 ] || fetch=200 ;; esac - out=$(fm_backend_herdr_cli "$FM_BACKEND_HERDR_SESSION" pane read "$FM_BACKEND_HERDR_PANE" --source recent --lines "$fetch" 2>/dev/null) || return 1 - printf '%s' "$out" | tail -n "$lines" + out=$(fm_backend_herdr_cli "$FM_BACKEND_HERDR_SESSION" pane read "$FM_BACKEND_HERDR_PANE" --source recent --lines "$fetch" --format ansi 2>/dev/null) || return 1 + printf '%s' "$out" | tail -n "$lines" | fm_composer_strip_ansi } fm_backend_herdr_capture_ansi() { # diff --git a/bin/fm-claude-stop-autoarm.sh b/bin/fm-claude-stop-autoarm.sh index 762b1a3dcf4..716e46c0aa5 100755 --- a/bin/fm-claude-stop-autoarm.sh +++ b/bin/fm-claude-stop-autoarm.sh @@ -10,11 +10,21 @@ # - Scope: only a genuine primary checkout (plain checkout or validly marked # secondmate home) with AGENTS.md, bin/, and the effective state dir - the # exact fm-turnend-guard.sh scope. Child crew/scout worktrees stay inert. -# - Identity: only when THIS session's harness ancestor holds state/.lock. -# When an existing numeric owner fails the shared harness-liveness predicate, -# the hook delegates guarded recovery to bin/fm-lock.sh and then re-verifies -# ownership. A live owner, missing lock, malformed lock, or unresolved -# ancestry remains inert, so a competing session never arms or rewakes. +# - Identity: only when THIS session holds state/.lock, proven by harness +# ancestry, by the delivering Claude session's own pid, or by the session the +# lock's durable binding records. The extra proofs are required because +# Claude Code serves hook commands from a shared worker pool whose top +# process is reparented to init, which can leave a hook with no ancestry path +# back to its live session at all, and because the lock records the +# OUTERMOST pid of the run that acquired it, which is not always the pid the +# harness exports for the same session; all three proofs and the disjunction +# over them are owned by bin/fm-session-lock-lib.sh, so the admission check +# and the post-recovery re-check cannot drift apart. +# When an existing numeric owner fails the shared harness-liveness +# predicate, the hook delegates guarded recovery to bin/fm-lock.sh and then +# re-verifies ownership through that same disjunction. A live owner, a +# missing or malformed lock, and a lock naming another session all remain +# inert, so a competing session never arms or rewakes. # - AFK: while state/.afk exists the away daemon owns the watcher and triage; # this hook exits 0 and NEVER rewakes the primary (checked again at # translation time so a mid-cycle AFK transition is honored). @@ -62,7 +72,13 @@ # suppresses any later automatic continuation in that unresolved episode. # # This hook never blocks the Stop decision itself and never prints to stdout: -# exit 0 is always silent, and exit 2 carries the rewake banner on stderr. +# exit 0 is always silent, and exit 2 carries the rewake banner on stderr. The +# opt-in FM_CLAUDE_AUTOARM_TRACE diagnostic is the one exception: when it is +# set to a non-empty value, every gate that ends the run inert BEFORE it claims +# the cycle names itself on stderr, which is what diagnoses a hook that never +# claims the home under real hook conditions. Exits after that claim stay silent +# and are read from the epoch ledger instead. It changes no decision and is +# never set in normal operation. # On any uncertainty such as unresolvable ancestry, malformed lock state, or # lock contention, it exits 0 and leaves continuity to the synchronous guard and # the model. @@ -83,6 +99,13 @@ case "$AUTOARM_ATTEMPTS" in *) AUTOARM_ATTEMPTS=2 ;; esac +# Opt-in gate diagnostic. Silent unless FM_CLAUDE_AUTOARM_TRACE is set to a +# non-empty value, so the production contract above is unchanged. +trace() { # + [ -n "${FM_CLAUDE_AUTOARM_TRACE:-}" ] || return 0 + printf 'autoarm-trace: %s\n' "$*" >&2 +} + # shellcheck source=bin/fm-primary-scope-lib.sh . "$SCRIPT_DIR/fm-primary-scope-lib.sh" # shellcheck source=bin/fm-supervision-lib.sh @@ -105,10 +128,16 @@ PAYLOAD=$(cat 2>/dev/null || true) # the declared multi-hour timeout - the exact wedge grok 1.0.0 produced # (docs/turnend-guard.md "Harness integrations"). Cursor's own park adapter owns # its turn boundary, so stand down on a Cursor-delivered payload. -fm_hook_payload_is_foreign_host "$PAYLOAD" && exit 0 +if fm_hook_payload_is_foreign_host "$PAYLOAD"; then + trace 'inert: payload delivered by a foreign host' + exit 0 +fi # --- scope: genuine primary checkout only ----------------------------------- -fm_primary_scope_matches "$FM_ROOT" "$STATE" || exit 0 +if ! fm_primary_scope_matches "$FM_ROOT" "$STATE"; then + trace "inert: $FM_ROOT is not a primary home with state dir $STATE" + exit 0 +fi # --- identity: only the lock-owning session's hooks may arm ------------------ # A prior session may have died after leaving its numeric harness pid in .lock. @@ -117,31 +146,63 @@ fm_primary_scope_matches "$FM_ROOT" "$STATE" || exit 0 # idle or away home remains byte-for-byte inert. Missing or malformed locks are # uncertainty rather than stale-owner evidence and remain inert. RECOVER_SESSION_LOCK=0 -if ! fm_session_lock_owned_by_self "$STATE"; then +if fm_session_lock_owned_by_this_claude_session "$STATE" "$PAYLOAD"; then + case "$FM_SESSION_LOCK_PROOF" in + ancestry) trace 'identity: proven by harness ancestry' ;; + claude-session) + trace "identity: proven by the delivering Claude session (pid ${CLAUDE_PID:-?})" + ;; + claude-session-binding) + trace "identity: proven by the lock's recorded session (${CLAUDE_CODE_SESSION_ID:-?})" + ;; + esac +else LOCK_PID=$(cat "$STATE/.lock" 2>/dev/null || true) case "$LOCK_PID" in - ''|*[!0-9]*) exit 0 ;; + ''|*[!0-9]*) + trace 'inert: no usable session lock, and neither identity proof applies' + exit 0 + ;; esac - fm_harness_pid_alive "$LOCK_PID" && exit 0 + if fm_harness_pid_alive "$LOCK_PID"; then + trace "inert: live session $LOCK_PID owns this home and neither identity proof applies (CLAUDE_PID=${CLAUDE_PID:-unset})" + exit 0 + fi + trace "identity: recorded owner $LOCK_PID is dead, recovery pending" RECOVER_SESSION_LOCK=1 fi # --- AFK: the away daemon owns the watcher and triage; never rewake ---------- -[ -e "$STATE/.afk" ] && exit 0 +if [ -e "$STATE/.afk" ]; then + trace 'inert: away mode owns supervision' + exit 0 +fi # --- need: in-flight work or an X-mode relay poll ---------------------------- need_supervision() { fm_supervision_needed "$STATE" "$GRACE" } -need_supervision || exit 0 +if ! need_supervision; then + trace 'inert: nothing in flight and no relay poll to run' + exit 0 +fi # --- stale session-lock recovery --------------------------------------------- # Delegate the claim to fm-lock.sh so its live-owner refusal and write semantics # remain the single acquisition owner, then re-verify current-session identity -# before touching any auto-arm state. +# before touching any auto-arm state. The re-check uses the same three-proof +# predicate as the admission gate above on purpose: if fm-lock.sh ever elects a +# different pid than it does today, this verification must not silently narrow +# to ancestry alone. if [ "$RECOVER_SESSION_LOCK" -eq 1 ]; then - "$SCRIPT_DIR/fm-lock.sh" >/dev/null 2>&1 || exit 0 - fm_session_lock_owned_by_self "$STATE" || exit 0 + if ! "$SCRIPT_DIR/fm-lock.sh" >/dev/null 2>&1; then + trace 'inert: guarded session-lock recovery refused' + exit 0 + fi + if ! fm_session_lock_owned_by_this_claude_session "$STATE" "$PAYLOAD"; then + trace 'inert: ownership unproven after session-lock recovery' + exit 0 + fi fi # --- single-flight generation claim -------------------------------------------- diff --git a/bin/fm-lock.sh b/bin/fm-lock.sh index 52d7c8aee4b..7ce816abe72 100755 --- a/bin/fm-lock.sh +++ b/bin/fm-lock.sh @@ -3,6 +3,11 @@ # Writes the harness (agent) process PID found by walking the shell's ancestry, # which lives as long as the firstmate session - unlike the transient subshell # PID of any one tool call, which is dead moments after it is written. +# It also records, in the adjacent state/.lock.session binding, which harness +# session acquired that pid, because ancestry is not a stable session identity: +# a call served by a reparented worker pool never reaches its own session. That +# binding is what lets the acquiring session keep proving ownership; a lock that +# carries no binding behaves exactly as it did before it existed. # Usage: fm-lock.sh acquire; exit 1 unless ownership is verified # fm-lock.sh status print holder and liveness; always exits 0 set -u @@ -55,17 +60,12 @@ release_claim_lock() { trap release_claim_lock EXIT trap 'exit 1' HUP INT TERM -if [ -f "$LOCK" ] && [ ! -L "$LOCK" ]; then - old=$(cat "$LOCK" 2>/dev/null || true) - if [ "$old" = "$me" ]; then - echo "lock acquired: harness pid $me" - exit 0 - fi - if fm_harness_pid_alive "$old"; then - echo "error: another live firstmate session holds the lock (pid $old); operate read-only until resolved" >&2 - exit 1 - fi -fi +backfill_session_binding() { + local id + id=$(fm_harness_session_identity) || return 0 + fm_session_lock_binding_names_session "$STATE" "$id" && return 0 + fm_session_lock_publish_identity "$STATE" "$old" || true +} if ! fm_lock_try_acquire "$CLAIM_LOCK"; then sweep_pid=$(sed -n 's/^pid=//p' "$STATE/.startup-network.status" 2>/dev/null | tail -1) @@ -86,6 +86,17 @@ if [ -e "$LOCK" ] || [ -L "$LOCK" ]; then echo "error: session lock is unreadable; operate read-only until resolved" >&2 exit 1 } + if fm_session_lock_owned_by_self "$STATE"; then + backfill_session_binding + release_claim_lock + echo "lock acquired: harness pid $old" + exit 0 + fi + if [ "$old" != "$me" ] && fm_session_lock_owned_by_session_identity "$STATE"; then + release_claim_lock + echo "lock acquired: harness pid $old" + exit 0 + fi if [ "$old" != "$me" ] && fm_harness_pid_alive "$old"; then echo "error: another live firstmate session holds the lock (pid $old); operate read-only until resolved" >&2 exit 1 @@ -103,5 +114,10 @@ if [ ! -f "$LOCK" ] || [ -L "$LOCK" ] || [ "$written" != "$me" ]; then echo "error: session lock ownership verification failed; operate read-only until resolved" >&2 exit 1 fi +# Bind the verified lock to this session's own identity, so this home stays +# provable from a worker pool that cannot reach the session through ancestry. +# A failure here is deliberately not fatal: the home simply keeps the +# ancestry-only behavior it had before this binding existed. +fm_session_lock_publish_identity "$STATE" "$me" || true release_claim_lock echo "lock acquired: harness pid $me" diff --git a/bin/fm-merge-local.sh b/bin/fm-merge-local.sh index 70ac9b7be2c..6501e81afa9 100755 --- a/bin/fm-merge-local.sh +++ b/bin/fm-merge-local.sh @@ -6,10 +6,44 @@ # locally instead of via a GitHub PR). It is the one sanctioned exception to hard # rule #1 "never run state-changing git in projects/", and it is narrow: it only # runs for mode=local-only tasks, only after the captain approves (or yolo=on -# auto-approves), and only as a clean fast-forward - it refuses a diverged branch -# and tells you to have the crewmate rebase. See AGENTS.md prime directives, -# project management, and task lifecycle. -# Usage: fm-merge-local.sh +# auto-approves), and only as a clean fast-forward unless the captain explicitly +# takes the --drop-local-commits escape hatch below - otherwise it refuses any +# landing that would drop a local commit and names every one of them. See +# AGENTS.md prime directives, project management, and task lifecycle. +# +# The local-commit guard exists because a task worktree is always freshened from +# origin's default-branch tip, never from the local default branch. On a project +# whose local default branch carries commits origin does not have (a fork that +# cannot merge upstream, or local adoptions of unmerged upstream work), a branch +# cut from that fresh base does not contain them, so landing it naively would +# erase them. The guard names those commits and the reconciliation that keeps +# them: `git merge ` inside the task branch. +# +# --drop-local-commits is the deliberate escape hatch for the case where losing +# them IS the intent. It is never the default and never silent: when the guard +# did not trigger it drops nothing and says so rather than passing as a silent +# no-op, and when it did it prints every dropped commit, pins the pre-reset tip +# of the default branch under refs/fm-dropped// and records them +# in state/.local-merge-drop before touching the branch, and then resets the +# default branch to the task branch. That rescue ref is what makes the +# recorded SHAs recoverable: after the reset the dropped commits hang off nothing +# else, and the reflog that would otherwise hold them expires +# (gc.reflogExpireUnreachable, 30 days by default) and is then pruned by `git gc`. +# A ref never expires, so the commits stay in the project for as long as it does. +# The ref is keyed by the rescued tip, not by the task, so a later drop on the +# same task plants its own ref beside the earlier one instead of displacing it, +# and the branch never moves when a ref cannot be planted. The branch move is a +# compare-and-swap against the inspected default tip, so a concurrent newer tip +# is preserved; a newly planted rescue is removed again when that compare-and-swap +# refuses, while a rescue from an earlier completed drop remains intact. Checkout +# synchronization also refuses concurrent index or worktree changes instead of +# overwriting them. Each ref is released on its own +# - `git update-ref -d refs/fm-dropped//` frees exactly +# that drop's commits and leaves every other rescue alone - which is the +# deliberate counterpart of the guarantee: until then those commits stay in the +# project. Being destructive, the whole escape hatch needs the captain's explicit +# word. +# Usage: fm-merge-local.sh [--drop-local-commits] set -eu SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -17,13 +51,30 @@ FM_ROOT="${FM_ROOT_OVERRIDE:-$(cd "$SCRIPT_DIR/.." && pwd)}" FM_HOME="${FM_HOME:-${FM_ROOT_OVERRIDE:-$FM_ROOT}}" STATE="${FM_STATE_OVERRIDE:-$FM_HOME/state}" "$FM_ROOT/bin/fm-guard.sh" || true + # Role partition: landing local-only work is MAIN-owned; the Pi supervision # branch reports readiness and never lands (contract: bin/fm-lease-lib.sh; # no-op in homes without a branch actor). # shellcheck source=bin/fm-lease-lib.sh . "$SCRIPT_DIR/fm-lease-lib.sh" fm_lease_forbid_branch "local-only landing (fm-merge-local)" -ID=${1:?usage: fm-merge-local.sh } + +DROP_LOCAL=no +ID= +while [ $# -gt 0 ]; do + case "$1" in + --drop-local-commits) DROP_LOCAL=yes ;; + -h|--help) awk 'NR>1 && !/^#/{exit} NR>1' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; + -*) echo "error: unknown option '$1'; usage: fm-merge-local.sh [--drop-local-commits] " >&2; exit 1 ;; + *) + [ -z "$ID" ] || { echo "error: unexpected argument '$1'; usage: fm-merge-local.sh [--drop-local-commits] " >&2; exit 1; } + ID=$1 + ;; + esac + shift +done +[ -n "$ID" ] || { echo "usage: fm-merge-local.sh [--drop-local-commits] " >&2; exit 1; } + META="$STATE/$ID.meta" [ -f "$META" ] || { echo "error: no meta for task $ID at $META" >&2; exit 1; } @@ -61,11 +112,113 @@ if [ -n "$(git -C "$PROJ" status --porcelain 2>/dev/null | head -1)" ]; then exit 1 fi -# Clean fast-forward only: DEFAULT must be an ancestor of BRANCH. -if ! git -C "$PROJ" merge-base --is-ancestor "$DEFAULT" "$BRANCH"; then - echo "REFUSED: $BRANCH is not a fast-forward of $DEFAULT (it has diverged)." >&2 - echo "Have the crewmate rebase $BRANCH onto $DEFAULT, then retry." >&2 - exit 1 +# Commits on the local default branch that the task branch does not contain. +# Empty means the merge is a clean fast-forward and nothing local is dropped; +# non-empty is exactly the trap this guard exists for. +DROPPED=$(git -C "$PROJ" rev-list "$BRANCH..$DEFAULT") + +if [ -n "$DROPPED" ]; then + if [ "$DROP_LOCAL" != yes ]; then + { + echo "REFUSED: landing $BRANCH would drop $(printf '%s\n' "$DROPPED" | wc -l | tr -d ' ') commit(s) that exist only on local $DEFAULT:" + git -C "$PROJ" log --no-decorate --format=' %h %s' "$BRANCH..$DEFAULT" + echo "$BRANCH was cut from origin/$DEFAULT, so it never contained them." + echo "Reconcile first: run 'git merge $DEFAULT' inside $BRANCH (in its own worktree), resolve any conflicts, then retry this merge." + echo "If dropping those commits is genuinely intended, re-run with --drop-local-commits (destructive; needs the captain's explicit word)." + } >&2 + exit 1 + fi + + before=$(git -C "$PROJ" rev-parse "$DEFAULT") + target=$(git -C "$PROJ" rev-parse "$BRANCH") + RECORD="$STATE/$ID.local-merge-drop" + # Keyed by the rescued tip, so successive drops on one task never contend for + # the same name and none can displace another's commits. + RESCUE_REF="refs/fm-dropped/$ID/$(git -C "$PROJ" rev-parse --short "$before")" + + # Plant it BEFORE the reset, and never move the branch without it: it is the + # only thing that keeps the dropped commits reachable once the reflog is + # pruned. Creating it with an empty expected value makes clobbering impossible. + rescue_created=no + held=$(git -C "$PROJ" rev-parse --verify --quiet "$RESCUE_REF^{commit}" || true) + if [ "$held" = "$before" ]; then + echo "note: rescue ref $RESCUE_REF already holds $before from an earlier drop; leaving it as it is" + else + git -C "$PROJ" update-ref "$RESCUE_REF" "$before" "" || { + echo "error: could not plant rescue ref $RESCUE_REF at $before in $PROJ; refusing to drop commits with no way back. $DEFAULT is untouched." >&2 + exit 1 + } + rescue_created=yes + fi + + attempt="$(date -u '+%Y-%m-%dT%H:%M:%SZ').${BASHPID:-$$}" + { + echo "# $attempt fm-merge-local.sh --drop-local-commits" + echo "attempt=$attempt" + echo "status=pending" + echo "task=$ID" + echo "project=$PROJ" + echo "default=$DEFAULT" + echo "branch=$BRANCH" + echo "default_before=$before" + echo "rescue_ref=$RESCUE_REF" + for sha in $DROPPED; do + git -C "$PROJ" log -1 --no-decorate --format='pending=%H %s' "$sha" + done + } >> "$RECORD" + + if ! git -C "$PROJ" update-ref "refs/heads/$DEFAULT" "$target" "$before"; then + current=$(git -C "$PROJ" rev-parse "refs/heads/$DEFAULT") + rescue_released=not-created + if [ "$rescue_created" = yes ]; then + if git -C "$PROJ" update-ref -d "$RESCUE_REF" "$before"; then + rescue_released=yes + else + rescue_released=failed + fi + fi + { + echo "attempt=$attempt" + echo "status=failed" + echo "default_current=$current" + echo "rescue_ref_released=$rescue_released" + } >> "$RECORD" + if [ "$rescue_released" = failed ]; then + echo "warning: could not release newly planted rescue ref $RESCUE_REF; inspect it before cleanup" >&2 + fi + echo "error: local $DEFAULT advanced concurrently from $before to $current; preserved the newer tip and refused to overwrite it without touching the checkout" >&2 + exit 1 + fi + { + echo "attempt=$attempt" + echo "status=completed" + for sha in $DROPPED; do + git -C "$PROJ" log -1 --no-decorate --format='dropped=%H %s' "$sha" + done + } >> "$RECORD" + if ! git -C "$PROJ" read-tree -m -u "$before" "$target"; then + { + echo "attempt=$attempt" + echo "status=sync-failed" + echo "default_current=$(git -C "$PROJ" rev-parse "refs/heads/$DEFAULT")" + } >> "$RECORD" + echo "error: landed $BRANCH without overwriting $DEFAULT, but concurrent index or worktree changes prevented safe checkout synchronization and were left untouched" >&2 + exit 1 + fi + + echo "DROPPING $(printf '%s\n' "$DROPPED" | wc -l | tr -d ' ') local-only commit(s) from $DEFAULT as explicitly authorized:" + for sha in $DROPPED; do + git -C "$PROJ" log -1 --no-decorate --format=' %H %s' "$sha" + done + echo "recorded in $RECORD; rescue ref $RESCUE_REF now holds $DEFAULT's pre-reset tip, so those commits stay in $PROJ (not just in the reflog) and stay recoverable by full SHA" + echo "release this drop's commits for good with: git -C $PROJ update-ref -d $RESCUE_REF (other drops keep their own ref)" + after=$(git -C "$PROJ" rev-parse --short "refs/heads/$DEFAULT") + echo "reset local $DEFAULT to $BRANCH ($(git -C "$PROJ" rev-parse --short "$before") -> $after) in $PROJ" + exit 0 +fi + +if [ "$DROP_LOCAL" = yes ]; then + echo "note: --drop-local-commits was unnecessary; $BRANCH already contains every local $DEFAULT commit" fi before=$(git -C "$PROJ" rev-parse --short "$DEFAULT") diff --git a/bin/fm-session-lock-lib.sh b/bin/fm-session-lock-lib.sh index d77e563f0b4..3bae4c8fc60 100644 --- a/bin/fm-session-lock-lib.sh +++ b/bin/fm-session-lock-lib.sh @@ -1,10 +1,18 @@ #!/usr/bin/env bash # Shared session-lock harness identity. # -# ONE owner of the "which verified-harness process holds this home's session -# lock, and does the current process descend from that same harness?" decision. -# bin/fm-lock.sh uses it to acquire and inspect state/.lock; -# bin/fm-claude-stop-autoarm.sh uses it to prove a Stop hook fires inside the +# ONE owner of "may this run act for the session that holds this home's lock?", +# over three independent proofs and the disjunctions that arbitrate them. +# Ancestry asks whether the current process descends from the verified harness +# the lock records. The durable session-identity binding beside the lock answers +# the same ownership question when ancestry cannot, because a call served by a +# reparented worker pool never reaches its own session; see +# fm_session_lock_owned_by_current_session. Delivered Claude session identity +# instead asks whether the session that emitted THIS hook event is the one the +# lock names, which a hook can prove from its payload before any binding exists; +# docs/watcher-continuity.md owns that contract. +# bin/fm-lock.sh uses this file to acquire, inspect, and bind state/.lock; +# bin/fm-claude-stop-autoarm.sh uses it to prove a Stop hook fires for the # lock-owning primary session before it may arm or rewake. # This file is sourced by scripts and has no side effects on source. @@ -125,6 +133,23 @@ fm_harness_ancestry_pids() { [ "$printed" -eq 1 ] } +# True when pid $1 is a member of that contiguous harness ancestry: the ONE +# owner of "is this pid a harness process the current run genuinely sits inside". +# Both membership questions in this file ask it - of the pid the lock records, +# and of the served-session pid the harness reports. An ancestry that cannot be +# resolved answers false, so every caller stays fail-closed. +fm_harness_ancestry_contains() { # + local want=$1 pids pid + [ -n "$want" ] || return 1 + pids=$(fm_harness_ancestry_pids) || return 1 + while IFS= read -r pid; do + [ "$pid" = "$want" ] && return 0 + done < + local payload=${1-} id + [ -n "$payload" ] || return 1 + id=$(printf '%s' "$payload" \ + | tr ',{}' '\n' \ + | sed -n 's/^[[:space:]]*"session_id"[[:space:]]*:[[:space:]]*"\([A-Za-z0-9._-]\{1,\}\)"[[:space:]]*$/\1/p' \ + | sed -n '1p') + [ -n "$id" ] || return 1 + printf '%s\n' "$id" +} + +# True when state dir $1 holds a session lock owned by the very Claude Code +# session that delivered hook payload $2. +# +# This is the second, ancestry-independent membership proof, and it exists +# because a Claude Code hook does not reliably run under its own session. Claude +# Code serves hook and tool commands from a shared per-user worker pool +# (claude bg-spare -> claude bg-pty-host -> claude daemon run) whose top process +# is reparented to init once the session that first started it exits. A hook +# served by such a pool has a contiguous claude ancestry that does not contain +# the live session at all, so the ancestry proof fails through no fault of the +# session and the hook goes inert (docs/watcher-continuity.md). +# +# The proof is a conjunction, and every part is required: +# 1. the delivered payload names a session id, which no inherited environment +# can supply - it describes THIS event; +# 2. the session id the delivering session exported matches that payload, so +# the environment read below belongs to the session that emitted the event +# rather than to some ancestor session it was inherited from; +# 3. the exported session pid is EXACTLY the pid recorded in this home's lock, +# which is stricter than ancestry membership rather than weaker; +# 4. that pid is still a live Claude process, so a recycled or dead pid never +# passes. +# +# A foreign session therefore still fails: it exports its own pid, which is not +# the pid this home's lock records. A missing, malformed, or foreign-owned lock +# fails closed, and so does every home whose lock names another session, which +# is what keeps several firstmate homes on one machine independent. +fm_session_lock_owned_by_claude_hook() { # + local state=$1 payload=${2-} lock_pid payload_session + case "${CLAUDE_PID:-}" in + ''|*[!0-9]*) return 1 ;; + esac + [ -n "${CLAUDE_CODE_SESSION_ID:-}" ] || return 1 + payload_session=$(fm_claude_payload_session_id "$payload") || return 1 + [ "$payload_session" = "$CLAUDE_CODE_SESSION_ID" ] || return 1 + lock_pid=$(cat "$state/.lock" 2>/dev/null || true) + case "$lock_pid" in + ''|*[!0-9]*) return 1 ;; + esac + [ "$lock_pid" = "$CLAUDE_PID" ] || return 1 + fm_harness_pid_alive "$lock_pid" || return 1 + [ "$FM_HARNESS_IS_CLAUDE" -eq 1 ] || return 1 +} + # True when state dir $1 holds a session lock whose pid is ANY harness ancestor # of the current process: this script runs inside the session that owns the # home's fleet lock. Membership is the honest test of that question, because the @@ -161,16 +241,238 @@ fm_harness_pid_alive() { # lock, a malformed lock, a lock held by a harness outside this ancestry, or an # ancestry that cannot be resolved all fail closed. fm_session_lock_owned_by_self() { - local state=$1 lock_pid pids pid + local state=$1 lock_pid lock_pid=$(cat "$state/.lock" 2>/dev/null || true) case "$lock_pid" in ''|*[!0-9]*) return 1 ;; esac - pids=$(fm_harness_ancestry_pids) || return 1 - while IFS= read -r pid; do - [ "$pid" = "$lock_pid" ] && return 0 - done < + printf '%s/.lock.session\n' "$1" +} + +# Print the harness session identity of the CURRENT process, or return 1 when +# the harness exposes none. +# +# Claude Code is currently the only verified harness that exports one. The value +# is the session's own id, which is why it is stable across a compaction and, +# unlike process ancestry, identical whichever worker pool serves the call. +# Anything that is not a plain identifier is refused rather than sanitized, so a +# surprising value can only withhold the proof, never widen it. +fm_harness_session_identity() { + local id=${CLAUDE_CODE_SESSION_ID:-} + case "$id" in + ''|*[!A-Za-z0-9._-]*) return 1 ;; + esac + printf '%s\n' "$id" +} + +# True when the session identity this process carries actually describes THIS +# process, rather than one it merely inherited through the environment. +# +# A session identity travels down every child of a tool call, so the string +# alone proves nothing: a crewmate launched on a harness that does not overwrite +# these variables would carry its launching session's identity, and any number of +# unrelated processes started from one environment would carry the same one. That +# is a false-success path, not a theoretical one. +# +# The corroboration is structural: the harness reports the session process it is +# serving this call for, and a call genuinely served by that session has that +# very process inside its own contiguous harness ancestry. Inheritance cannot +# fake it, because an unrelated process's ancestry contains its own harness +# instead. Note this is membership, NOT equality with the lock pid: the whole +# point is that the ancestry reaches the pool process serving the call while +# never reaching the pid the lock records. +fm_harness_session_is_ours() { + local claimed=${CLAUDE_PID:-} + case "$claimed" in + ''|*[!0-9]*) return 1 ;; + esac + fm_harness_pid_alive "$claimed" || return 1 + fm_harness_ancestry_contains "$claimed" +} + +# Record, beside state dir $1's session lock, that lock pid $2 was acquired by +# THIS process's session. Best effort by contract: every failure leaves the home +# on the ancestry proof alone, which is exactly today's behavior. +# +# The stale record is discarded BEFORE the new one is written, so a home whose +# harness exposes no session identity is left with no binding at all rather than +# a previous owner's. That ordering is what stops a recycled lock pid from ever +# meeting a foreign session id in the same record. +fm_session_lock_publish_identity() { # + local state=$1 pid=$2 id path tmp + path=$(fm_session_lock_identity_path "$state") + command rm -f -- "$path" 2>/dev/null || return 1 + id=$(fm_harness_session_identity) || return 0 + # Never bind a lock to an identity this process only inherited: that would + # hand the home to whichever session the environment happens to name. + fm_harness_session_is_ours || return 0 + tmp=$(mktemp "$state/.lock.session.XXXXXX" 2>/dev/null) || return 1 + if ! { printf 'pid=%s\nsession=%s\n' "$pid" "$id" > "$tmp"; } 2>/dev/null; then + command rm -f -- "$tmp" 2>/dev/null + return 1 + fi + mv -f "$tmp" "$path" 2>/dev/null || { + command rm -f -- "$tmp" 2>/dev/null + return 1 + } +} + +# True when the durable binding beside state dir $1's session lock records that +# lock for session $2. This is the shared core of every recorded-identity proof +# below, so the record's own rules are stated once here: +# 1. the lock holds a plain numeric pid; +# 2. the binding beside it is a regular file naming exactly that pid, so a +# record left by a previous owner can never speak for the current lock; +# 3. the session it names is exactly the session asked about; +# 4. the lock pid is still a live harness process. +# +# A foreign session fails at 3 because it carries its own id. An absent or +# malformed lock fails at 1, an absent or stale binding at 2, and a dead owner at +# 4. Two firstmate homes on one machine stay independent because each reads its +# own state dir. An old lock carrying only a pid has no binding, so it fails at 2 +# and that home keeps exactly the ancestry-only behavior it had before. +# +# Each caller supplies the CORROBORATION that the session it asks about is really +# the one behind this run, because the environment string alone proves nothing. +fm_session_lock_binding_names_session() { # + local state=$1 want=$2 lock_pid path recorded_pid recorded_session + [ -n "$want" ] || return 1 + lock_pid=$(cat "$state/.lock" 2>/dev/null || true) + case "$lock_pid" in + ''|*[!0-9]*) return 1 ;; + esac + path=$(fm_session_lock_identity_path "$state") + [ -f "$path" ] && [ ! -L "$path" ] || return 1 + recorded_pid=$(sed -n 's/^pid=//p' "$path" 2>/dev/null | sed -n '1p') + recorded_session=$(sed -n 's/^session=//p' "$path" 2>/dev/null | sed -n '1p') + [ "$recorded_pid" = "$lock_pid" ] || return 1 + [ -n "$recorded_session" ] && [ "$recorded_session" = "$want" ] || return 1 + fm_harness_pid_alive "$lock_pid" || return 1 +} + +# True when state dir $1's session lock was acquired by the very session this +# process belongs to, proven by recorded identity instead of process ancestry. +# +# This exists because ancestry is not a stable session identity. Claude Code +# serves a session's hooks and tool calls from more than one worker pool, and a +# pool whose top process has been reparented to init yields a contiguous harness +# run that never reaches the session's own lineage. The same session therefore +# presents one ancestry through one pool and a different one through another, and +# once the pool that could reach it is gone it can never again prove it owns its +# own lock. +# +# The corroboration here is ancestry: this process carries a harness session +# identity AND that identity is confirmed by its own ancestry rather than merely +# inherited as an environment string (fm_harness_session_is_ours). A caller with +# no hook event has nothing stronger available, so a process that merely +# inherited another session's environment is refused. +fm_session_lock_owned_by_session_identity() { # + local state=$1 id + id=$(fm_harness_session_identity) || return 1 + fm_harness_session_is_ours || return 1 + fm_session_lock_binding_names_session "$state" "$id" +} + +# True when state dir $1 holds a session lock the durable binding records for the +# very session that delivered hook payload $2. +# +# This is the third membership proof, and it exists because the second one is +# strictly pid-shaped: it requires the pid the harness exports to BE the pid the +# lock records. Claude Code serves hook and tool commands from a shared per-user +# worker pool, and bin/fm-lock.sh writes the OUTERMOST pid of the contiguous +# harness run that acquired the lock, so the two are not always the same process +# even though they are the same session. When they differ, ancestry cannot reach +# the recorded owner either, and the hook goes permanently inert on a home whose +# lock is alive and genuinely its own - the measured shape reported as "live +# session N owns this home and neither identity proof applies". +# +# The proof is a conjunction, and every part is required: +# 1. the delivered payload names a session id, which no inherited environment +# can supply - it describes THIS event; +# 2. the session id the delivering session exported matches that payload, so +# the environment read here belongs to the session that emitted the event +# rather than to some ancestor session it was inherited from. This is the +# SAME corroboration the pid proof uses, and it replaces the ancestry +# corroboration fm_session_lock_owned_by_session_identity needs, because a +# caller with no hook event has no way to prove its environment describes +# the call it is making; +# 3. the durable binding beside the lock names exactly the pid the lock +# records, so a record left by a previous owner can never speak for it; +# 4. the session that binding names is exactly this session; +# 5. the recorded owner is still a live harness process. +# +# A foreign session fails at 4 because it carries its own id, so several +# firstmate homes on one machine stay independent exactly as before. A home whose +# lock carries no binding fails at 3 and keeps the previous two-proof behavior. +fm_session_lock_owned_by_claude_hook_binding() { # + local state=$1 payload=${2-} payload_session + [ -n "${CLAUDE_CODE_SESSION_ID:-}" ] || return 1 + payload_session=$(fm_claude_payload_session_id "$payload") || return 1 + [ "$payload_session" = "$CLAUDE_CODE_SESSION_ID" ] || return 1 + fm_session_lock_binding_names_session "$state" "$CLAUDE_CODE_SESSION_ID" +} + +# True when the session behind the current run owns state dir $1's fleet lock, +# proven by harness ancestry OR by the durable identity the lock records. This +# disjunction is the ONE owner of "may this run mutate this home" for every +# caller that has NO hook event to reason about, so the gate that admits a +# session start and the sweeps it authorizes cannot drift apart. A hook that +# does carry a payload asks fm_session_lock_owned_by_this_claude_session below. +# +# Ancestry is tried first and left exactly as it was, because it is the only +# proof available for a harness that exposes no session identity. The recorded +# identity is required because ancestry alone cannot answer the question from a +# reparented worker pool. Neither member is a fallback for the other, and a +# caller that needs only one of them still calls that one directly. +fm_session_lock_owned_by_current_session() { # + fm_session_lock_owned_by_self "$1" && return 0 + fm_session_lock_owned_by_session_identity "$1" +} + +# True when the Claude session behind the current run holds state dir $1's +# session lock, proven either by harness ancestry or by the identity delivered +# with hook payload $2. This disjunction is the ONE owner of "may this HOOK +# EVENT act for this home", so the gate that admits a hook run and any later +# re-verification cannot drift apart. +# +# All three members are load-bearing. Ancestry is tried first and left unchanged, +# because a legitimate claude-launched-by-claude wrapper chain records the +# OUTERMOST pid in the lock while CLAUDE_PID names the inner session, so +# preferring the delivered identity would refuse that case. The delivered pid +# identity is required because a hook served by the shared per-user worker pool +# reparented to init has no ancestry path back to its live session at all. The +# durable binding is required because that same OUTERMOST-pid rule means the pid +# the harness exports and the pid the lock records can be different processes of +# one session, which leaves the first two proofs with nothing to match on. None +# of them is a fallback for the others: a caller that needs only one still calls +# that one directly. +# +# FM_SESSION_LOCK_PROOF names the member that carried the verdict, so a caller +# can report which proof applied; it is empty when neither holds. +# shellcheck disable=SC2034 # Read by sourcing callers, not inside this file. +FM_SESSION_LOCK_PROOF='' +# shellcheck disable=SC2034 # FM_SESSION_LOCK_PROOF is a caller-read output. +fm_session_lock_owned_by_this_claude_session() { # + local state=$1 payload=${2-} + FM_SESSION_LOCK_PROOF='' + if fm_session_lock_owned_by_self "$state"; then + FM_SESSION_LOCK_PROOF=ancestry + return 0 + fi + if fm_session_lock_owned_by_claude_hook "$state" "$payload"; then + FM_SESSION_LOCK_PROOF=claude-session + return 0 + fi + if fm_session_lock_owned_by_claude_hook_binding "$state" "$payload"; then + FM_SESSION_LOCK_PROOF=claude-session-binding + return 0 + fi return 1 } diff --git a/bin/fm-sessionstart-run.sh b/bin/fm-sessionstart-run.sh index a970eced675..b0cc52400db 100755 --- a/bin/fm-sessionstart-run.sh +++ b/bin/fm-sessionstart-run.sh @@ -93,7 +93,7 @@ session_start_completed() { local lock_pid completion_pid [ -f "$STATE/.lock" ] && [ ! -L "$STATE/.lock" ] || return 1 [ -f "$COMPLETION_FILE" ] && [ ! -L "$COMPLETION_FILE" ] || return 1 - fm_session_lock_owned_by_self "$STATE" || return 1 + fm_session_lock_owned_by_current_session "$STATE" || return 1 lock_pid=$(cat "$STATE/.lock" 2>/dev/null) || return 1 completion_pid=$(cat "$COMPLETION_FILE" 2>/dev/null) || return 1 case "$lock_pid" in ''|*[!0-9]*) return 1 ;; esac diff --git a/bin/fm-spawn.sh b/bin/fm-spawn.sh index 9158fce64df..b3d3ff4119a 100755 --- a/bin/fm-spawn.sh +++ b/bin/fm-spawn.sh @@ -147,6 +147,9 @@ # containment test reads local refs only and never fetches, so this gate stays # usable offline; a stale remote-tracking ref can therefore make an unpushed # commit look contained, which is exactly why no remedy command is printed. +# That base stays origin's tip even when the LOCAL default branch is ahead of +# it, so an upstream PR never carries local-only commits; the spawn only warns +# about the divergence, and bin/fm-merge-local.sh guards the local landing. # Batch dispatch: pass one or more `id=repo` pairs instead of a single , e.g. # fm-spawn.sh fix-a-k3=projects/foo add-b-q7=projects/bar [--scout] # Each pair re-execs this script in single-task mode, so the single path stays the only @@ -1892,6 +1895,21 @@ EOF printf '%s' "$lines" >&2 } +# Warn when the project's LOCAL default branch carries commits origin's tip does +# not, which is normal on a fork that cannot merge upstream and on local +# adoptions of unmerged upstream work. The base itself stays origin's tip on +# purpose: a branch destined for an upstream PR must never carry local-only +# commits. The cost lands later instead, when that branch is landed locally, so +# say so at spawn. Advisory only - it never changes the base and never stops a +# launch. bin/fm-merge-local.sh owns the refusal that actually protects them. +warn_local_default_divergence() { # + local worktree=$1 default=$2 ahead + git -C "$worktree" rev-parse --verify --quiet "refs/heads/$default^{commit}" >/dev/null || return 0 + ahead=$(git -C "$worktree" rev-list --count "origin/$default..refs/heads/$default" 2>/dev/null || true) + case "$ahead" in ''|0) return 0 ;; esac + echo "warning: local $default is $ahead commit(s) ahead of origin/$default; this worker starts from origin/$default (correct for an upstream PR), so landing its branch locally will need 'git merge $default' inside the branch first" >&2 +} + freshen_spawn_worktree_base() { # local worktree=$1 default target expected actual status if ! git -C "$worktree" fetch --quiet origin; then @@ -1936,6 +1954,7 @@ freshen_spawn_worktree_base() { # echo "error: pooled worktree '$worktree' is at '${actual:-unknown}', not current '$target' ('$expected'); refusing to launch" >&2 return 1 fi + warn_local_default_divergence "$worktree" "$default" } herdr_projection_meta_field_exact() { # diff --git a/bin/fm-startup-network.sh b/bin/fm-startup-network.sh index 380138ae25f..b7e222842bf 100755 --- a/bin/fm-startup-network.sh +++ b/bin/fm-startup-network.sh @@ -218,7 +218,7 @@ cmd_start() { # # the worker: re-reading the lock later would only prove that SOME session # holds it, which is exactly the case this guard exists to reject. lock_pid=$(cat "$STATE/.lock" 2>/dev/null || true) - if [ "$locked" = 1 ] && ! fm_session_lock_owned_by_self "$STATE"; then + if [ "$locked" = 1 ] && ! fm_session_lock_owned_by_current_session "$STATE"; then return 1 fi @@ -433,7 +433,7 @@ cmd_run() { # fi fm_lock_release "$PUBLISH_LOCK" [ "$internal" -eq 1 ] || return 1 - elif [ "$locked" = 1 ] && ! fm_session_lock_owned_by_self "$STATE"; then + elif [ "$locked" = 1 ] && ! fm_session_lock_owned_by_current_session "$STATE"; then downgraded=1 locked=0 fi diff --git a/bin/fm-teardown.sh b/bin/fm-teardown.sh index ad9e042ba11..037245fe78a 100755 --- a/bin/fm-teardown.sh +++ b/bin/fm-teardown.sh @@ -73,6 +73,28 @@ # releases its durable treehouse lease so the pool slot is freed, # never left leased forever. If the treehouse return fails, teardown leaves the # leased home and state in place instead of hiding a still-held lease. +# Rerun tenant safety (teardown-rerun-reissue, observed 2026-08-08): a teardown +# that returned this task's pool worktree and then failed on a later step (for +# example a refused focus-unsafe pane close) is rerun by design - but by then +# the pool may have reissued the SAME slot to a newer task, so re-running any +# worktree-scoped step would reset the worktree under the live tenant and kill +# its session. Ownership is decided from two provable signals: the best-effort +# durable state/.worktree-returned marker, published immediately after this +# task's own successful return, and the current fleet bindings - another task's +# meta recording the same worktree path proves the slot was reissued even when +# the marker is lost. Teardown takes the shared task-set lock before its task +# meta lock and holds it through return and marker publication; spawn takes the +# same lock before publishing a new binding, so a failed marker write cannot +# expose a return/reissue interval in which a rerun sees neither signal. A +# binding whose task carries its own worktree-returned +# marker is ignored: that task provably gave the slot back already, so its +# meta records a past tenancy, not a live claim. When either signal shows the +# worktree is no longer this task's, the safety inspection, run-abort, +# worktree process reap, branch delete, hook removal, and the return itself +# are all skipped with one line, while tasktmp reaping, the pane close retry, +# and durable-record cleanup continue unchanged. +# The marker is removed with the rest of the volatile state once teardown +# completes. # Usage: fm-teardown.sh [--force] # --force skips ordinary-task dirty and landed-work checks, skips scout report # checks, and discards secondmate child work for kind=secondmate. Only use it @@ -219,6 +241,8 @@ CONTROL_LOCK="$STATE/.control-$ID.lock" CONTROL_LOCK_HELD=0 META_LOCK= META_LOCK_HELD=0 +WORKTREE_TRANSITION_LOCK= +WORKTREE_TRANSITION_LOCK_HELD=0 DESCENDANT_LOCK_PATHS=() DESCENDANT_TASK_STATES=() DESCENDANT_TASK_IDS=() @@ -245,6 +269,10 @@ teardown_release_locks() { fm_lock_release "$LOCAL_REGISTRY_LOCK" || true LOCAL_REGISTRY_LOCK= fi + if [ "$WORKTREE_TRANSITION_LOCK_HELD" = 1 ]; then + fm_lock_release "$WORKTREE_TRANSITION_LOCK" || true + WORKTREE_TRANSITION_LOCK_HELD=0 + fi if [ "$META_LOCK_HELD" = 1 ]; then fm_lock_release "$META_LOCK" || true META_LOCK_HELD=0 @@ -257,6 +285,12 @@ teardown_release_locks() { return "$status" } trap teardown_release_locks EXIT +WORKTREE_TRANSITION_LOCK=$(fm_task_set_lock_path "$STATE") || { + echo "error: could not resolve the worktree transition lock for $STATE" >&2 + exit 1 +} +fm_lock_acquire_wait "$WORKTREE_TRANSITION_LOCK" || exit 1 +WORKTREE_TRANSITION_LOCK_HELD=1 fm_lock_try_acquire "$CONTROL_LOCK" || { echo "error: another lifecycle action is already running for task $ID; nothing was changed" >&2 exit 1 @@ -2559,8 +2593,68 @@ remove_secondmate_registry_entry() { return "$rc" } +# Rerun tenant-safety guard (see the script header). WORKTREE_OWNED_BY_TASK=0 +# means the recorded worktree is provably no longer this task's and every +# worktree-scoped step below must be skipped. +WORKTREE_RETURN_MARKER="$STATE/$ID.worktree-returned" +WORKTREE_OWNED_BY_TASK=1 +WORKTREE_SKIP_REASON= + +# Echoes the id of another task whose meta records the same worktree path +# without its own worktree-returned marker, proving the pool slot was +# reissued; a marker-bearing binding is a stale record of an already-returned +# tenancy and is skipped. Returns non-zero when no other task holds a live +# binding. +worktree_reissued_to_other_task() { + local meta other_id other_wt abs_wt abs_other other_marker + [ -n "$WT" ] || return 1 + abs_wt=$(canonical_existing_dir "$WT") || abs_wt=$WT + for meta in "$STATE"/*.meta; do + [ -f "$meta" ] && [ ! -L "$meta" ] || continue + other_id=$(basename "$meta" .meta) + [ "$other_id" != "$ID" ] || continue + other_wt=$(fm_meta_get "$meta" worktree) + [ -n "$other_wt" ] || continue + abs_other=$(canonical_existing_dir "$other_wt") || abs_other=$other_wt + [ "$abs_other" = "$abs_wt" ] || continue + other_marker="$STATE/$other_id.worktree-returned" + if [ -e "$other_marker" ] || [ -L "$other_marker" ]; then + continue + fi + printf '%s\n' "$other_id" + return 0 + done + return 1 +} + +resolve_worktree_ownership() { + local tenant + [ "$KIND" != secondmate ] || return 0 + [ -n "$WT" ] || return 0 + if [ -e "$WORKTREE_RETURN_MARKER" ] || [ -L "$WORKTREE_RETURN_MARKER" ]; then + WORKTREE_OWNED_BY_TASK=0 + WORKTREE_SKIP_REASON="this task's return already succeeded on a previous attempt" + elif tenant=$(worktree_reissued_to_other_task); then + WORKTREE_OWNED_BY_TASK=0 + WORKTREE_SKIP_REASON="the pool slot is now recorded for task $tenant" + fi + if [ "$WORKTREE_OWNED_BY_TASK" -ne 1 ]; then + echo "teardown: skipping worktree return and every other worktree step for $ID: $WORKTREE_SKIP_REASON" + fi +} + validate_pr_poll_cleanup "$STATE" "$ID" || exit 1 +if [ "$KIND" = secondmate ] || [ "$BACKEND" = orca ] || [ -z "$WT" ]; then + fm_lock_release "$WORKTREE_TRANSITION_LOCK" || exit 1 + WORKTREE_TRANSITION_LOCK_HELD=0 +fi +resolve_worktree_ownership +if [ "$WORKTREE_OWNED_BY_TASK" -ne 1 ] && [ "$WORKTREE_TRANSITION_LOCK_HELD" = 1 ]; then + fm_lock_release "$WORKTREE_TRANSITION_LOCK" || exit 1 + WORKTREE_TRANSITION_LOCK_HELD=0 +fi + if [ "$KIND" = secondmate ]; then LOCAL_REGISTRY_LOCK=$(secondmate_registry_lock_path "$STATE") fm_lock_acquire_wait "$LOCAL_REGISTRY_LOCK" || exit 1 @@ -2664,7 +2758,7 @@ if [ "$BACKEND" = orca ] && [ "$KIND" != scout ] && [ "$KIND" != secondmate ] && ORCA_PATH_MATCH_VERIFIED=1 fi -if [ -d "$WT" ] && [ "$FORCE" != "--force" ]; then +if [ -d "$WT" ] && [ "$FORCE" != "--force" ] && [ "$WORKTREE_OWNED_BY_TASK" = 1 ]; then if validate_worktree_teardown_safety; then : else @@ -2722,8 +2816,14 @@ fi # dedicated process-event and firstmate-home removal machinery further below, # not by task-worktree cleanup. if [ "$KIND" != secondmate ]; then - conclude_task_no_mistakes_run "$WT" - reap_task_worktree_processes worktree "$WT" "$TASK_TMP" + if [ "$WORKTREE_OWNED_BY_TASK" = 1 ]; then + conclude_task_no_mistakes_run "$WT" + reap_task_worktree_processes worktree "$WT" "$TASK_TMP" + else + # The worktree now belongs to another tenant, but the per-task tasktmp root + # is still uniquely this task's and is removed below, so still reap it. + reap_task_worktree_processes tasktmp "$TASK_TMP" + fi fi # Fix 3 (see script header): sweep remote job workers abandoned by an already @@ -2749,7 +2849,7 @@ if [ "$BACKEND" = orca ] && [ "$KIND" != secondmate ]; then fi [ -z "$T_ORCA" ] || fm_backend_kill "$BACKEND" "$T" "$(meta_value "$META" zellij_tab_id)" "fm-$ID" 2>/dev/null || true fm_backend_remove_worktree "$BACKEND" "$ORCA_WORKTREE_ID" -elif [ -d "$WT" ] && [ "$KIND" != secondmate ]; then +elif [ -d "$WT" ] && [ "$KIND" != secondmate ] && [ "$WORKTREE_OWNED_BY_TASK" = 1 ]; then branch=$(git -C "$WT" rev-parse --abbrev-ref HEAD 2>/dev/null || echo HEAD) if [ "$branch" != "HEAD" ]; then if git -C "$WT" checkout --detach -q 2>/dev/null; then @@ -2771,6 +2871,14 @@ elif [ -d "$WT" ] && [ "$KIND" != secondmate ]; then echo "error: treehouse return failed for worktree $WT; teardown aborted" >&2 exit 1 } + # Record the completed return durably BEFORE any later step can fail, so a + # sanctioned rerun never re-returns a slot the pool may reissue meanwhile. + touch "$WORKTREE_RETURN_MARKER" 2>/dev/null \ + || echo "warning: could not record the completed worktree return for $ID at $WORKTREE_RETURN_MARKER; a rerun will rely on live fleet bindings to skip the return" >&2 + if [ "$WORKTREE_TRANSITION_LOCK_HELD" = 1 ]; then + fm_lock_release "$WORKTREE_TRANSITION_LOCK" || exit 1 + WORKTREE_TRANSITION_LOCK_HELD=0 + fi fi HERDR_PRESENTATION_JOURNAL="$STATE/$ID.herdr-presentation" @@ -2876,6 +2984,7 @@ rm -f "$STATE/$ID.turn-ended" \ "$STATE/$ID.pi-ext.ts" "$STATE/$ID.grok-turnend-token" \ "$STATE/$ID.kimi-turnend-token" "$STATE/$ID.muse-session" \ "$STATE/$ID.muse-session-current" "$STATE/$ID.cursor-session" \ + "$STATE/$ID.worktree-returned" \ "$STATE/$ID.control-relaunch" "$STATE/$ID.control-relaunch.meta-prior" \ "$STATE/$ID.control-relaunch.brief-prior" "$STATE/$ID.control-relaunch.note" \ "$STATE/$ID.reconcile-nudged" diff --git a/bin/fm-test-run.sh b/bin/fm-test-run.sh index 4857a212bbc..4c024f98fca 100755 --- a/bin/fm-test-run.sh +++ b/bin/fm-test-run.sh @@ -224,7 +224,7 @@ family_for_basename() { printf '%s\n' pure-contract-unit ;; fm-daemon.test.sh|fm-guard-stale-banner.test.sh|fm-pi-watch-extension.test.sh|\ - fm-session-lock-ancestry.test.sh|fm-cursor-primary.test.sh|\ + fm-session-lock-ancestry.test.sh|fm-session-lock-identity.test.sh|fm-cursor-primary.test.sh|\ fm-supervision-events.test.sh|fm-turnend-guard.test.sh|fm-wake-daemon-lifecycle-e2e.test.sh|\ fm-wake-drain-unread-status.test.sh|\ fm-tool-update-check.test.sh|\ @@ -287,7 +287,7 @@ family_for_basename() { fm-teardown-endpoint-safety.test.sh) printf '%s\n' backend-dispatch ;; - fm-check-unregister.test.sh|fm-pr-check-security.test.sh|fm-pr-merge.test.sh|\ + fm-check-unregister.test.sh|fm-merge-local.test.sh|fm-pr-check-security.test.sh|fm-pr-merge.test.sh|\ fm-review-diff.test.sh|fm-teardown.test.sh|fm-x-mode.test.sh) printf '%s\n' pr-forge ;; @@ -606,6 +606,7 @@ tests/fm-send-resolve-key.test.sh 19619 tests/fm-send-secondmate-marker-herdr-e2e.test.sh 51 tests/fm-send-secondmate-marker.test.sh 6252 tests/fm-session-lock-ancestry.test.sh 1414 +tests/fm-session-lock-identity.test.sh 1414 tests/fm-session-start.test.sh 156952 tests/fm-sessionstart-hook-live-e2e.test.sh 20 tests/fm-sessionstart-instruction-refresh-live-e2e.test.sh 22 diff --git a/bin/fm-turnend-guard.sh b/bin/fm-turnend-guard.sh index 7d9601308af..b414a77adb7 100755 --- a/bin/fm-turnend-guard.sh +++ b/bin/fm-turnend-guard.sh @@ -60,10 +60,12 @@ # the first fresh exhausted-failure epoch preserves the bounded progression, # while later fresh failed epochs consume it instead of resetting it; # 3. only when neither materializes is the auto-arm genuinely absent: re-block -# with the repair banner, bounded to FM_CLAUDE_TURNEND_BLOCK_BUDGET -# (default 3) consecutive blocks per session - safely below Claude Code's -# hard 8-consecutive-block override - then allow one loud attended -# fail-open only for an already verified failure episode. +# with the repair banner, accounting distinct failed auto-arm epochs up to +# FM_CLAUDE_TURNEND_BLOCK_BUDGET (default 3), while the independent +# FM_CLAUDE_TURNEND_STALL_BUDGET (default and maximum 7) bounds the actual +# uninterrupted blocked Stops safely before Claude Code's hard +# 8-consecutive-block override; then allow one loud attended fail-open for +# either a verified failure episode or the bounded stalled series. set -u SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -78,9 +80,14 @@ CURSOR_MODE=0 SYNC_WAIT_MS=${FM_CLAUDE_AUTOARM_SYNC_WAIT_MS:-800} EPOCH_FRESH=${FM_CLAUDE_AUTOARM_EPOCH_FRESH:-15} BLOCK_BUDGET=${FM_CLAUDE_TURNEND_BLOCK_BUDGET:-3} +STALL_BUDGET=${FM_CLAUDE_TURNEND_STALL_BUDGET:-7} case "$SYNC_WAIT_MS" in ''|*[!0-9]*) SYNC_WAIT_MS=800 ;; esac case "$EPOCH_FRESH" in ''|*[!0-9]*|0) EPOCH_FRESH=15 ;; esac case "$BLOCK_BUDGET" in ''|*[!0-9]*|0) BLOCK_BUDGET=3 ;; esac +case "$STALL_BUDGET" in + 1|2|3|4|5|6|7) : ;; + *) STALL_BUDGET=7 ;; +esac for arg in "$@"; do case "$arg" in @@ -207,22 +214,37 @@ fi # The Stop-owned auto-arm fires on the same Stop event. Give it a brief bounded # window to prove it owns recovery for this event epoch before consuming one of # Claude's bounded continuations. -budget_account_current_epoch() { - local current_epoch outcome old_session old_count old_epoch tmp initialized +# Consecutive-block accounting. COUNT is keyed on the auto-arm's epoch identity +# so one event epoch is charged at most once, which is correct while the auto-arm +# is running: a fresh epoch means the automatic path made a decision. It stops +# being a bound at all once the auto-arm stops writing entirely, because then the +# epoch never changes, COUNT never advances, and the guard re-blocks for as long +# as the fault lasts (measured: 172 consecutive blocked stops with COUNT frozen +# at 1 and the one attended alarm never reachable). STALLED counts the blocked +# stops themselves, so a completely inert automatic path is still bounded. +budget_account_current_epoch() { # [block] + local charge_block=${1:-} current_epoch outcome old_session old_count old_epoch + local old_stalled tmp initialized fm_lock_try_acquire "$BUDGET_LOCK" || return 1 current_epoch=$(sed -n '1s/^epoch=\([0-9][0-9]*\) .*/\1/p' "$STATE/.claude-autoarm-epoch" 2>/dev/null || true) outcome=$(sed -n '1s/^.*outcome=\([a-z][a-z-]*\) .*$/\1/p' "$STATE/.claude-autoarm-epoch" 2>/dev/null || true) initialized=0 COUNT=0 + STALLED=0 if [ -f "$BUDGET_FILE" ]; then old_session=$(sed -n '1s/^session=//p' "$BUDGET_FILE" 2>/dev/null || true) old_count=$(sed -n '2s/^count=//p' "$BUDGET_FILE" 2>/dev/null || true) old_epoch=$(sed -n '3s/^epoch=//p' "$BUDGET_FILE" 2>/dev/null || true) + old_stalled=$(sed -n '4s/^stalled=//p' "$BUDGET_FILE" 2>/dev/null || true) case "$old_count" in ''|*[!0-9]*) old_count=0 ;; esac + case "$old_stalled" in + ''|*[!0-9]*) old_stalled=0 ;; + esac if [ "$old_session" = "$SESSION_ID" ]; then COUNT=$old_count + STALLED=$old_stalled if [ -n "$current_epoch" ] && [ "$old_epoch" = "$current_epoch" ]; then : else @@ -230,6 +252,7 @@ budget_account_current_epoch() { fi fi fi + [ "$charge_block" != block ] || STALLED=$((STALLED + 1)) if [ ! -f "$BUDGET_FILE" ] || [ "${old_session:-}" != "$SESSION_ID" ]; then case "$outcome" in failed|failed-suppressed) @@ -244,7 +267,8 @@ budget_account_current_epoch() { esac fi tmp="$BUDGET_FILE.tmp.$$" - if ! printf 'session=%s\ncount=%s\nepoch=%s\n' "$SESSION_ID" "$COUNT" "$current_epoch" > "$tmp" 2>/dev/null \ + if ! printf 'session=%s\ncount=%s\nepoch=%s\nstalled=%s\n' \ + "$SESSION_ID" "$COUNT" "$current_epoch" "$STALLED" > "$tmp" 2>/dev/null \ || ! mv -f "$tmp" "$BUDGET_FILE" 2>/dev/null; then rm -f "$tmp" 2>/dev/null || true fm_lock_release "$BUDGET_LOCK" @@ -308,10 +332,40 @@ autoarm_owns_recovery() { return 1 } +# The episode the one attended fail-open is allowed for. Historically that was +# only a VERIFIED failure episode: the auto-arm recorded an exhausted failure and +# spent its one notice. That proof can only be produced by an auto-arm that runs, +# so a hook that never claims the home at all - the shape this bound exists for - +# could never reach the alarm no matter how many turns blocked. A long +# uninterrupted run of blocked stops is the second, mechanism-independent +# evidence that supervision is genuinely down, so it opens the same one alarm. +# Away mode is excluded from both, because the away daemon owns supervision and +# nobody is attending the alarm. +# +# The stalled episode is deduplicated by the counter itself, on the single value +# that ends the run, and NOT by the failure-alarm marker. That marker also tells +# the auto-arm to stop creating exit-2 continuations until a watcher is verified +# healthy again, which is right for a mechanism that ran and failed, and wrong +# here: an auto-arm that comes back would have its very first genuine wake +# swallowed. A series that resumes after a turn the guard let through is a new +# episode and alarms again on its own count. +fail_open_episode() { + FAIL_OPEN_REASON= + [ ! -e "$STATE/.afk" ] || return 1 + if [ "$STALLED" -eq $((STALL_BUDGET + 1)) ]; then + FAIL_OPEN_REASON=stalled + return 0 + fi + if [ "$COUNT" -gt "$BLOCK_BUDGET" ] && failure_episode_verified; then + FAIL_OPEN_REASON=failure + return 0 + fi + return 1 +} + terminal_fail_open() { - local pid role old_session old_count - [ "$COUNT" -gt "$BLOCK_BUDGET" ] || return 1 - failure_episode_verified || return 1 + local pid role old_session old_count old_stalled + fail_open_episode || return 1 [ ! -e "$FAILURE_ALARM" ] || return 1 # A live open generation claim is a concurrent recovery decision to step # aside for, exactly like the legacy live-owner case below. @@ -343,13 +397,18 @@ terminal_fail_open() { fi old_session=$(sed -n '1s/^session=//p' "$BUDGET_FILE" 2>/dev/null || true) old_count=$(sed -n '2s/^count=//p' "$BUDGET_FILE" 2>/dev/null || true) + old_stalled=$(sed -n '4s/^stalled=//p' "$BUDGET_FILE" 2>/dev/null || true) case "$old_count" in ''|*[!0-9]*) old_count=0 ;; esac + case "$old_stalled" in + ''|*[!0-9]*) old_stalled=0 ;; + esac + COUNT=$old_count + STALLED=$old_stalled role=$(fm_lock_role "$OWNER_LOCK" 2>/dev/null || true) if [ "$role" != terminal-check ] || [ "$old_session" != "$SESSION_ID" ] \ - || [ "$old_count" -le "$BLOCK_BUDGET" ] || ! failure_episode_verified \ - || [ -e "$FAILURE_ALARM" ]; then + || ! fail_open_episode || [ -e "$FAILURE_ALARM" ]; then fm_lock_release "$BUDGET_LOCK" fm_lock_release "$OWNER_LOCK" return 1 @@ -373,7 +432,14 @@ terminal_fail_open() { fm_lock_release "$OWNER_LOCK" return 2 fi - if ! (set -C; : > "$FAILURE_ALARM") 2>/dev/null; then + if ! budget_store_stall held 0; then + fm_lock_release "$BUDGET_LOCK" + fm_lock_release "$OWNER_LOCK" + return 1 + fi + if [ "$FAIL_OPEN_REASON" = failure ] \ + && ! (set -C; : > "$FAILURE_ALARM") 2>/dev/null; then + budget_store_stall held "$old_stalled" || true fm_lock_release "$BUDGET_LOCK" fm_lock_release "$OWNER_LOCK" return 1 @@ -394,12 +460,41 @@ failure_episode_verified() { esac } +# A stop this guard lets through is progress, so the consecutive-block series +# ends here. Only an uninterrupted run of blocked stops reaches the stall bound. +budget_store_stall() { + local mode=${1:-acquire} stalled=${2:-0} session count epoch tmp status=0 acquired=0 + [ -f "$BUDGET_FILE" ] || return 0 + if [ "$mode" = acquire ]; then + fm_lock_try_acquire "$BUDGET_LOCK" || return 1 + acquired=1 + fi + session=$(sed -n '1s/^session=//p' "$BUDGET_FILE" 2>/dev/null || true) + count=$(sed -n '2s/^count=//p' "$BUDGET_FILE" 2>/dev/null || true) + epoch=$(sed -n '3s/^epoch=//p' "$BUDGET_FILE" 2>/dev/null || true) + case "$count" in + ''|*[!0-9]*) count=0 ;; + esac + tmp="$BUDGET_FILE.tmp.$$" + if ! printf 'session=%s\ncount=%s\nepoch=%s\nstalled=%s\n' \ + "$session" "$count" "$epoch" "$stalled" > "$tmp" 2>/dev/null \ + || ! mv -f "$tmp" "$BUDGET_FILE" 2>/dev/null; then + status=1 + fi + rm -f "$tmp" 2>/dev/null || true + if [ "$acquired" -eq 1 ]; then + fm_lock_release "$BUDGET_LOCK" || status=1 + fi + return "$status" +} + i=0 while [ "$i" -lt $((SYNC_WAIT_MS / 100)) ]; do if autoarm_owns_recovery; then if fm_watcher_healthy "$STATE" "$WATCH" "$GRACE" "$FM_HOME"; then fm_failure_episode_reset "$STATE" || exit 2 fi + budget_store_stall || exit 2 exit 0 fi sleep 0.1 @@ -409,12 +504,13 @@ if autoarm_owns_recovery; then if fm_watcher_healthy "$STATE" "$WATCH" "$GRACE" "$FM_HOME"; then fm_failure_episode_reset "$STATE" || exit 2 fi + budget_store_stall || exit 2 exit 0 fi # The auto-arm genuinely failed to establish: consume the bounded re-block # budget before considering the verified one-time attended fail-open. -budget_account_current_epoch || block_stop +budget_account_current_epoch block || block_stop terminal_fail_open terminal_status=$? if [ "$terminal_status" -eq 0 ]; then @@ -425,8 +521,16 @@ if [ "$terminal_status" -eq 0 ]; then else NEED_DESC="X-mode relay polling active" fi - printf '{"systemMessage":"FIRSTMATE SUPERVISION IS GENUINELY DOWN: %s, the Stop-owned auto-arm exhausted its bounded retries and one failure notice, no watcher or automatic continuation exists, and the block budget is exhausted. Keep this session attended and diagnose the automatic Stop-hook and watcher startup before relying on unattended supervision."}\n' "$NEED_DESC" + if [ "$FAIL_OPEN_REASON" = stalled ]; then + printf '{"systemMessage":"FIRSTMATE SUPERVISION IS GENUINELY DOWN: %s, and this turn boundary has now blocked %s times in a row without the Stop-owned auto-arm ever claiming this home, so the automatic path is not running at all. Keep this session attended and diagnose the automatic Stop-hook and watcher startup before relying on unattended supervision."}\n' \ + "$NEED_DESC" "$STALLED" + else + printf '{"systemMessage":"FIRSTMATE SUPERVISION IS GENUINELY DOWN: %s, the Stop-owned auto-arm exhausted its bounded retries and one failure notice, no watcher or automatic continuation exists, and the block budget is exhausted. Keep this session attended and diagnose the automatic Stop-hook and watcher startup before relying on unattended supervision."}\n' "$NEED_DESC" + fi + exit 0 +fi +if [ "$terminal_status" -eq 2 ]; then + budget_store_stall || exit 2 exit 0 fi -[ "$terminal_status" -eq 2 ] && exit 0 block_stop diff --git a/docs/architecture.md b/docs/architecture.md index 83126a8f5ba..2401de52b26 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -195,6 +195,12 @@ Crewmates never intentionally touch your project clone; [treehouse](https://gith For ship and scout work, `fm-spawn.sh` refuses to launch unless the resolved task path is a real git worktree root that is distinct from the project primary checkout. `fm-spawn.sh` also owns the base-freshness boundary for every fresh ship and scout: no worker starts until its clean task worktree matches the fetched tip of origin's resolved default branch, and any unsafe or unverifiable base stops the spawn. Its header owns the exact refusal mechanics, while `tests/fm-spawn-pool-base-freshen.test.sh` owns the portable regression coverage. +That base stays origin's tip even on a project whose local default branch is ahead of it, because a branch destined for an upstream PR must never carry local-only commits; the spawn only warns about the divergence. +The cost of that choice lands later, when such a branch is merged into the local default branch, so `bin/fm-merge-local.sh` refuses any landing that would drop a commit the local default branch has and the task branch does not, names those commits, and points at the `git merge ` reconciliation inside the task branch. +Its `--drop-local-commits` escape hatch exists for the case where losing them is the intent: never the default, never silent, recorded in `state/.local-merge-drop` before the branch moves, and destructive enough to need the captain's explicit word. +Recovery does not rest on the reflog, which expires and is then pruned by `git gc`: the drop pins the pre-reset tip under its own `refs/fm-dropped//` rescue ref in the project before moving the branch, and refuses to move the branch when that ref cannot be planted. +Dropped commits therefore stay in the project for as long as their ref does, so successive drops on one task coexist and releasing one is an explicit, per-drop act that leaves every other rescue alone. +Its header owns the exact ref naming, drop record, and release command, and `tests/fm-merge-local.test.sh` owns the regression coverage. The firstmate repo has one extra exposure because it can dispatch crewmates to work on itself. Its operating checkout (`FM_ROOT`) and the disposable crewmate worktrees are all linked git worktrees of the same repository, so the valid discriminator is branch state, not whether the checkout is linked. @@ -299,7 +305,8 @@ Every GitHub refusal states what it could not observe as plainly as what it did, A confirmed merge leaves a durable role-routed outcome instead of living only in the merging agent's memory, and [`bin/fm-merge-outcome-lib.sh`](../bin/fm-merge-outcome-lib.sh)'s header owns its destination, shape, identity, normal-case deduplication, and at-least-once recovery. The same emitter handles a merge firstmate performed and one its poll detected, while the watcher immediately delivers the emitter's local actionable poll row. Teardown is fail-closed for ship worktrees: dirty worktrees refuse, and committed work must be landed before the worktree is returned. -[`bin/fm-teardown.sh`](../bin/fm-teardown.sh)'s header owns the landed-work proofs, PR-discovery fallback, and stale-lock recovery procedure. +Those worktree-scoped checks and the return itself apply only while the pool slot is still provably this task's: a sanctioned teardown rerun that finds the slot already returned or reissued to a newer task skips them instead of resetting the live tenant's worktree. +[`bin/fm-teardown.sh`](../bin/fm-teardown.sh)'s header owns the landed-work proofs, PR-discovery fallback, stale-lock recovery procedure, and the rerun tenant-safety ownership signals. ## Optional Relay diff --git a/docs/configuration.md b/docs/configuration.md index 15859ac1d87..edea9018739 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -843,7 +843,9 @@ FM_GUARD_GRACE=300 # seconds before guard warnings, arm health checks, and FM_CLAUDE_AUTOARM_ATTEMPTS=2 # bounded Stop-owned arm attempts per Claude auto-arm cycle; accepted values are 1, 2, or 3 FM_CLAUDE_AUTOARM_SYNC_WAIT_MS=800 # milliseconds the --claude turn-end guard waits for watcher health, an open Stop auto-arm generation claim, or a fresh epoch before deciding recovery ownership or failure progression FM_CLAUDE_AUTOARM_EPOCH_FRESH=15 # seconds a recorded auto-arm outcome remains eligible for the current event epoch's recovery or failure decision -FM_CLAUDE_TURNEND_BLOCK_BUDGET=3 # consecutive --claude guard re-blocks before the verified one-time attended fail-open; safely below Claude Code's 8-block override +FM_CLAUDE_AUTOARM_TRACE= # opt-in diagnostic; any non-empty value names on stderr the gate that ends a Claude Stop auto-arm run inert before it claims the cycle, changes no decision, and is unset in normal operation +FM_CLAUDE_TURNEND_BLOCK_BUDGET=3 # distinct failed auto-arm epochs accounted before the verified one-time attended fail-open; the separate stall budget below bounds consecutive blocks before Claude Code's override +FM_CLAUDE_TURNEND_STALL_BUDGET=7 # consecutive --claude guard re-blocks, counted unkeyed, before the same one-time attended fail-open opens without any auto-arm failure evidence; values above 7 clamp to 7 so the alarm always precedes Claude's eight-block override FM_ARM_CONFIRM_TIMEOUT=10 # seconds fm-watch-arm waits to confirm a fresh watcher before reporting FAILED; default 30 on Git Bash/MSYS FM_ARM_ATTACH_POLL=0.5 # seconds between checks while fm-watch-arm is attached to an existing healthy watcher cycle FM_OPENCODE_ARM_READY_TIMEOUT_MS=12000 # milliseconds the OpenCode primary watcher plugin waits for an arm attempt to report started, healthy, wake, or failure; default 35000 on Windows to stay above the MSYS confirm budget diff --git a/docs/herdr-backend.md b/docs/herdr-backend.md index 390e8f15172..c74c5b3b0ce 100644 --- a/docs/herdr-backend.md +++ b/docs/herdr-backend.md @@ -233,6 +233,7 @@ The poll density bounds the residual possibility of an extremely fast complete t `pane read --lines N` can return empty output when N is below the viewport height. The capture owner requests at least 200 lines from Herdr and trims locally to the caller's bound. +It also requests ANSI and strips it locally for plain-text callers, because Herdr 0.8.0's text-format recent read can visibly scroll an idle alt-screen pane while harvesting history; the ANSI path returns retained rows without that harvest. This generous floor is required for small composer and peek reads. Herdr's native agent state can read idle while a harness waits on its own long foreground tool. diff --git a/docs/scripts.md b/docs/scripts.md index c316a2808ac..dca6b8c81ed 100644 --- a/docs/scripts.md +++ b/docs/scripts.md @@ -41,7 +41,7 @@ The shared no-mistakes gate refusal for fleet lifecycle entrypoints is summarize | `fm-ensure-agents-md.sh` | Ensure a project's real `AGENTS.md`, its `CLAUDE.md` `@AGENTS.md` pointer, and the canonical self-governance section | | `fm-guard.sh` | Warn on primary-checkout tangles, pending queued wakes, and unhealthy supervision | | `fm-primary-scope-lib.sh` | Shared marker-or-plain-checkout primary-home predicate for tracked hooks | -| `fm-session-lock-lib.sh` | Shared session-lock harness identity (ancestry walk and holder liveness) for fm-lock.sh and the Claude Stop auto-arm | +| `fm-session-lock-lib.sh` | Shared session-lock harness identity (ancestry walk, holder liveness, the durable session-identity binding a reparented worker pool cannot reach through ancestry, and the delivering Claude session's own identity) for fm-lock.sh and the Claude Stop auto-arm | | `fm-claude-stop-autoarm.sh` | Claude Stop `asyncRewake` hook owning tokenless watcher continuity with single-flight exit-2 rewake (docs/watcher-continuity.md) | | `fm-turnend-guard.sh` | Shared primary turn-end guard predicate so no turn ends blind (docs/turnend-guard.md) | | `fm-turnend-guard-grok.sh` | Grok Stop-hook adapter for the primary turn-end guard | @@ -65,7 +65,7 @@ The shared no-mistakes gate refusal for fleet lifecycle entrypoints is summarize | `backends/cmux.sh` | Experimental cmux session-provider adapter | | `fm-config-push.sh` | Push declared inherited local material to live local or remote secondmates and send the placement-specific config reread when changed | | `fm-project-mode.sh` | Resolve a project's registered delivery posture from `data/projects.md` for fleet sync and home seeding | -| `fm-merge-local.sh` | Fast-forward a `local-only` project's local default branch after approval | +| `fm-merge-local.sh` | Fast-forward a `local-only` project's local default branch after approval, refusing a landing that would drop local-only commits | | `fm-review-diff.sh` | Review a crewmate branch or resolved PR head against the authoritative base | | `fm-marker-lib.sh` | Compatibility entry point for the from-firstmate carrier owned by `fm-operational-input.sh` | | `fm-task-inbox-lib.sh` | Single owner of durable steering-inbox records, acknowledgement, doorbells, and the delivery-attempt ladder | diff --git a/docs/turnend-guard.md b/docs/turnend-guard.md index 134c2f5dc41..a8e8cbe61da 100644 --- a/docs/turnend-guard.md +++ b/docs/turnend-guard.md @@ -85,9 +85,15 @@ A legacy build's lock-holding claim (recognizable by its `autoarm` role file) st Fresh `failed` and `failed-suppressed` outcomes enter or advance the failure progression instead of acting as unconditional recovery proof. The auto-arm itself rechecks the healthy watcher predicate and retries a bounded number of times before reporting a genuine failure. The first fresh exhausted-failure epoch preserves its handoff without consuming a blocked-stop count, while later fresh failed epochs advance the same monotonic progression instead of resetting it. -When none of those proofs appears, it re-blocks up to `FM_CLAUDE_TURNEND_BLOCK_BUDGET` times (default 3, below Claude's 8-block override). +When none of those proofs appears, it accounts distinct failed auto-arm epochs up to `FM_CLAUDE_TURNEND_BLOCK_BUDGET` (default 3) before the verified failure path may fail open. +That budget is keyed on the auto-arm's epoch identity, so it bounds an auto-arm that RUNS but keeps failing, and it bounds nothing at all once the auto-arm stops writing entirely: the epoch never changes, the count never advances, and the guard re-blocks for as long as the fault lasts (2026-08-27: 172 consecutive blocked stops over five hours with the count frozen at 1 and no alarm ever reachable). +`FM_CLAUDE_TURNEND_STALL_BUDGET` (default and maximum 7) therefore counts the blocked stops themselves, unkeyed, and the stop that passes it opens the same one attended alarm without requiring the verified failure episode below, because that evidence can only be produced by an auto-arm that runs. Larger overrides clamp to 7, so every supported path opens on the eighth Stop attempt, after seven actual blocks and before Claude's hard override after eight consecutive blocks. +That episode is deduplicated by the counter alone and never writes the failure-alarm marker, because that marker also suppresses the auto-arm's exit-2 continuations until positive watcher recovery and would swallow the first genuine wake of an auto-arm that comes back. +A stop this guard lets through ends that run, so only an uninterrupted series reaches the bound, and away mode is excluded from both because the away daemon owns supervision and nobody is attending the alarm. +The counter reset is committed while the budget lock is already held; if the verified-failure alarm marker cannot be published, the prior stalled value is restored and the Stop remains blocked, so publication failure cannot restart or strand the bounded series. In Claude mode, positive watcher recovery clears the block budget, failure notice, and attended alarm together under the existing budget lock before either hook reports ordinary recovery. -The one loud attended fail-open is available only when the auto-arm has recorded an exhausted failure, its one notice is already consumed, the block budget is exhausted, and a final check finds neither a healthy watcher nor an automatic continuation. +The one loud attended fail-open is available when the auto-arm has recorded an exhausted failure, its one notice is already consumed, the block budget is exhausted, and a final check finds neither a healthy watcher nor an automatic continuation - or, independently of any auto-arm evidence, when the run of consecutive blocked stops passes the stall budget above. +Its alarm names which of the two applied, so a completely inert automatic path is not reported as an exhausted one. Each epoch identity is accounted at most once under the budget lock. Whenever both coordination locks are needed, positive auto-arm recovery and the terminal check acquire the auto-arm owner lock before the budget lock. After that alarm, the Stop auto-arm suppresses further exit-2 continuations until positive watcher recovery, so the final fail-open remains reachable. @@ -160,6 +166,7 @@ That warning uses `bin/fm-supervision-instructions.sh --repair-line`, so it alwa ## Regression coverage `tests/fm-turnend-guard.test.sh` covers the predicate, main and secondmate primary scope, child-worktree exclusion, `FM_HOME` and `FM_STATE_OVERRIDE` precedence, the live-lock and fresh-beacon guard predicate, the cooperative `--claude` open-generation claim wait, monotonic failed-epoch progression, bounded attended fail-open, post-alarm continuation suppression, positive recovery reset, generation and legacy claim cases that must block or clear instead of allowing a blind stop, Pi logical-run latching, missing-`jq` behavior, all five primary registrations, Grok native and legacy selection, typed field precedence, malformed input, and exactly-one-path safety. +It also covers the consecutive-block bound over a frozen ledger, asserting that the epoch-keyed count stays frozen so the case cannot pass vacuously, that a stop the guard lets through ends the run, and that away mode never opens the alarm. `tests/fm-guard-stale-banner.test.sh` covers the pull-guard predicate, including the persistent-model fresh-leftover-beacon negative control, the auto-arm model's healthy fresh-beacon-without-a-watcher case and stale-beacon alarm, and the extension model's live-watcher path, ownership-qualified fresh hand-off, held-lock failures, independently broken ownership signals, stale-beacon alarm, queued-wake warning, and Pi and pi-signed harness routing. It also covers true-reason banner wording and reason-keyed episode dedup surviving a beacon mtime change. `tests/fm-cursor-primary.test.sh` covers the Cursor park end to end over real processes with no harness installed: each tracked Claude-shaped entrypoint standing down on a Cursor payload, both follow-up sources, the bounded repair nag and its reset, the nested loop bounds, supersession, away-mode and lock-ownership inertness, Pi-host stand-down without Cursor identity and continued parking when `PI_CODING_AGENT` leaks alongside `CURSOR_AGENT` or `CURSOR_INVOKED_AS`, child-worktree exclusion, and that the adapter never exits 2. diff --git a/docs/verification/runtime-backends.md b/docs/verification/runtime-backends.md index 46cb9b27e70..12545051f69 100644 --- a/docs/verification/runtime-backends.md +++ b/docs/verification/runtime-backends.md @@ -294,7 +294,7 @@ The CLI matrix was checked directly: | Explicit session routing | `herdr ... --session ` | Reached the named session even while another server was running. | | Literal send | `herdr pane send-text --session ` | Left text unsubmitted until Enter. | | Keys | `herdr pane send-keys enter|escape|ctrl+c --session ` | Enter and Escape worked; Ctrl-C interrupted foreground work. | -| Capture | `herdr pane read --source recent --lines N` | Small N could return empty below viewport height; a 200-line request plus local trim was stable. | +| Capture | `herdr pane read --source recent --lines N --format ansi` | Small N could return empty below viewport height, so a 200-line request plus local trim was stable; ANSI plus local stripping also avoided the visible idle alt-screen history harvest caused by Herdr 0.8.0's text-format recent read. | | Native state | `herdr agent get ` | Working and done transitions were visible on some harnesses; live Claude Code 2.1.236 on Herdr 0.8.0 kept `agent_status=idle` for an entire landed turn, including a multi-second tool call, so submit confirmation falls through to the shared composer verdict. Native `busy` remains positive activity evidence, while native `idle` cannot close a turn and the adapter's semantic lifecycle decides worker state. | | Restart | guarded named-session stop then start | Workspace, tab, pane, and labels persisted; the agent process and registration did not. | | Close | `herdr pane close --session ` | The exact one-pane task tab closed; closing a final tab could remove the workspace. | diff --git a/docs/verification/supervision.md b/docs/verification/supervision.md index 927c500562c..ddab873cfb9 100644 --- a/docs/verification/supervision.md +++ b/docs/verification/supervision.md @@ -61,6 +61,53 @@ The third is recorded below. | Codex | codex-cli 0.146.0 | `source=startup` under `codex exec`, token quoted back | Not reachable from a tracked project registration; see the limit below | `codex exec resume --last` reports `source=resume` | | Pi | 0.82.0 | `source=startup`, token quoted back in both `-p` and the TUI | `/new` raises `session_start` reason `new`, which the extension maps to `clear`; `/compact` raises `session_compact`, and both freshly injected source-stamped tokens were quoted back | `pi -c` reports reason `startup`, not `resume` | +### Session identity behind the fleet lock + +The binding beside the fleet lock records which session acquired it when Claude exposes a corroborated identity, because process ancestry is not a stable session identity: Claude Code serves one session's hooks and tool calls from more than one worker pool, and a pool whose top process is reparented to init yields a contiguous harness run that never reaches the session's own lineage. +A newly written lock publishes that binding best effort, while an existing lock with a missing or invalid binding is backfilled only under the acquisition lock after current ancestry re-proves ownership; an already-valid binding is preserved. +Whether the harness supplies a usable identity at all is vendor behavior, so it was measured on 2026-08-15 against Claude Code 2.1.232 and 2.1.233 in a throwaway lab whose only `SessionStart` hook logged the delivered payload, the exported environment, and the full process ancestry. + +Six session opens were recorded, covering every source the run tier routes on. + +| Source | Delivery | `CLAUDE_PID` | Session process | Hook's parent | Ancestry resolves to | +| --- | --- | --- | --- | --- | --- | +| `startup` | 2.1.233 headless `-p` | 26545 | 26545 | 26545 | 26545 | +| `startup` | 2.1.233 interactive | 37003 | 37003 | 37003 | 37003 | +| `compact` | 2.1.233 interactive `/compact` | 37003 | 37003 | 37003 | 37003 | +| `startup` | 2.1.232 interactive | 72191 | 72191 | 72191 | 72191 | +| `clear` | 2.1.232 `/clear` | 72191 | 72191 | 72191 | 72191 | +| `resume` | 2.1.232 `--continue` | 19340 | 19340 | 19340 | 19340 | + +Two facts hold across all six, and both are what the durable binding rests on. +Every payload carried a `session_id`, byte-equal to the `CLAUDE_CODE_SESSION_ID` the session exported, with no mismatch. +`CLAUDE_PID` named the session process itself, and the hook command ran as a direct child of that process. + +The second fact bounds the defect rather than describing it: on this path ancestry already reaches the session, so a foreground session is never refused its own lock at its own session open. +The refusal observed in the field came from a session hosted in a Claude Code background job, whose calls are served by a reparented pool (`claude bg-spare` under `claude bg-pty-host`, itself reparented to init) that stops short of the lock owner. +That is why the binding is required and why `CLAUDE_PID` equality alone would not answer it. +One reparented pool was measured, on the TOOL CALL path, from the refused session itself: + +| Path | `CLAUDE_PID` | Process it names | That process's parent | Observed harness ancestry | `state/.lock` | +| --- | --- | --- | --- | --- | --- | +| tool call served by a reparented worker pool, 2026-08-15, Claude Code 2.1.233 | 27316 | 27316 `claude bg-spare --bg-spare /tmp/cc-daemon-501/51f9f9bc/spare/d6bfe5e1.claim.sock` | 27305 `claude bg-pty-host ... d6bfe5e1.pty.sock ...`, PPID 1 | [27316, 27305] | 89187, live, `claude --dangerously-skip-permissions` | + +On that path `CLAUDE_PID` does name a pool process, and that process is a member of the current ancestry, so `fm_harness_session_is_ours` answers true while ancestry itself never reaches 89187. +That row confirms the premise for the tool-call path and for nothing else. +The case "the `SessionStart` hook itself fires from a reparented pool" is NOT covered by any recorded measurement: every session open in the six-row table above was served by the session process itself. +Were `CLAUDE_PID` to name the session rather than the pool on such a path, corroboration would fail and the read-only refusal would stand. +`CLAUDE_PID` remains load-bearing as corroboration either way, since a session identity inherited through the environment can otherwise be replayed by any process the session launched. + +`tests/fm-session-lock-identity.test.sh` pins the resulting logic portably with a deterministic process table, including the serialized legacy-binding backfill, valid-binding preservation, and concurrent-takeover refusal. +The core process-table logic was verified on 2026-08-15 under both GNU bash 3.2.57 on Darwin 25.6.0 and GNU bash 5.3.9 on aarch64 Alpine. +`tests/fm-sessionstart-hook-live-e2e.test.sh` refreshes only the session-open half of this record, the six-row table above: it reaches no reparented worker pool, so the tool-call row stays a hand-recorded measurement. +Run it after every Claude Code upgrade before trusting those six rows. +Its 2026-08-15 run against Claude Code 2.1.233 checked five real session opens and found a corroborated session identity on every one: + +```text +# claude 2.1.233 (Claude Code): session identity usable on 5 session-open(s) +ok - claude 2.1.233 (Claude Code): every session open carries a corroborated session identity for the fleet lock +``` + Two harness-specific consequences are load-bearing rather than incidental. Codex's interactive TUI fired no project `SessionStart` hook at all in the same lab where `codex exec` fired it reliably, which matches the earlier 2026-07-28 finding for 0.145.0. @@ -314,7 +361,11 @@ That inertness result is scoped to the builds it exercised: it did not establish The secondmate-home scope and manual-repair wake path were measured with Claude Code 2.1.207 on 2026-07-12, when a native background completion re-invoked the idle model with no human input. The current Stop-owned main/secondmate inclusion and child-worktree exclusion are covered deterministically by `tests/fm-claude-stop-autoarm.test.sh`. -Session-lock ownership in `bin/fm-session-lock-lib.sh` is decided against a session's whole contiguous harness ancestry rather than one chosen pid, so the Stop auto-arm reaches its lock owner wherever that owner sits: the outermost pid of Claude Code's multi-level `bg-spare` hook worker chain, or an inner pid when a harness-named daemon parents the session. +Session-lock ownership in `bin/fm-session-lock-lib.sh` rests on three proofs, and each gate asks only for the ones it can honestly use. +The ancestry proof is decided against a session's whole contiguous harness ancestry rather than one chosen pid, so it reaches its lock owner wherever that owner sits: the outermost pid of Claude Code's multi-level `bg-spare` hook worker chain, or an inner pid when a harness-named daemon parents the session. +The Stop auto-arm additionally accepts the session identity Claude delivers with the event, which turns on a strict `lock_pid = CLAUDE_PID` equality and therefore does decide on one chosen pid; [`watcher-continuity.md`](../watcher-continuity.md) owns that hook-event contract. +The session-start ownership gates instead additionally accept the recorded session identity described in [Session identity behind the fleet lock](#session-identity-behind-the-fleet-lock), because they carry no hook payload to reason about. +The Cursor turn-end guard deliberately still asks for ancestry only. Harness identity is read from the executable path and `argv[0]` as well as the command basename, because Claude Code's native installer names the per-session executable by its version (`.../share/claude/versions/2.1.220`): `ps -o comm=` reports that path on macOS and the bare version string on Linux, and neither basename names a harness. `tests/fm-session-lock-ancestry.test.sh` pins both platforms' reporting semantics behind a deterministic process table and runs the real Stop auto-arm in version-named, daemon-parented, and combined real process trees. `tests/fm-watch-arm.test.sh` runs real watcher and arm cycles against durable on-disk state to verify that a delivered reason survives until post-handling acknowledgement and stops replaying after acknowledgement, while an unrelated queue append cannot make a watcher cycle that delivered nothing look successful. @@ -486,6 +537,82 @@ ok - unacknowledged recovery is announced at most once per generation and the su FM_TEST_SUMMARY total=1 failed=0 skipped_gate=0 duration_ms=59357 ``` +### Claude Stop hook session identity, 2026-08-15 + +Claude Code 2.1.233 on macOS 25.6.0 was instrumented with a Stop hook that records the delivered payload, the hook's process ancestry, and its `CLAUDE_*` environment. +The pass covered a headless `claude -p` session and an interactive session in a pty, and both delivered these identity fields to the hook: + +```text +CLAUDE_PID= +CLAUDE_CODE_SESSION_ID= +payload {"session_id":"", ... ,"hook_event_name":"Stop"} +``` + +The same machine showed why ancestry alone is not sufficient. +Claude Code serves session commands from a shared per-user worker pool under `/tmp/cc-daemon-/`, and a pool whose owning session has exited survives with its top process reparented to init: + +```text +27305 1 claude bg-pty-host --bg-pty-host /tmp/cc-daemon-501/.../d6bfe5e1.pty.sock 200 50 -- .../versions/2.1.232 ... +27316 27305 claude bg-spare --bg-spare /tmp/cc-daemon-501/.../d6bfe5e1.claim.sock +25610 27316 /bin/zsh -c ... bin/fm-watch-arm.sh ... +``` + +A command served by that pool has a contiguous claude ancestry of `{27316, 27305}` and no path to the live session, whose pid the home's lock records. +The affected home's `state/.claude-autoarm-epoch` had not advanced for about fourteen hours while work was in flight, matching an identity gate that refuses every firing. + +`FM_CLAUDE_AUTOARM_TRACE=1 bin/fm-claude-stop-autoarm.sh` names the gate that ends a run, which is how an inert hook is diagnosed without changing any decision. +Against a fixture home whose lock names a live claude process the hook's own chain cannot reach, the three cases are: + +```text +autoarm-trace: inert: live session owns this home and neither identity proof applies (CLAUDE_PID=unset) # no identity exported, exit 0 +autoarm-trace: identity: proven by the delivering Claude session (pid ) # own session, exit 2, watcher armed +autoarm-trace: inert: live session owns this home and neither identity proof applies (CLAUDE_PID=) # foreign session, exit 0 +``` + +Those three lines are the strings the current code emits, captured on 2026-08-15 by running the hook with `FM_CLAUDE_AUTOARM_TRACE=1` against such a fixture home, once per case. + +The live guard ran twice against Claude Code 2.1.233 on 2026-08-15, after the detached-path alarm was added to it, the second time with the lab cleanup suppressed so its artifacts could be measured: + +```sh +FM_CLAUDE_LIVE_E2E=1 tests/fm-claude-stop-autoarm-live-e2e.test.sh +``` + +```text +ok - Claude 2.1.233 (Claude Code) live E2E reclaimed a stale session lock through session start, completed two tokenless Stop-owned rewake cycles, preserved the competing-live-owner boundary, and still delivers the session identity the ownership proof depends on +``` + +The lab registers two `asyncRewake` Stop hooks, the tracked auto-arm plus a probe that records what the harness hands a detached hook, and both delivery paths were measured rather than assumed. +The synchronous probe and the detached one each logged three records, matching the session's three Stops, so Claude Code runs EVERY registered `asyncRewake` Stop hook at a turn end rather than only the first, and the detached alarm has a delivery path that genuinely reaches it. +On both paths the session exported `CLAUDE_PID=20323`, exactly the pid `state/.lock` recorded, and `CLAUDE_CODE_SESSION_ID` equalled the delivered payload's `session_id`, both `1ea063c6-94c4-459e-ad19-8603c2bbfb6c`. +The auto-arm hook ran on those same Stops: `state/arm-ran` held two hook-owned cycles, `state/.claude-autoarm-epoch` recorded `epoch=4 owner_pid=25124 outcome=rewake`, no owner lock was left behind, and no turn was blocked for ending blind. +The tokenless-cycle count in that guard is not a property of the identity proof, and it reproduces identically on the unpatched hook. +It was keyed on a drain count until 2026-08-27, when the tracked `SessionStart` hook's own digest made drains stop being a stable count of Stop-owned cycles; it now ends the fixture's in-flight need on the cycle count the case actually measures. + +### Away-mode return, 2026-08-27 + +Claude Code 2.1.247 on macOS 25.6.0 arm64 with tmux 3.7b, in throwaway project and home directories with an isolated lab tmux session for the away daemon's captain pane. +No live fleet home, worktree, or tmux session was touched. + +The lab enters away mode with the real `bin/fm-afk-launch.sh`, ends a turn while the away daemon owns supervision, leaves through the real `bin/fm-afk-return.sh`, and then measures the next Stop. +Its per-Stop probe records the away-mode flag and the epoch ledger, so the ORDER of the episode is evidence rather than assumption. + +```sh +FM_CLAUDE_LIVE_E2E=1 tests/fm-claude-stop-autoarm-live-e2e.test.sh +``` + +```text +ok - Claude 2.1.247 (Claude Code) live E2E reclaimed a stale session lock through session start, completed two tokenless Stop-owned rewake cycles, preserved the competing-live-owner boundary, and still delivers the session identity the ownership proof depends on +ok - Claude 2.1.247 (Claude Code) live E2E entered away mode, crossed a turn boundary while the away daemon owned supervision, returned through the real return gate, and had the Stop-owned auto-arm claim the home again at the next Stop with no model-issued arm +``` + +What that pass establishes, and what it does not: + +- The away-mode episode really crossed a turn boundary: the probe logged a Stop with the flag still set, and the auto-arm recorded nothing while it was set, which is its contract. +- The first Stop after the return claimed the home on its own, wrote a fresh generation, armed a cycle, and the model issued no arm command of its own. +- It does NOT discriminate the third membership proof, because a lab session's lock and exported session pid are the same process; the shape where they differ is pinned portably in `tests/fm-session-lock-identity.test.sh` and `tests/fm-claude-stop-autoarm.test.sh`, each asserting the inert divergence first so the case cannot pass vacuously. +- It does NOT exercise the consecutive-block bound, which needs many blocked turns; that is pinned portably in `tests/fm-turnend-guard.test.sh`. + + Deterministic entry points: ```sh diff --git a/docs/watcher-continuity.md b/docs/watcher-continuity.md index 968ed0821f8..df3c10e5d13 100644 --- a/docs/watcher-continuity.md +++ b/docs/watcher-continuity.md @@ -12,6 +12,14 @@ Pi same-process session replacement follows the generation-owner contract in `.p Cursor's `.cursor/hooks.json` `stop` hook (`bin/fm-turnend-guard-cursor.sh`) owns routine tokenless re-arm for a Cursor primary by parking that awaited hook on `bin/fm-watch-arm.sh` and returning an actionable close as one follow-up; [`turnend-guard.md`](turnend-guard.md#harness-integrations) owns its Pi-host stand-down, loop bounds, and supersession baton. Claude's `.claude/settings.json` Stop `asyncRewake` hook (`bin/fm-claude-stop-autoarm.sh`) owns routine tokenless re-arm. The hook fires on every Stop, and an eligible primary with supervision need admits one home-scoped owner that foregrounds `bin/fm-watch-arm.sh` inside the hook-owned process tree. +The hook may act only for the session that holds this home's lock, and `bin/fm-session-lock-lib.sh` owns all three proofs of that ownership. +Harness ancestry is the first proof and is unchanged. +The delivering session's own pid is the second, and it is required because Claude Code serves hook commands from a shared per-user worker pool whose top process is reparented to init once the session that first started it exits, which can leave a hook whose contiguous harness ancestry never reaches its own live session. +That second proof is a conjunction: the delivered payload names a session id, the delivering session exported that same session id, the session pid it exported is exactly the pid this home's lock records, and that pid is still a live Claude process. +Requiring pid equality is stricter than ancestry membership rather than weaker, so a session that does not hold the lock still cannot claim the home and several firstmate homes on one machine stay independent. +The lock's own durable session binding is the third, and it is required because `bin/fm-lock.sh` records the OUTERMOST pid of the contiguous harness run that acquired the lock, which for one session is not always the pid the harness exports for a later event; when they differ, ancestry cannot reach the recorded owner either and the first two proofs have nothing left to match on, which leaves the hook permanently inert on a home whose lock is alive and genuinely its own. +That third proof keeps the payload-and-environment corroboration of the second and replaces its pid equality with the binding: the record beside the lock must name exactly the pid the lock holds, the session it names must be this delivering session, and that pid must still be a live harness process. +A foreign session therefore still carries its own id and stays out, and a lock with no binding keeps exactly the two-proof behavior. A numeric session-lock owner that fails the shared `fm_harness_pid_alive` predicate is reclaimed through `bin/fm-lock.sh` before auto-arm state changes, while a live owner, absent lock, or malformed lock keeps the competing hook inert. The stale-owner claim occurs only after the existing AFK and supervision-need gates pass. After each non-actionable arm close, the hook rechecks the identity-matched watcher lock and fresh beacon before retrying a bounded number of times. @@ -109,8 +117,13 @@ The same suite covers ordinary same-process session replacement for `/new`, `/re `tests/fm-subagent-pretool-check.test.sh` proves Claude retains only the non-status Bash seatbelts. `tests/fm-claude-stop-autoarm.test.sh` covers the auto-arm's scope, stale and live session owners, unchanged AFK and need boundaries, single-flight, bounded failure retries, benign live-watcher cycle ends, one-notice failure episodes, and exit-2 translation. It also covers generation-claim single-flight, stuck-claim supersession, superseded-owner silence, notice-marker refusal and retry, ownership-atomic episode reset, and the legacy upgrade shim; [`turnend-guard.md`](turnend-guard.md) owns those behavior contracts. +The same suite runs all three ownership proofs over real processes, including a hook served from a worker chain that cannot reach its own session, a home whose lock records a DIFFERENT process of the delivering session, the foreign-session refusal that is the safety counter-proof, an unconfirmed, absent, or malformed delivered identity, a missing or malformed recorded owner, and a main home's session refused against a secondmate home on the same machine. +It also covers the first Stop after an away-mode return, where the episode's own terminal ledger entry must not freeze the next claim. +`tests/fm-session-lock-identity.test.sh` owns the proofs themselves, each case asserting the inert divergence first so it cannot pass vacuously. +It also covers abandoned single-flight claims: a legacy lock-holding claim whose owner is gone is reclaimed once so a lapsed home re-arms, while one still genuinely deciding and the guard's own terminal check keep the gate closed ([`turnend-guard.md`](turnend-guard.md) owns that boundary). `FM_CLAUDE_LIVE_E2E=1 tests/fm-claude-stop-autoarm-live-e2e.test.sh` starts with the reproduced stale-lock state, runs session start first, completes two tokenless cycles, and checks the competing-live-owner negative control. -`tests/fm-turnend-guard.test.sh` covers the cooperative `--claude` guard, including monotonic failed-epoch progression, the integrated bounded fail-open, post-alarm continuation suppression, and positive recovery reset; [`turnend-guard.md`](turnend-guard.md#regression-coverage) lists that suite's full generation and legacy claim coverage. +It also fails, naming the installed harness and version, if Claude Code stops delivering the session identity the second proof depends on, on the synchronous Stop path or on the detached `asyncRewake` path the hook itself runs on. +`tests/fm-turnend-guard.test.sh` covers the cooperative `--claude` guard, including monotonic failed-epoch progression, the integrated bounded fail-open, post-alarm continuation suppression, and positive recovery reset; [`turnend-guard.md`](turnend-guard.md#regression-coverage) lists that suite's full generation, legacy claim, and abandoned-claim coverage. ## Active limits and verification diff --git a/tests/fm-backend-herdr.test.sh b/tests/fm-backend-herdr.test.sh index 426d7ceac26..da9e9e7938c 100755 --- a/tests/fm-backend-herdr.test.sh +++ b/tests/fm-backend-herdr.test.sh @@ -2909,9 +2909,12 @@ test_capture_calls_pane_read() { out=$( PATH="$fb:$PATH" FM_HERDR_LOG="$log" FM_HERDR_RESPONSES="$resp" \ bash -c '. "$0/bin/backends/herdr.sh"; fm_backend_herdr_capture default:w1:p2 250' "$ROOT" ) [ "$out" = $'line one\nline two\nline three' ] || fail "capture did not pass through pane read output, got '$out'" - assert_contains "$(cat "$log")" "HERDR_SESSION=default"$'\x1f''pane'$'\x1f''read'$'\x1f''w1:p2'$'\x1f''--source'$'\x1f''recent'$'\x1f''--lines'$'\x1f''250' \ - "capture did not call pane read with the right pane id and line bound" - pass "fm_backend_herdr_capture: calls 'pane read --source recent --lines N' with the session set" + # --format ansi is load-bearing, not cosmetic: on herdr 0.8.0 a text-format + # recent read of an idle alt-screen agent triggers the wheel-event history + # harvest and the pane visibly scrolls for seconds (herdrdev/herdr#2669). + assert_contains "$(cat "$log")" "HERDR_SESSION=default"$'\x1f''pane'$'\x1f''read'$'\x1f''w1:p2'$'\x1f''--source'$'\x1f''recent'$'\x1f''--lines'$'\x1f''250'$'\x1f''--format'$'\x1f''ansi' \ + "capture did not call pane read with the right pane id, line bound, and the harvest-skipping ansi format" + pass "fm_backend_herdr_capture: calls 'pane read --source recent --lines N --format ansi' with the session set" } test_capture_works_around_small_lines_bug() { diff --git a/tests/fm-calm-pi-extension.test.sh b/tests/fm-calm-pi-extension.test.sh index 2efcc1b4e8a..7e25eafb76a 100755 --- a/tests/fm-calm-pi-extension.test.sh +++ b/tests/fm-calm-pi-extension.test.sh @@ -28,6 +28,19 @@ record_pi_version_evidence() { [ -n "$version" ] || fail "$context could not determine the installed Pi version" } +# Names the tools that are actually absent so a skipped E2E can never read as a +# passing one. The three E2E cases below are the only coverage of Calm's real +# terminal behavior, and tmux is commonly installed outside a trimmed PATH while +# pi is not, so reporting "pi or tmux" for either case hid which one was missing. +# A run that silently skipped all three then looked like a clean pass, which is +# exactly how a Calm export defect was once misread as a bash-version regression. +missing_e2e_tools() { + local missing="" + command -v pi >/dev/null 2>&1 || missing="pi" + command -v tmux >/dev/null 2>&1 || missing="${missing:+$missing and }tmux" + printf '%s\n' "$missing" +} + cleanup() { if command -v tmux >/dev/null 2>&1; then tmux -L "$TMUX_SOCKET" kill-server 2>/dev/null || true @@ -50,6 +63,22 @@ wait_for_text() { return 1 } +# Pane variant of wait_for_text: leaves the last capture in the caller's `pane` +# so the assertions right after a wait read the very screen the wait settled on, +# and returns non-zero once the deadline passes without appearing. The +# caller supplies its own failure message, so each wait names the state it was +# waiting for instead of letting a later, unrelated assertion misreport it. +wait_for_pane_text() { + local needle=$1 i=0 + while [ "$i" -lt 120 ]; do + pane=$(tmux -L "$TMUX_SOCKET" capture-pane -p -t "$TMUX_SESSION" -S - 2>/dev/null || true) + printf '%s\n' "$pane" | grep -Fq "$needle" && return 0 + sleep 0.05 + i=$((i + 1)) + done + return 1 +} + find_chrome() { local candidate if [ -n "${FM_CHROME_BIN:-}" ] && [ -x "$FM_CHROME_BIN" ]; then @@ -1620,7 +1649,7 @@ JS test_operational_followup_turn_e2e() { local project home config sessions version label case_name calm_state expected_notifications session_file pane i captain_line handled_line geometry_gap exact_session if ! command -v pi >/dev/null 2>&1 || ! command -v tmux >/dev/null 2>&1; then - echo "skip: pi or tmux not found for Pi operational follow-up E2E" + echo "skip: $(missing_e2e_tools) not found for Pi operational follow-up E2E" return 0 fi version=$(pi --version 2>/dev/null || true) @@ -1792,14 +1821,7 @@ TS tmux -L "$TMUX_SOCKET" new-session -d -s "$TMUX_SESSION" -x 160 -y 36 \ "cd '$project' && env FM_HOME='$home' PI_CODING_AGENT_DIR='$config' FM_OPERATIONAL_INPUT_SCRIPT='$OPERATIONAL_INPUT' PI_OFFLINE=1 pi --approve --no-context-files --no-skills --no-prompt-templates --no-extensions $extensions $session_arg; rc=\$?; printf '\nPI_EXIT=%s\n' \"\$rc\"; sleep 20" - i=0 - while [ "$i" -lt 120 ]; do - pane=$(tmux -L "$TMUX_SOCKET" capture-pane -p -t "$TMUX_SESSION" -S - 2>/dev/null || true) - printf '%s\n' "$pane" | grep -Fq 'followup-e2e.ts' && break - sleep 0.05 - i=$((i + 1)) - done - printf '%s\n' "$pane" | grep -Fq 'followup-e2e.ts' \ + wait_for_pane_text 'followup-e2e.ts' \ || fail "Pi follow-up $case_name case ($label) did not reach the ready composer" tmux -L "$TMUX_SOCKET" send-keys -t "$TMUX_SESSION" -l "/followup-e2e $label $shape" @@ -1817,11 +1839,18 @@ TS fail "Pi follow-up $label case did not process the monitoring notification" fi - pane=$(tmux -L "$TMUX_SOCKET" capture-pane -p -t "$TMUX_SESSION" -S - 2>/dev/null || true) + # Pi writes the session file before it paints the transcript, so capturing the + # pane the moment that file settles can catch the screen still on its start-up + # state and read as zero captain answers rather than one. Wait for the follow-up + # answer, which Pi renders after the captain answer, so every assertion below + # sees a fully painted transcript. Failing here first keeps an unpainted screen + # from being reported as a duplicate: the count below is taken only once this + # anchor holds, and is never satisfied by it, so a genuine duplicate still fails. + wait_for_pane_text "MONITOR_HANDLED_${label}_ONE" \ + || fail "Pi follow-up $label case never painted the follow-up answer on screen before the wait expired" [ "$(printf '%s\n' "$pane" | grep -Fc "CAPTAIN_ANSWER_$label" || true)" -eq 1 ] \ || fail "Pi follow-up $label case rendered a duplicate captain answer" assert_contains "$pane" "CAPTAIN_PROMPT_$label" "Pi follow-up $label case hid the genuine captain prompt" - assert_contains "$pane" "MONITOR_HANDLED_${label}_ONE" "Pi follow-up $label case did not render the intended processing result" if [ "$calm_state" = on ]; then assert_not_contains "$pane" "MONITOR_${label}_ONE" "Pi follow-up $label case rendered a Calm-hidden operational user row" if [ "$label" = exact_watcher ]; then @@ -1920,15 +1949,9 @@ JS printf '%s\n' on >"$home/config/calm" tmux -L "$TMUX_SOCKET" new-session -d -s "$TMUX_SESSION" -x 160 -y 36 \ "cd '$project' && env FM_HOME='$home' PI_CODING_AGENT_DIR='$config' FM_OPERATIONAL_INPUT_SCRIPT='$OPERATIONAL_INPUT' PI_OFFLINE=1 pi --approve --no-context-files --no-skills --no-prompt-templates --no-extensions -e ./.pi/extensions/fm-calm.ts -e ./followup-e2e.ts --session '$exact_session'; rc=\$?; printf '\nPI_EXIT=%s\n' \"\$rc\"; sleep 20" - i=0 - while [ "$i" -lt 120 ]; do - pane=$(tmux -L "$TMUX_SOCKET" capture-pane -p -t "$TMUX_SESSION" -S - 2>/dev/null || true) - printf '%s\n' "$pane" | grep -Fq 'MONITOR_HANDLED_exact_watcher_ONE' && break - sleep 0.05 - i=$((i + 1)) - done + wait_for_pane_text 'MONITOR_HANDLED_exact_watcher_ONE' \ + || fail "Pi restart lost the operational processing response" assert_contains "$pane" "CAPTAIN_PROMPT_exact_watcher" "Pi restart lost the genuine captain prompt" - assert_contains "$pane" "MONITOR_HANDLED_exact_watcher_ONE" "Pi restart lost the operational processing response" assert_not_contains "$pane" "FIRSTMATE WATCHER WAKE: signal: /home/fixture/github/kunchenguid/firstmate/state/oss-triage-t4.status" \ "Pi restart replayed the Calm-hidden exact watcher row" captain_line=$(printf '%s\n' "$pane" | grep -Fn 'CAPTAIN_ANSWER_exact_watcher' | tail -1 | cut -d: -f1) @@ -1974,7 +1997,7 @@ test_hidden_block_geometry_e2e() { local project home config sessions session_file snapshot expanded_snapshot calm_off_snapshot restarted_snapshot local version skill_line final_line gap i if ! command -v pi >/dev/null 2>&1 || ! command -v tmux >/dev/null 2>&1; then - echo "skip: pi or tmux not found for Pi Calm hidden-block geometry E2E" + echo "skip: $(missing_e2e_tools) not found for Pi Calm hidden-block geometry E2E" return 0 fi version=$(pi --version 2>/dev/null || true) @@ -3082,7 +3105,7 @@ JS test_interactive_terminal_e2e() { local project config home session_file export_file export_dom default_snapshot expanded_snapshot hidden_snapshot active_before_snapshot active_hidden_snapshot export_snapshot export_settled_snapshot restored_snapshot working_snapshot working_response_snapshot restarted_snapshot resumed_restored_snapshot hash_before hash_after now version chrome chrome_pid chrome_wait chrome_reap_wait active_wait active_screen_wait boat_frame_one boat_frame_two boat_resized_snapshot boat_focus_snapshot boat_cleared_snapshot boat_hull_line boat_sail_line boat_column_one boat_column_two boat_line boat_color_snapshot boat_color_line boat_water_snapshot boat_water_line boat_water_first boat_water_changed boat_narrow_snapshot boat_narrow_sails boat_freeze_snapshot boat_resume_snapshot boat_freeze_column boat_freeze_sail boat_resume_column boat_resume_sail if ! command -v pi >/dev/null 2>&1 || ! command -v tmux >/dev/null 2>&1; then - echo "skip: pi or tmux not found for Pi calm interactive E2E" + echo "skip: $(missing_e2e_tools) not found for Pi calm interactive E2E" return 0 fi version=$(pi --version 2>/dev/null || true) diff --git a/tests/fm-claude-stop-autoarm-live-e2e.test.sh b/tests/fm-claude-stop-autoarm-live-e2e.test.sh index c7e2cab880b..bb3cb4980b8 100755 --- a/tests/fm-claude-stop-autoarm-live-e2e.test.sh +++ b/tests/fm-claude-stop-autoarm-live-e2e.test.sh @@ -7,6 +7,23 @@ # dead owner; at least two tokenless auto-arm and rewake cycles then complete # with zero model-issued arm commands; and the cooperative guard consumes no # forced continuation while the hook's launch is healthy. +# A second isolated session then covers the away-mode return: a real away-mode +# episode is entered with bin/fm-afk-launch.sh, crosses a real turn boundary +# while the away daemon owns supervision, is left with bin/fm-afk-return.sh, and +# the very next Stop must bring the auto-arm back on its own with no model-issued +# arm command. That is the 2026-08-27 lapse, where the epoch ledger stayed frozen +# on the episode's terminal entry and every later turn boundary blocked instead. +# It also guards the vendor-emitted half of the hook's session-identity proof: +# the installed Claude Code must still deliver a session pid and session id to +# its Stop hooks, and the delivered payload must still name the same session, +# because bin/fm-session-lock-lib.sh cannot prove ownership from ancestry alone +# when Claude serves the hook from its shared worker pool. That proof is +# measured on BOTH delivery paths, synchronous and detached asyncRewake, because +# the hook that consumes it is registered asyncRewake and a release that kept +# the signals only on the synchronous path would return the fleet to the +# inert-hook bug with every other check still green. Losing either signal on +# either path fails here, naming the harness and version, instead of silently +# returning the whole fleet to the inert-hook bug. # The project and FM_HOME are isolated; Claude keeps using its existing managed # authentication. No live fleet home, worktree, or session is touched. # shellcheck disable=SC2016 # the model, not this test shell, reads the prompt text @@ -33,7 +50,17 @@ LIVE_OWNER_HOME="$LAB/live-owner-home" TRANSCRIPT="$LAB/claude.jsonl" CLAUDE_VERSION=$(claude --version) +AFK_HOME="$LAB/afk-home" +AFK_PROJECT="$LAB/afk-project" +AFK_TRANSCRIPT="$LAB/claude-afk.jsonl" +AFK_CAPTAIN_SESSION="fm-autoarm-live-afk-$$" + cleanup() { + if [ -d "$AFK_PROJECT" ]; then + FM_HOME="$AFK_HOME" FM_SUPERVISOR_BACKEND=tmux FM_SUPERVISOR_TARGET=unused \ + "$AFK_PROJECT/bin/fm-afk-launch.sh" stop >/dev/null 2>&1 || true + fi + tmux kill-session -t "$AFK_CAPTAIN_SESSION" >/dev/null 2>&1 || true rm -rf "$LAB" } trap cleanup EXIT @@ -46,8 +73,10 @@ cp -R "$ROOT/bin/." "$PROJECT/bin/" cp "$ROOT/.claude/settings.json" "$PROJECT/.claude/settings.json" # The lab keeps the real tracked .claude/settings.json SessionStart nudge, # Stop guard, and asyncRewake auto-arm registration. -# The only local hook records model-issued Bash calls without acquiring the -# session lock or otherwise changing lifecycle behavior. +# The local hooks only record model-issued Bash calls and what Claude hands a +# Stop hook on each delivery path, the synchronous one and the detached +# asyncRewake one the auto-arm itself runs on. Neither acquires the session lock +# or otherwise changes lifecycle behavior. cat > "$PROJECT/.claude/settings.local.json" <<'JSON' { "hooks": { @@ -58,11 +87,40 @@ cat > "$PROJECT/.claude/settings.local.json" <<'JSON' { "type": "command", "command": "\"$CLAUDE_PROJECT_DIR\"/bin/tool-logger.sh" } ] } + ], + "Stop": [ + { + "hooks": [ + { "type": "command", "command": "\"$CLAUDE_PROJECT_DIR\"/bin/stop-identity-probe.sh stop-identity.log" }, + { + "type": "command", + "command": "\"$CLAUDE_PROJECT_DIR\"/bin/stop-identity-probe.sh stop-identity-async.log", + "asyncRewake": true, + "timeout": 600 + } + ] + } ] } } JSON +# Records what the installed Claude Code actually hands a Stop hook. $1 names +# the log for this delivery path, so the synchronous and asyncRewake +# registrations never share a file. It exits 0 silently and holds no lock, so it +# changes no lifecycle behavior on either path. +cat > "$PROJECT/bin/stop-identity-probe.sh" <<'SH' +#!/usr/bin/env bash +PAYLOAD=$(cat 2>/dev/null || true) +{ + printf 'claude_pid=%s\n' "${CLAUDE_PID:-}" + printf 'env_session_id=%s\n' "${CLAUDE_CODE_SESSION_ID:-}" + printf 'payload=%s\n' "$PAYLOAD" +} >> "$FM_HOME/state/$1" 2>/dev/null +exit 0 +SH +chmod +x "$PROJECT/bin/stop-identity-probe.sh" + cat > "$PROJECT/bin/tool-logger.sh" <<'SH' #!/usr/bin/env bash P=$(cat 2>/dev/null || true) @@ -93,14 +151,16 @@ printf 'watcher: started pid=%s (beacon fresh)\n' "$$" printf 'stale: fixture-rapid-%s\n' "$N" exit 0 SH -# Drain fixture: session start invokes it once, then the model invokes it once -# per rewake. The third total drain ends the in-flight need after two complete -# Stop-owned cycles. +# Drain fixture: the in-flight need ends on the CYCLE count this test measures, +# not on a drain count. The tracked SessionStart hook runs one digest and the +# prompt has the model run a second, so drains alone are not a stable measure of +# Stop-owned cycles. Ending the need here, in the model's own handling turn, +# also keeps the last cycle from spending its bounded retry on a vanished need. cat > "$PROJECT/bin/fm-wake-drain.sh" <<'SH' #!/usr/bin/env bash N=$(cat "$FM_HOME/state/drain-count" 2>/dev/null || echo 0); N=$((N+1)); echo "$N" > "$FM_HOME/state/drain-count" echo "drain-run=$N" >> "$FM_HOME/state/drain-ran" -if [ "$N" -ge 3 ]; then +if [ "$(cat "$FM_HOME/state/arm-count" 2>/dev/null || echo 0)" -ge 2 ]; then rm -f "$FM_HOME/state/task.meta" fi printf 'stale: fixture-rapid drained\n' @@ -115,10 +175,71 @@ PROMPT='Run exactly `bin/fm-session-start.sh` with Bash as your first tool call. claude -p "$PROMPT" --dangerously-skip-permissions --effort low --output-format stream-json --verbose ) > "$TRANSCRIPT" 2>&1 || fail "Claude credentialed auto-arm session failed: $(tail -20 "$TRANSCRIPT")" +# Vendor-emitted session identity: the second ownership proof in +# bin/fm-session-lock-lib.sh reads these, so a release that stops delivering any +# of them must fail loudly here rather than degrade the whole fleet to an inert +# Stop hook. +IDENTITY_LOG="$HOME_DIR/state/stop-identity.log" +[ -s "$IDENTITY_LOG" ] || fail "Claude $CLAUDE_VERSION delivered no Stop hook identity at all" +PROBE_PID=$(sed -n 's/^claude_pid=//p' "$IDENTITY_LOG" | sed -n '1p') +PROBE_ENV_SESSION=$(sed -n 's/^env_session_id=//p' "$IDENTITY_LOG" | sed -n '1p') +PROBE_PAYLOAD=$(sed -n 's/^payload=//p' "$IDENTITY_LOG" | sed -n '1p') +case "$PROBE_PID" in + ''|*[!0-9]*) fail "Claude $CLAUDE_VERSION no longer exports a numeric CLAUDE_PID to its Stop hooks (got '$PROBE_PID')" ;; +esac +[ -n "$PROBE_ENV_SESSION" ] \ + || fail "Claude $CLAUDE_VERSION no longer exports CLAUDE_CODE_SESSION_ID to its Stop hooks" +# shellcheck source=bin/fm-session-lock-lib.sh +. "$ROOT/bin/fm-session-lock-lib.sh" # for fm_claude_payload_session_id +PROBE_PAYLOAD_SESSION=$(fm_claude_payload_session_id "$PROBE_PAYLOAD") \ + || fail "Claude $CLAUDE_VERSION no longer names a session_id in its Stop payload: $PROBE_PAYLOAD" +[ "$PROBE_PAYLOAD_SESSION" = "$PROBE_ENV_SESSION" ] \ + || fail "Claude $CLAUDE_VERSION delivered a Stop payload for session $PROBE_PAYLOAD_SESSION while exporting $PROBE_ENV_SESSION, so the two identity signals no longer corroborate each other" + +# The two proofs must still designate the same process: the session pid Claude +# exports is the pid this session recorded as the home's owner. Without that +# correspondence the identity proof would name a process the lock never records, +# and the hook would go inert again. The decision logic itself is pinned +# portably by tests/fm-claude-stop-autoarm.test.sh with real processes. +RECORDED_OWNER=$(cat "$HOME_DIR/state/.lock" 2>/dev/null || true) +[ "$PROBE_PID" = "$RECORDED_OWNER" ] \ + || fail "Claude $CLAUDE_VERSION exported session pid $PROBE_PID while this session recorded owner $RECORDED_OWNER, so the session-identity proof no longer matches the recorded owner" + +# The same proof on the delivery path the auto-arm actually runs on. Claude +# registers bin/fm-claude-stop-autoarm.sh with "asyncRewake": true, so it is +# served detached, which is the very context the shared worker pool broke. A +# release that kept exporting identity to synchronous Stop hooks and stopped on +# the detached ones would leave every check above green while the hook went +# inert again, so the detached path carries its own alarm. An absent log is that +# regression, not a reason to skip. +ASYNC_IDENTITY_LOG="$HOME_DIR/state/stop-identity-async.log" +async_wait=0 +while [ "$async_wait" -lt 60 ] && [ ! -s "$ASYNC_IDENTITY_LOG" ]; do + async_wait=$((async_wait + 1)) + sleep 0.5 +done +[ -s "$ASYNC_IDENTITY_LOG" ] \ + || fail "Claude $CLAUDE_VERSION delivered no Stop hook identity at all on the detached asyncRewake path, the path bin/fm-claude-stop-autoarm.sh runs on" +ASYNC_PROBE_PID=$(sed -n 's/^claude_pid=//p' "$ASYNC_IDENTITY_LOG" | sed -n '1p') +ASYNC_PROBE_ENV_SESSION=$(sed -n 's/^env_session_id=//p' "$ASYNC_IDENTITY_LOG" | sed -n '1p') +ASYNC_PROBE_PAYLOAD=$(sed -n 's/^payload=//p' "$ASYNC_IDENTITY_LOG" | sed -n '1p') +case "$ASYNC_PROBE_PID" in + ''|*[!0-9]*) fail "Claude $CLAUDE_VERSION no longer exports a numeric CLAUDE_PID to its detached asyncRewake Stop hooks (got '$ASYNC_PROBE_PID')" ;; +esac +[ -n "$ASYNC_PROBE_ENV_SESSION" ] \ + || fail "Claude $CLAUDE_VERSION no longer exports CLAUDE_CODE_SESSION_ID to its detached asyncRewake Stop hooks" +ASYNC_PROBE_PAYLOAD_SESSION=$(fm_claude_payload_session_id "$ASYNC_PROBE_PAYLOAD") \ + || fail "Claude $CLAUDE_VERSION no longer names a session_id in the Stop payload it delivers on the detached asyncRewake path: $ASYNC_PROBE_PAYLOAD" +[ "$ASYNC_PROBE_PAYLOAD_SESSION" = "$ASYNC_PROBE_ENV_SESSION" ] \ + || fail "Claude $CLAUDE_VERSION delivered a detached asyncRewake Stop payload for session $ASYNC_PROBE_PAYLOAD_SESSION while exporting $ASYNC_PROBE_ENV_SESSION, so the two identity signals no longer corroborate each other on that path" +[ "$ASYNC_PROBE_PID" = "$RECORDED_OWNER" ] \ + || fail "Claude $CLAUDE_VERSION exported session pid $ASYNC_PROBE_PID to its detached asyncRewake Stop hooks while this session recorded owner $RECORDED_OWNER, so the session-identity proof no longer matches the recorded owner on the path the hook runs on" + ARM_RUNS=$(wc -l < "$HOME_DIR/state/arm-ran" 2>/dev/null | tr -d ' ') [ "$ARM_RUNS" = 2 ] || fail "expected exactly 2 hook-owned arm cycles, got $ARM_RUNS: $(cat "$HOME_DIR/state/arm-ran" 2>/dev/null)" -DRAIN_RUNS=$(wc -l < "$HOME_DIR/state/drain-ran" 2>/dev/null | tr -d ' ') -[ "$DRAIN_RUNS" = 3 ] || fail "expected one session-start drain plus two model wake drains, got $DRAIN_RUNS drains" +MODEL_DRAINS=$(grep -c 'bin/fm-wake-drain.sh' "$HOME_DIR/state/tool-calls.log" 2>/dev/null || true) +[ "${MODEL_DRAINS:-0}" = 2 ] \ + || fail "expected the model to handle exactly two Stop-owned wakes, got ${MODEL_DRAINS:-0}: $(cat "$HOME_DIR/state/tool-calls.log" 2>/dev/null)" REWAKES=$(grep -c 'Stop hook feedback' "$TRANSCRIPT" 2>/dev/null || true) [ "$REWAKES" -ge 2 ] || fail "expected at least 2 exit-2 rewake deliveries, got $REWAKES" grep -q 'stale: fixture-rapid-1' "$TRANSCRIPT" || fail "first rapid rewake reason missing from the transcript" @@ -161,4 +282,135 @@ printf '%s\n' '{"session_id":"live-owner-control"}' \ [ ! -s "$LAB/live-owner.out" ] && [ ! -s "$LAB/live-owner.err" ] || fail "competing Stop hook produced a rewake while another live session owned the home" wait "$LIVE_OWNER_PID" -printf 'ok - Claude %s live E2E reclaimed a stale session lock through session start, completed two tokenless Stop-owned rewake cycles, and preserved the competing-live-owner boundary\n' "$CLAUDE_VERSION" +printf 'ok - Claude %s live E2E reclaimed a stale session lock through session start, completed two tokenless Stop-owned rewake cycles, preserved the competing-live-owner boundary, and still delivers the session identity the ownership proof depends on\n' "$CLAUDE_VERSION" + +# --- away-mode return --------------------------------------------------------- +# The second isolated session. It enters away mode for real, ends a turn while +# the away daemon owns supervision, returns through the real return gate, and +# must then have supervision back at the very next Stop with no model-issued arm. +command -v tmux >/dev/null 2>&1 || fail "tmux not found; the away-mode lab needs the verified reference backend" + +git clone -q "$ROOT" "$AFK_PROJECT" +cp -R "$ROOT/bin/." "$AFK_PROJECT/bin/" +cp "$ROOT/.claude/settings.json" "$AFK_PROJECT/.claude/settings.json" +cp "$PROJECT/.claude/settings.local.json" "$AFK_PROJECT/.claude/settings.local.json" +cp "$PROJECT/bin/stop-identity-probe.sh" "$AFK_PROJECT/bin/stop-identity-probe.sh" +cp "$PROJECT/bin/tool-logger.sh" "$AFK_PROJECT/bin/tool-logger.sh" + +# Records, per Stop, whether the away-mode flag was still set and what the epoch +# ledger read, so the ORDER of the episode is provable rather than assumed. +cat > "$AFK_PROJECT/bin/stop-episode-probe.sh" <<'SH' +#!/usr/bin/env bash +cat >/dev/null 2>&1 || true +printf 'afk=%s ledger=%s\n' \ + "$([ -e "$FM_HOME/state/.afk" ] && echo yes || echo no)" \ + "$(sed -n 1p "$FM_HOME/state/.claude-autoarm-epoch" 2>/dev/null)" \ + >> "$FM_HOME/state/stop-episode.log" 2>/dev/null +exit 0 +SH +chmod +x "$AFK_PROJECT/bin/stop-episode-probe.sh" +# The episode probe rides the same local Stop registration as the identity probe. +python3 - "$AFK_PROJECT/.claude/settings.local.json" <<'PY' +import json, sys +path = sys.argv[1] +with open(path) as fh: + settings = json.load(fh) +settings["hooks"]["Stop"][0]["hooks"].insert( + 0, {"type": "command", + "command": '"$CLAUDE_PROJECT_DIR"/bin/stop-episode-probe.sh'}) +with open(path, "w") as fh: + json.dump(settings, fh) +PY + +# Arm fixture: never starts a real watcher and never reports a healthy one, so +# the guard's verdict depends only on what the auto-arm actually claims. +cat > "$AFK_PROJECT/bin/fm-watch-arm.sh" <<'SH' +#!/usr/bin/env bash +N=$(cat "$FM_HOME/state/arm-count" 2>/dev/null || echo 0); N=$((N+1)); echo "$N" > "$FM_HOME/state/arm-count" +printf 'arm-run=%s afk=%s\n' "$N" "$([ -e "$FM_HOME/state/.afk" ] && echo yes || echo no)" \ + >> "$FM_HOME/state/arm-ran" +printf 'watcher: started pid=%s (beacon fresh)\n' "$$" +printf 'stale: afk-lab-%s\n' "$N" +exit 0 +SH +# Watcher fixture: the away daemon runs this as its own child, so it must be +# bounded or the daemon's SIGTERM shutdown would sit in the return gate. +cat > "$AFK_PROJECT/bin/fm-watch.sh" <<'SH' +#!/usr/bin/env bash +touch "${FM_HOME:-.}/state/.last-watcher-beat" 2>/dev/null || true +sleep 2 +exit 0 +SH +# The in-flight need must survive the whole episode: bin/fm-afk-return.sh drains +# the queue itself and the tracked SessionStart hook runs a digest that drains +# too, so a drain-counted end would retire the work before the post-return Stop +# and the auto-arm would leave through its "nothing left to supervise" gate with +# nothing proven. It ends once a cycle has actually run after the return. +cat > "$AFK_PROJECT/bin/fm-wake-drain.sh" <<'SH' +#!/usr/bin/env bash +N=$(cat "$FM_HOME/state/drain-count" 2>/dev/null || echo 0); N=$((N+1)); echo "$N" > "$FM_HOME/state/drain-count" +echo "drain-run=$N" >> "$FM_HOME/state/drain-ran" +if grep -q 'afk=no' "$FM_HOME/state/arm-ran" 2>/dev/null; then + rm -f "$FM_HOME/state/task.meta" +fi +printf 'stale: afk-lab drained\n' +SH +chmod +x "$AFK_PROJECT/bin/fm-watch-arm.sh" "$AFK_PROJECT/bin/fm-watch.sh" \ + "$AFK_PROJECT/bin/fm-wake-drain.sh" + +mkdir -p "$AFK_HOME/state" "$AFK_HOME/config" "$AFK_HOME/data" +printf 'tmux\n' > "$AFK_HOME/config/backend" +printf 'project=fixture\nwindow=fixture\nbackend=tmux\n' > "$AFK_HOME/state/task.meta" + +# The away daemon injects into the captain pane it is handed, so the lab owns an +# isolated tmux session for that and never touches a real one. +tmux new-session -d -s "$AFK_CAPTAIN_SESSION" \ + || fail "could not create the isolated away-mode lab captain session" +AFK_CAPTAIN_PANE=$(tmux list-panes -t "$AFK_CAPTAIN_SESSION" \ + -F '#{session_name}:#{window_index}.#{pane_index}' | sed -n '1p') +[ -n "$AFK_CAPTAIN_PANE" ] || fail "could not resolve the lab captain pane" + +AFK_PROMPT='Run exactly `bin/fm-afk-launch.sh start` with Bash as your first tool call, then reply with exactly AFKON and stop. The next time anything wakes you, run exactly `bin/fm-afk-return.sh` once with Bash, then reply with exactly AFKOFF and stop. After that, whenever something wakes you, run exactly `bin/fm-wake-drain.sh` once with Bash, then reply with exactly ACK and stop. Never run bin/fm-watch-arm.sh or any other arm command, and never use any other tool.' + +( + cd "$AFK_PROJECT" || exit 1 + FM_HOME="$AFK_HOME" FM_SUPERVISOR_TARGET="$AFK_CAPTAIN_PANE" FM_SUPERVISOR_BACKEND=tmux \ + CLAUDE_CODE_ENABLE_PROMPT_SUGGESTION=false \ + claude -p "$AFK_PROMPT" --dangerously-skip-permissions --effort low \ + --output-format stream-json --verbose +) > "$AFK_TRANSCRIPT" 2>&1 || fail "Claude away-mode session failed: $(tail -20 "$AFK_TRANSCRIPT")" + +# The detached auto-arm of the last Stop finishes after the session returns. +afk_wait=0 +while [ "$afk_wait" -lt 60 ]; do + case "$(sed -n 's/^.*outcome=\([a-z][a-z-]*\) .*$/\1/p' "$AFK_HOME/state/.claude-autoarm-epoch" 2>/dev/null)" in + rewake|clean|failed|failed-suppressed) break ;; + esac + afk_wait=$((afk_wait + 1)) + sleep 0.5 +done + +EPISODE_LOG="$AFK_HOME/state/stop-episode.log" +[ -s "$EPISODE_LOG" ] || fail "the away-mode lab session ended no turn at all" +grep -q '^afk=yes ' "$EPISODE_LOG" \ + || fail "the away-mode episode never crossed a turn boundary, so this run proves nothing: $(cat "$EPISODE_LOG")" +grep -q '^afk=no ' "$EPISODE_LOG" \ + || fail "the away-mode return never happened, so the next Stop was never the post-return one: $(cat "$EPISODE_LOG")" +[ ! -e "$AFK_HOME/state/.afk" ] || fail "bin/fm-afk-return.sh left the home in away mode" + +AFK_LEDGER=$(sed -n '1p' "$AFK_HOME/state/.claude-autoarm-epoch" 2>/dev/null || true) +[ -n "$AFK_LEDGER" ] \ + || fail "the Stop-owned auto-arm never claimed this home after the away-mode return (empty ledger); supervision would have stayed down" +case "$AFK_LEDGER" in + *outcome=afk*) fail "the auto-arm stayed frozen on the away-mode episode's own entry after the return: $AFK_LEDGER" ;; +esac +grep -q 'afk=no' "$AFK_HOME/state/arm-ran" \ + || fail "no watcher cycle ran after the away-mode return: $(cat "$AFK_HOME/state/arm-ran" 2>/dev/null)" +if [ -f "$AFK_HOME/state/tool-calls.log" ]; then + ! grep -q 'fm-watch-arm.sh' "$AFK_HOME/state/tool-calls.log" \ + || fail "the model re-armed the watcher by hand after the away-mode return, the exact incident behavior: $(cat "$AFK_HOME/state/tool-calls.log")" +fi +grep -q 'bin/fm-afk-return.sh' "$AFK_HOME/state/tool-calls.log" 2>/dev/null \ + || fail "the lab session never ran the away-mode return: $(cat "$AFK_HOME/state/tool-calls.log" 2>/dev/null)" + +printf 'ok - Claude %s live E2E entered away mode, crossed a turn boundary while the away daemon owned supervision, returned through the real return gate, and had the Stop-owned auto-arm claim the home again at the next Stop with no model-issued arm\n' "$CLAUDE_VERSION" diff --git a/tests/fm-claude-stop-autoarm.test.sh b/tests/fm-claude-stop-autoarm.test.sh index 042d04ba947..edebc102cc1 100755 --- a/tests/fm-claude-stop-autoarm.test.sh +++ b/tests/fm-claude-stop-autoarm.test.sh @@ -79,6 +79,39 @@ run_autoarm() { return "$rc" } +# Run the hook the way Claude Code actually delivers a Stop: from a worker chain +# whose contiguous harness ancestry does NOT contain the session that owns the +# home. $2 is the pid the delivering session exports, $3 its session id, and $4 +# the session id the payload carries. Any extra env assignments must be exported +# before invocation. +run_autoarm_detached_chain() { # + local dir=$1 claude_pid=$2 env_session=$3 payload_session=$4 rc=0 + printf '{"session_id":"%s","hook_event_name":"Stop","stop_hook_active":false}\n' "$payload_session" \ + | CLAUDE_PID="$claude_pid" CLAUDE_CODE_SESSION_ID="$env_session" FM_HOME="$dir" \ + "$FAKE_CLAUDE" -c '"$FM_HOME/bin/fm-claude-stop-autoarm.sh"' 2>&1 || rc=$? + return "$rc" +} + +# Start a live fake-claude process that owns $1's session lock and is reachable +# from nothing the hook runs under. Prints its pid; the caller must reap it. +start_detached_lock_owner() { # + local dir=$1 pid + # The trailing no-op keeps the fake harness alive instead of letting bash + # exec the final sleep into a non-harness process. Its stdout is closed off + # deliberately: this helper is called from a command substitution, and a + # long-lived child holding that pipe open would block the caller until the + # owner had already exited. + "$FAKE_CLAUDE" -c 'printf "%s\n" "$$" > "$1/state/.lock"; sleep 60; :' _ "$dir" >/dev/null 2>&1 & + pid=$! + while [ ! -s "$dir/state/.lock" ]; do sleep 0.05; done + printf '%s\n' "$pid" +} + +stop_detached_lock_owner() { # + kill "$1" 2>/dev/null || true + wait "$1" 2>/dev/null || true +} + # Arm fixture variants, installed per test as /bin/fm-watch-arm.sh. write_arm_fixture() { local dir=$1 kind=$2 @@ -358,6 +391,218 @@ test_resolves_outermost_claude_pid_in_nested_bgspare_chain() { pass "auto-arm: resolves the outermost pid of a nested contiguous claude ancestry (bg-spare chain)" } +test_detached_chain_claims_home_for_its_own_session() { + local dir owner out status + dir=$(make_primary_dir "$TMP_ROOT/detached-own") + : > "$dir/state/task.meta" + write_arm_fixture "$dir" actionable + owner=$(start_detached_lock_owner "$dir") + + # The hook fires with no ancestry path to $owner at all - the shape Claude Code + # produces when it serves the hook from a worker whose top process has been + # reparented away from the session. Only the delivering session's own identity + # can prove ownership here. + out=$(run_autoarm_detached_chain "$dir" "$owner" sess-own sess-own 2>&1); status=$? + stop_detached_lock_owner "$owner" + + expect_code 2 "$status" "a hook with no ancestry path to its own session must still claim the home it owns" + [ -e "$dir/state/arm-ran" ] || fail "the owning session's hook never armed from a detached worker chain" + [ "$(epoch_outcome "$dir")" = rewake ] || fail "a detached-chain claim must record outcome=rewake" + assert_contains "$out" "firstmate watcher wake" "the detached-chain claim must deliver its wake" + pass "auto-arm: claims the home from a worker chain that cannot reach its own session" +} + +test_detached_chain_refuses_a_foreign_session() { + local dir owner out status owner_after + dir=$(make_primary_dir "$TMP_ROOT/detached-foreign") + : > "$dir/state/task.meta" + write_arm_fixture "$dir" actionable + owner=$(start_detached_lock_owner "$dir") + + # A different live Claude session, delivering its own coherent identity, from + # the same detached chain shape. This is the safety counter-proof: the identity + # path must never let a session claim a home it does not own. + out=$(run_autoarm_detached_chain "$dir" 9999999 sess-foreign sess-foreign 2>&1); status=$? + owner_after=$(cat "$dir/state/.lock") + stop_detached_lock_owner "$owner" + + expect_code 0 "$status" "a session that does not hold the lock must never claim the home" + [ "$owner_after" = "$owner" ] || fail "a foreign session replaced the recorded owner: expected $owner, got $owner_after" + [ ! -e "$dir/state/arm-ran" ] || fail "a foreign session armed another session's home" + [ ! -e "$dir/state/.claude-autoarm-epoch" ] || fail "a foreign session wrote an epoch for another session's home" + pass "auto-arm: a session that does not hold the lock still cannot claim the home" +} + +test_detached_chain_refuses_incoherent_or_missing_identity() { + local dir owner status + dir=$(make_primary_dir "$TMP_ROOT/detached-incoherent") + : > "$dir/state/task.meta" + write_arm_fixture "$dir" actionable + owner=$(start_detached_lock_owner "$dir") + + # The exported identity is only usable when the delivered event confirms it. + # An environment inherited from some other session, replayed against an + # unrelated event, proves nothing and must be refused. + status=0 + run_autoarm_detached_chain "$dir" "$owner" sess-env sess-other >/dev/null 2>&1 || status=$? + expect_code 0 "$status" "an exported identity that the delivered event does not confirm must be refused" + [ ! -e "$dir/state/arm-ran" ] || fail "hook armed on an identity the delivered event did not confirm" + + status=0 + run_autoarm_detached_chain "$dir" '' '' sess-own >/dev/null 2>&1 || status=$? + expect_code 0 "$status" "a hook with no session identity at all must stay inert" + [ ! -e "$dir/state/arm-ran" ] || fail "hook armed with no session identity" + + status=0 + run_autoarm_detached_chain "$dir" "not-a-pid" sess-own sess-own >/dev/null 2>&1 || status=$? + expect_code 0 "$status" "a malformed session pid must be refused" + [ ! -e "$dir/state/arm-ran" ] || fail "hook armed on a malformed session pid" + + stop_detached_lock_owner "$owner" + pass "auto-arm: an unconfirmed, absent, or malformed session identity never claims the home" +} + +test_detached_chain_refuses_missing_or_malformed_lock() { + local dir status + dir=$(make_primary_dir "$TMP_ROOT/detached-nolock") + : > "$dir/state/task.meta" + write_arm_fixture "$dir" actionable + + status=0 + run_autoarm_detached_chain "$dir" 4242 sess-own sess-own >/dev/null 2>&1 || status=$? + expect_code 0 "$status" "a home with no recorded owner must not be claimed through the identity path" + [ ! -e "$dir/state/arm-ran" ] || fail "hook armed a home with no recorded owner" + [ ! -e "$dir/state/.lock" ] || fail "hook created a session lock it was not entitled to write" + + printf 'not-a-pid\n' > "$dir/state/.lock" + status=0 + run_autoarm_detached_chain "$dir" 4242 sess-own sess-own >/dev/null 2>&1 || status=$? + expect_code 0 "$status" "a malformed recorded owner must be refused, not overwritten" + [ "$(cat "$dir/state/.lock")" = not-a-pid ] || fail "hook overwrote a malformed session lock" + [ ! -e "$dir/state/arm-ran" ] || fail "hook armed a home whose recorded owner is malformed" + pass "auto-arm: a missing or malformed recorded owner is refused through every identity path" +} + +test_detached_chain_keeps_homes_independent() { + local main_dir sm_dir main_owner sm_owner status + main_dir=$(make_primary_dir "$TMP_ROOT/two-homes-main") + sm_dir=$(make_secondmate_dir "$TMP_ROOT/two-homes-secondmate") + : > "$main_dir/state/task.meta" + : > "$sm_dir/state/task.meta" + write_arm_fixture "$main_dir" actionable + write_arm_fixture "$sm_dir" actionable + main_owner=$(start_detached_lock_owner "$main_dir") + sm_owner=$(start_detached_lock_owner "$sm_dir") + + # The main home's session, firing against the secondmate home: same machine, + # same identity mechanism, different owner. It must not claim it. + status=0 + run_autoarm_detached_chain "$sm_dir" "$main_owner" sess-main sess-main >/dev/null 2>&1 || status=$? + expect_code 0 "$status" "the main home's session must not claim a secondmate home" + [ ! -e "$sm_dir/state/arm-ran" ] || fail "the main home's session armed the secondmate home" + [ "$(cat "$sm_dir/state/.lock")" = "$sm_owner" ] || fail "the main home's session replaced the secondmate home's owner" + + # The secondmate home's own session still claims its own home. + status=0 + run_autoarm_detached_chain "$sm_dir" "$sm_owner" sess-sm sess-sm >/dev/null 2>&1 || status=$? + stop_detached_lock_owner "$main_owner" + stop_detached_lock_owner "$sm_owner" + + expect_code 2 "$status" "a secondmate home's own session must claim it from a detached chain" + [ -e "$sm_dir/state/arm-ran" ] || fail "the secondmate home's own session did not arm it" + [ ! -e "$main_dir/state/arm-ran" ] || fail "claiming the secondmate home touched the main home" + pass "auto-arm: two homes on one machine stay independent under the session-identity proof" +} + +# --- returning from away mode ------------------------------------------------- +# The measured shape (2026-08-27): an away-mode episode left the epoch ledger on +# a terminal "afk" outcome whose owner was long dead, the captain returned, and +# the Stop-owned auto-arm then never claimed the home again. The model re-armed +# the watcher by hand 175 times over five hours while every turn boundary +# blocked. These cases pin the two independent things that must hold at the +# first Stop after the return: a terminal ledger entry left by that episode is +# not a claim to defer to, and a home whose lock records a DIFFERENT process of +# this same session is still this session's home. + +# Write the durable session binding beside a home's lock: . +bind_session_identity() { + printf 'pid=%s\nsession=%s\n' "$2" "$3" > "$1/state/.lock.session" +} + +# Start a live process the fake harness owns, reachable from nothing the hook +# runs under. Prints its pid; the caller must reap it. This is the pool worker +# that serves a hook call without being the pid the lock records. +start_live_harness_process() { + local pid + "$FAKE_CLAUDE" -c 'sleep 60; :' >/dev/null 2>&1 & + pid=$! + printf '%s\n' "$pid" +} + +test_post_afk_ledger_never_freezes_the_next_claim() { + local dir owner out status dead + dir=$(make_primary_dir "$TMP_ROOT/post-afk-ledger") + : > "$dir/state/task.meta" + write_arm_fixture "$dir" actionable + # The away-mode episode's last cycle: it recorded the terminal afk outcome and + # exited, so the ledger names a generation whose owner is gone. + sleep 0 & + dead=$! + wait "$dead" 2>/dev/null || true + record_autoarm_epoch "$dir" 918 "$dead" afk + # The captain has returned: bin/fm-afk-return.sh cleared the away-mode flag. + [ ! -e "$dir/state/.afk" ] || fail "the fixture must represent a home that is no longer away" + owner=$(start_detached_lock_owner "$dir") + + out=$(run_autoarm_detached_chain "$dir" "$owner" sess-return sess-return 2>&1); status=$? + stop_detached_lock_owner "$owner" + + expect_code 2 "$status" "the first Stop after the return must claim the home, not defer to the away episode's spent entry" + [ -e "$dir/state/arm-ran" ] || fail "the home was left unarmed with work in flight after the return" + assert_contains "$out" "firstmate watcher wake" "the reclaiming cycle must translate its wake" + [ "$(epoch_field "$dir" epoch)" -gt 918 ] || fail "the frozen away-mode generation was never superseded: $(epoch_field "$dir" epoch)" + [ "$(epoch_outcome "$dir")" = rewake ] || fail "the new cycle did not record its own outcome: $(epoch_outcome "$dir")" + pass "auto-arm: a terminal away-mode ledger entry never freezes the first claim after the return" +} + +test_hook_claims_home_whose_lock_records_another_process_of_its_session() { + local dir owner worker out status + dir=$(make_primary_dir "$TMP_ROOT/post-afk-binding") + : > "$dir/state/task.meta" + write_arm_fixture "$dir" actionable + owner=$(start_detached_lock_owner "$dir") + worker=$(start_live_harness_process) + + # No binding yet: with the lock recording one process and the harness + # exporting another, neither existing proof can answer, and the hook is inert. + # That is the reproduced failure, and it is asserted so the case below cannot + # pass vacuously. + status=0 + out=$(FM_CLAUDE_AUTOARM_TRACE=1 run_autoarm_detached_chain "$dir" "$worker" sess-cap sess-cap 2>&1) || status=$? + expect_code 0 "$status" "the fixture is vacuous: the hook already claimed the home without any binding" + assert_contains "$out" "neither identity proof applies" "the unbound home must report the measured inert reason" + [ ! -e "$dir/state/arm-ran" ] || fail "the fixture is vacuous: the hook armed before the binding existed" + + # The binding the acquiring session recorded beside its own lock is what + # answers: the lock's owner and this delivering session are one session. + bind_session_identity "$dir" "$owner" sess-cap + out=$(run_autoarm_detached_chain "$dir" "$worker" sess-cap sess-cap 2>&1); status=$? + expect_code 2 "$status" "the session the lock is bound to must claim its own home" + [ -e "$dir/state/arm-ran" ] || fail "the bound session did not arm its own home" + assert_contains "$out" "firstmate watcher wake" "the claiming cycle must translate its wake" + [ "$(cat "$dir/state/.lock")" = "$owner" ] || fail "claiming through the binding replaced the recorded owner" + + # A different session on the same machine still cannot use that binding. + rm -f "$dir/state/arm-ran" "$dir/state/.claude-autoarm-epoch" + status=0 + run_autoarm_detached_chain "$dir" "$worker" sess-other sess-other >/dev/null 2>&1 || status=$? + stop_detached_lock_owner "$owner" + stop_detached_lock_owner "$worker" + expect_code 0 "$status" "a foreign session claimed a home bound to another session" + [ ! -e "$dir/state/arm-ran" ] || fail "a foreign session armed a home bound to another session" + pass "auto-arm: a Stop hook claims the home its session is bound to even when the lock records another process of it" +} + test_inert_when_fleet_idle() { local dir out status dir=$(make_primary_dir "$TMP_ROOT/idle") @@ -1156,6 +1401,13 @@ test_inert_when_lock_held_by_other_harness test_inert_when_afk test_stale_lock_recovery_preserves_afk_and_need_gates test_resolves_outermost_claude_pid_in_nested_bgspare_chain +test_detached_chain_claims_home_for_its_own_session +test_detached_chain_refuses_a_foreign_session +test_detached_chain_refuses_incoherent_or_missing_identity +test_detached_chain_refuses_missing_or_malformed_lock +test_detached_chain_keeps_homes_independent +test_post_afk_ledger_never_freezes_the_next_claim +test_hook_claims_home_whose_lock_records_another_process_of_its_session test_inert_when_fleet_idle test_actionable_close_rewakes_with_reason test_actionable_close_with_live_successor_rewakes_once diff --git a/tests/fm-merge-local.test.sh b/tests/fm-merge-local.test.sh new file mode 100755 index 00000000000..9968726bd4c --- /dev/null +++ b/tests/fm-merge-local.test.sh @@ -0,0 +1,386 @@ +#!/usr/bin/env bash +# Regression tests for fm-merge-local.sh's local landing. +# +# A task branch is always cut from origin's default-branch tip, so on a project +# whose local default branch carries commits origin does not have, landing that +# branch naively erases them. These tests drive the real script and prove the +# nominal landing still fast-forwards, the trap is refused by name, the explicit +# escape hatch is the only way through, and what it drops stays recoverable for +# as long as its rescue ref lives. +set -u + +# shellcheck source=tests/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +MERGE="$ROOT/bin/fm-merge-local.sh" +TMP_ROOT=$(fm_test_tmproot fm-merge-local) + +git_c() { git -C "$1" -c user.name='Firstmate Tests' -c user.email='tests@example.invalid' "${@:2}"; } + +# make_case [local_only_commit=yes]: build a home plus a project +# whose local main optionally carries an adoption commit origin never had, and a +# task branch fm/ cut from origin/main. Echoes home|project. +make_case() { + local name=$1 id=$2 adopt=${3:-yes} case_dir home project origin base + case_dir="$TMP_ROOT/$name" + home="$case_dir/home" + project="$case_dir/project" + origin="$case_dir/origin.git" + mkdir -p "$home/state" + + git init --quiet -b main "$project" + printf 'base\n' > "$project/README.md" + git_c "$project" add README.md + git_c "$project" commit -qm initial + git clone --quiet --bare "$project" "$origin" + git -C "$project" remote add origin "file://$origin" + git -C "$project" fetch --quiet origin + git -C "$project" remote set-head origin --auto >/dev/null 2>&1 + base=$(git -C "$project" rev-parse HEAD) + + if [ "$adopt" = yes ]; then + printf 'local adoption that must survive\n' > "$project/adoption.txt" + git_c "$project" add adoption.txt + git_c "$project" commit -qm 'adopt upstream PR #1234 locally' + fi + + # The task branch starts at origin/main, exactly as a freshened pool worktree does. + git_c "$project" branch "fm/$id" "$base" + git_c "$project" checkout --quiet "fm/$id" + printf 'worker change\n' > "$project/worker.txt" + git_c "$project" add worker.txt + git_c "$project" commit -qm 'worker work' + git_c "$project" checkout --quiet main + + fm_write_meta "$home/state/$id.meta" "project=$project" "mode=local-only" "yolo=off" + printf '%s|%s\n' "$home" "$project" +} + +read_case_record() { + IFS='|' read -r HOME_DIR PROJECT_DIR <&1 +} + +test_clean_landing_fast_forwards() { + local rec id out status + id='merge-local-clean-r1' + rec=$(make_case clean "$id" no) + read_case_record "$rec" + + out=$(run_merge "$id") + status=$? + expect_code 0 "$status" "a clean landing should fast-forward without friction" + assert_contains "$out" "merged fm/$id into local main" "clean landing did not report the merge" + assert_grep 'worker change' "$PROJECT_DIR/worker.txt" "clean landing did not bring the worker's change" + if [ "${FM_TEST_EVIDENCE:-0}" = 1 ]; then + printf '# observed clean landing: %s\n' "$(printf '%s\n' "$out" | tail -n 1)" + fi + pass "a landing that drops nothing still fast-forwards with no added friction" +} + +test_landing_that_drops_a_local_commit_is_refused() { + local rec id out status before + id='merge-local-trap-r2' + rec=$(make_case trap "$id") + read_case_record "$rec" + before=$(git -C "$PROJECT_DIR" rev-parse main) + + out=$(run_merge "$id") + status=$? + [ "$status" -ne 0 ] || fail "a landing that erases a local commit succeeded" + assert_contains "$out" "would drop 1 commit(s)" "refusal did not count the dropped commits" + assert_contains "$out" "adopt upstream PR #1234 locally" "refusal did not name the commit that would disappear" + assert_contains "$out" "git merge main" "refusal did not name the reconciliation to run" + assert_contains "$out" "--drop-local-commits" "refusal did not name its explicit escape hatch" + [ "$(git -C "$PROJECT_DIR" rev-parse main)" = "$before" ] || fail "the refusal still moved main" + assert_grep 'local adoption that must survive' "$PROJECT_DIR/adoption.txt" \ + "the refused landing already erased the local adoption" + assert_absent "$HOME_DIR/state/$id.local-merge-drop" "a refusal wrote a drop record" + if [ "${FM_TEST_EVIDENCE:-0}" = 1 ]; then + printf '# observed refusal:\n%s\n' "$out" + fi + pass "a landing that would erase a local commit is refused, naming the commit and the fix" +} + +test_reconciled_branch_lands_cleanly() { + local rec id out status + id='merge-local-reconciled-r3' + rec=$(make_case reconciled "$id") + read_case_record "$rec" + + # The documented reconciliation: merge the local default branch into the task branch. + git_c "$PROJECT_DIR" checkout --quiet "fm/$id" + git_c "$PROJECT_DIR" merge --quiet --no-edit main + git_c "$PROJECT_DIR" checkout --quiet main + + out=$(run_merge "$id") + status=$? + expect_code 0 "$status" "a reconciled branch should land: $out" + assert_grep 'local adoption that must survive' "$PROJECT_DIR/adoption.txt" \ + "the reconciled landing lost the local adoption" + assert_grep 'worker change' "$PROJECT_DIR/worker.txt" "the reconciled landing lost the worker's change" + pass "the reconciliation the refusal names makes the same landing pass" +} + +test_escape_hatch_drops_explicitly_and_traceably() { + local rec id out status dropped_sha + id='merge-local-escape-r4' + rec=$(make_case escape "$id") + read_case_record "$rec" + dropped_sha=$(git -C "$PROJECT_DIR" rev-parse main) + + out=$(run_merge --drop-local-commits "$id") + status=$? + expect_code 0 "$status" "the explicit escape hatch should land: $out" + assert_contains "$out" "DROPPING 1 local-only commit(s)" "the escape hatch dropped commits silently" + assert_contains "$out" "$dropped_sha" "the escape hatch did not print the dropped commit's full SHA" + assert_present "$HOME_DIR/state/$id.local-merge-drop" "the escape hatch left no trace record" + assert_grep "status=completed" "$HOME_DIR/state/$id.local-merge-drop" \ + "the trace record does not mark the successful drop completed" + assert_grep "dropped=$dropped_sha" "$HOME_DIR/state/$id.local-merge-drop" \ + "the trace record does not carry the dropped commit's recoverable SHA" + [ "$(git -C "$PROJECT_DIR" rev-parse main)" = "$(git -C "$PROJECT_DIR" rev-parse "fm/$id")" ] \ + || fail "the escape hatch did not land the branch" + [ ! -e "$PROJECT_DIR/adoption.txt" ] || fail "the escape hatch claimed a drop it did not perform" + git -C "$PROJECT_DIR" cat-file -e "$dropped_sha^{commit}" \ + || fail "the dropped commit is not recoverable by the recorded SHA" + if [ "${FM_TEST_EVIDENCE:-0}" = 1 ]; then + printf '# observed escape hatch:\n%s\n' "$out" + fi + pass "the escape hatch drops only when asked, prints what it dropped, and records recoverable SHAs" +} + +test_escape_hatch_is_a_noop_on_a_clean_landing() { + local rec id out status + id='merge-local-escape-noop-r5' + rec=$(make_case escape-noop "$id" no) + read_case_record "$rec" + + out=$(run_merge --drop-local-commits "$id") + status=$? + expect_code 0 "$status" "the escape hatch should stay harmless on a clean landing: $out" + assert_contains "$out" "was unnecessary" "the escape hatch did not report itself unnecessary" + assert_contains "$out" "merged fm/$id into local main" "the clean landing did not fast-forward" + assert_absent "$HOME_DIR/state/$id.local-merge-drop" "a clean landing wrote a drop record" + pass "the escape hatch never turns a clean landing into a reset" +} + +test_dropped_commits_outlive_an_aggressive_gc() { + local rec id out status dropped_sha ref rescue + id='merge-local-rescue-r6' + rec=$(make_case rescue "$id") + read_case_record "$rec" + dropped_sha=$(git -C "$PROJECT_DIR" rev-parse main) + ref="refs/fm-dropped/$id/$(git -C "$PROJECT_DIR" rev-parse --short main)" + + out=$(run_merge --drop-local-commits "$id") + status=$? + expect_code 0 "$status" "the explicit escape hatch should land: $out" + assert_contains "$out" "$ref" "the escape hatch did not name the rescue ref that keeps the drop recoverable" + assert_contains "$out" "update-ref -d $ref" "the escape hatch did not say how to release the rescue ref" + assert_grep "rescue_ref=$ref" "$HOME_DIR/state/$id.local-merge-drop" \ + "the trace record does not name the rescue ref that holds the dropped commits" + rescue=$(git -C "$PROJECT_DIR" rev-parse --verify --quiet "$ref" || true) + [ "$rescue" = "$dropped_sha" ] \ + || fail "rescue ref holds '$rescue', expected the pre-reset tip $dropped_sha" + + # The reflog is exactly what the record used to depend on; prune it away. + git -C "$PROJECT_DIR" reflog expire --expire=now --expire-unreachable=now --all >/dev/null 2>&1 + git -C "$PROJECT_DIR" gc --prune=now --quiet >/dev/null 2>&1 || fail "gc failed in the test project" + git -C "$PROJECT_DIR" cat-file -e "$dropped_sha^{commit}" 2>/dev/null \ + || fail "the dropped commit died with the reflog, so the recorded SHA is dead" + assert_contains "$(git -C "$PROJECT_DIR" log -1 --format='%s' "$dropped_sha")" \ + 'adopt upstream PR #1234 locally' "the rescued commit is no longer readable after gc" + + # And the documented release really releases: the ref is the only thing holding them. + git -C "$PROJECT_DIR" update-ref -d "$ref" + git -C "$PROJECT_DIR" reflog expire --expire=now --expire-unreachable=now --all >/dev/null 2>&1 + git -C "$PROJECT_DIR" gc --prune=now --quiet >/dev/null 2>&1 || fail "gc failed after releasing the rescue ref" + if git -C "$PROJECT_DIR" cat-file -e "$dropped_sha^{commit}" 2>/dev/null; then + fail "the dropped commit survived the documented release, so the rescue ref is not what was holding it" + fi + pass "dropped commits are held by a rescue ref that outlives the reflog, and the documented release frees them" +} + +test_successive_drops_each_keep_their_own_rescue() { + local rec id out status first_sha second_sha first_ref second_ref + id='merge-local-rescue-twice-r7' + rec=$(make_case rescue-twice "$id") + read_case_record "$rec" + first_sha=$(git -C "$PROJECT_DIR" rev-parse main) + first_ref="refs/fm-dropped/$id/$(git -C "$PROJECT_DIR" rev-parse --short main)" + + out=$(run_merge --drop-local-commits "$id") + status=$? + expect_code 0 "$status" "the first authorized drop should land: $out" + + # A later local-only commit makes the same task droppable again. The second + # drop must succeed on its own, without the operator releasing the first rescue. + printf 'a second local adoption\n' > "$PROJECT_DIR/adoption-2.txt" + git_c "$PROJECT_DIR" add adoption-2.txt + git_c "$PROJECT_DIR" commit -qm 'adopt upstream PR #5678 locally' + second_sha=$(git -C "$PROJECT_DIR" rev-parse main) + second_ref="refs/fm-dropped/$id/$(git -C "$PROJECT_DIR" rev-parse --short main)" + [ "$second_ref" != "$first_ref" ] || fail "both drops resolved to the same rescue ref name" + + out=$(run_merge --drop-local-commits "$id") + status=$? + expect_code 0 "$status" "the second authorized drop should land without releasing the first rescue: $out" + assert_contains "$out" "$second_ref" "the second drop did not name its own rescue ref" + + [ "$(git -C "$PROJECT_DIR" rev-parse --verify --quiet "$first_ref")" = "$first_sha" ] \ + || fail "the second drop displaced the first drop's rescue ref" + [ "$(git -C "$PROJECT_DIR" rev-parse --verify --quiet "$second_ref")" = "$second_sha" ] \ + || fail "the second drop did not pin its own tip" + + git -C "$PROJECT_DIR" reflog expire --expire=now --expire-unreachable=now --all >/dev/null 2>&1 + git -C "$PROJECT_DIR" gc --prune=now --quiet >/dev/null 2>&1 || fail "gc failed in the test project" + git -C "$PROJECT_DIR" cat-file -e "$first_sha^{commit}" 2>/dev/null \ + || fail "the first drop's commits died once a second drop happened" + git -C "$PROJECT_DIR" cat-file -e "$second_sha^{commit}" 2>/dev/null \ + || fail "the second drop's commits are not held by their rescue ref" + assert_contains "$(git -C "$PROJECT_DIR" log -1 --format='%s' "$first_sha")" \ + 'adopt upstream PR #1234 locally' "the first drop's rescued commit is no longer readable" + assert_contains "$(git -C "$PROJECT_DIR" log -1 --format='%s' "$second_sha")" \ + 'adopt upstream PR #5678 locally' "the second drop's rescued commit is no longer readable" + pass "successive drops on one task each keep their own rescue ref, and neither release is forced by the other" +} + +test_redropping_a_recovered_tip_reuses_its_own_rescue() { + local rec id out status dropped_sha ref + id='merge-local-rescue-redrop-r8' + rec=$(make_case rescue-redrop "$id") + read_case_record "$rec" + dropped_sha=$(git -C "$PROJECT_DIR" rev-parse main) + ref="refs/fm-dropped/$id/$(git -C "$PROJECT_DIR" rev-parse --short main)" + + out=$(run_merge --drop-local-commits "$id") + status=$? + expect_code 0 "$status" "the first authorized drop should land: $out" + + # The operator recovers main from the rescue ref, then decides to drop again. + git_c "$PROJECT_DIR" reset --hard --quiet "$ref" + + out=$(run_merge --drop-local-commits "$id") + status=$? + expect_code 0 "$status" "re-dropping a recovered tip should land: $out" + assert_contains "$out" "already holds $dropped_sha" "the re-drop did not say the rescue ref was already in place" + [ "$(git -C "$PROJECT_DIR" rev-parse --verify --quiet "$ref")" = "$dropped_sha" ] \ + || fail "the re-drop moved the rescue ref off the tip it already held" + [ "$(git -C "$PROJECT_DIR" rev-parse main)" = "$(git -C "$PROJECT_DIR" rev-parse "fm/$id")" ] \ + || fail "the re-drop did not land the branch" + pass "re-dropping a tip a rescue ref already holds keeps that ref and says so" +} + +test_escape_hatch_preserves_concurrent_default_advance() { + local rec id out status before rescue_ref fakebin real_git concurrent + id='merge-local-concurrent-r9' + rec=$(make_case concurrent "$id") + read_case_record "$rec" + before=$(git -C "$PROJECT_DIR" rev-parse main) + rescue_ref="refs/fm-dropped/$id/$(git -C "$PROJECT_DIR" rev-parse --short main)" + fakebin="$HOME_DIR/fakebin" + real_git=$(command -v git) + mkdir -p "$fakebin" + cat > "$fakebin/git" <> "$PROJECT_DIR/adoption.txt" + printf '%s\n' "\$concurrent" > "$HOME_DIR/concurrent.sha" + exit 0 +fi +exec "$real_git" "\$@" +SH + chmod +x "$fakebin/git" + + out=$(PATH="$fakebin:$PATH" run_merge --drop-local-commits "$id") + status=$? + [ "$status" -ne 0 ] || fail "a landing overwrote the concurrently advanced default branch" + concurrent=$(cat "$HOME_DIR/concurrent.sha") + [ "$(git -C "$PROJECT_DIR" rev-parse main)" = "$concurrent" ] \ + || fail "the concurrent default-branch tip was not preserved" + if git -C "$PROJECT_DIR" rev-parse --verify --quiet "$rescue_ref" >/dev/null; then + fail "the refused landing retained a rescue ref for history it never dropped" + fi + assert_grep 'local adoption that must survive' "$PROJECT_DIR/adoption.txt" \ + "the checkout lost the concurrent tip's committed content" + assert_grep 'concurrent uncommitted edit' "$PROJECT_DIR/adoption.txt" \ + "the compare-and-swap refusal destroyed concurrent uncommitted work" + assert_absent "$PROJECT_DIR/worker.txt" "the refused landing still checked out the task branch" + assert_grep 'status=pending' "$HOME_DIR/state/$id.local-merge-drop" \ + "the refused drop did not retain its pending audit" + assert_grep 'status=failed' "$HOME_DIR/state/$id.local-merge-drop" \ + "the refused drop did not mark its audit failed" + assert_grep 'rescue_ref_released=yes' "$HOME_DIR/state/$id.local-merge-drop" \ + "the refused drop did not record release of its newly planted rescue" + grep -q '^status=completed$' "$HOME_DIR/state/$id.local-merge-drop" \ + && fail "the refused drop was recorded as completed" + grep -q '^dropped=' "$HOME_DIR/state/$id.local-merge-drop" \ + && fail "the refused drop audit claims commits were dropped" + assert_contains "$out" "advanced concurrently" "the compare-and-swap refusal did not explain the preserved tip" + pass "the destructive landing preserves and synchronizes a concurrent default-branch advance" +} + +test_escape_hatch_refuses_to_overwrite_concurrent_checkout_edit() { + local rec id out status before fakebin real_git + id='merge-local-sync-race-r10' + rec=$(make_case sync-race "$id") + read_case_record "$rec" + before=$(git -C "$PROJECT_DIR" rev-parse main) + fakebin="$HOME_DIR/fakebin" + real_git=$(command -v git) + mkdir -p "$fakebin" + cat > "$fakebin/git" <> "$PROJECT_DIR/adoption.txt" + exit 0 +fi +exec "$real_git" "\$@" +SH + chmod +x "$fakebin/git" + + out=$(PATH="$fakebin:$PATH" run_merge --drop-local-commits "$id") + status=$? + [ "$status" -ne 0 ] || fail "checkout synchronization overwrote a concurrent edit" + [ "$(git -C "$PROJECT_DIR" rev-parse main)" = "$(git -C "$PROJECT_DIR" rev-parse "fm/$id")" ] \ + || fail "the compare-and-swap branch landing did not complete before synchronization" + assert_grep 'concurrent checkout edit' "$PROJECT_DIR/adoption.txt" \ + "safe checkout synchronization destroyed the concurrent edit" + assert_grep 'status=completed' "$HOME_DIR/state/$id.local-merge-drop" \ + "the completed branch drop was not recorded" + assert_grep "dropped=$before" "$HOME_DIR/state/$id.local-merge-drop" \ + "the completed branch drop did not record its rescued commit" + assert_grep 'status=sync-failed' "$HOME_DIR/state/$id.local-merge-drop" \ + "the refused checkout synchronization was not recorded" + assert_contains "$out" "were left untouched" "the synchronization refusal did not explain concurrent-work preservation" + pass "the landed branch leaves concurrent checkout edits untouched when synchronization refuses" +} + +test_clean_landing_fast_forwards +test_landing_that_drops_a_local_commit_is_refused +test_reconciled_branch_lands_cleanly +test_escape_hatch_drops_explicitly_and_traceably +test_escape_hatch_is_a_noop_on_a_clean_landing +test_dropped_commits_outlive_an_aggressive_gc +test_successive_drops_each_keep_their_own_rescue +test_redropping_a_recovered_tip_reuses_its_own_rescue +test_escape_hatch_preserves_concurrent_default_advance +test_escape_hatch_refuses_to_overwrite_concurrent_checkout_edit + +echo "# all fm-merge-local tests passed" diff --git a/tests/fm-session-lock-identity.test.sh b/tests/fm-session-lock-identity.test.sh new file mode 100755 index 00000000000..6c4f3f93f44 --- /dev/null +++ b/tests/fm-session-lock-identity.test.sh @@ -0,0 +1,621 @@ +#!/usr/bin/env bash +# tests/fm-session-lock-identity.test.sh - durable session-identity binding for +# the fleet lock (bin/fm-session-lock-lib.sh, bin/fm-lock.sh). +# +# The defect these cases pin: process ancestry is not a stable session identity. +# Claude Code serves one session's hooks and tool calls from more than one worker +# pool, and a pool whose top process has been reparented to init yields a +# contiguous harness run that never reaches the session's own lineage. The +# session that acquired the lock through a pool that COULD reach it can then +# never prove ownership again, and every mutating session-start step is skipped. +# +# Every case drives the library behind a deterministic fake ps, so the same two +# platform reporting semantics are covered from either host, and the end-to-end +# cases run the REAL bin/fm-lock.sh. The unit cases deliberately assert the +# DIVERGENCE - ancestry refusing while identity accepts - so a future change that +# quietly made ancestry succeed could not leave the case passing vacuously. +# shellcheck disable=SC2016 # single quotes are deliberate: expressions expand inside the fixture child +set -u + +# shellcheck source=tests/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +TMP_ROOT=$(fm_test_tmproot fm-session-lock-identity) + +LIB="$ROOT/bin/fm-session-lock-lib.sh" + +# A process table shaped exactly like the measured reparented pool: +# this process -> claude bg-spare 27316 -> claude bg-pty-host 27305 -> init +# The session that owns the lock is 89187, a live claude that this ancestry +# never reaches. 4242 is a second, unrelated live claude session. +write_reparented_pool_ps() { # + cat > "$1/ps" <<'SH' +#!/usr/bin/env bash +set -u +field= pid= +while [ "$#" -gt 0 ]; do + case "$1" in + -o) field=$2; shift 2 ;; + -p) pid=$2; shift 2 ;; + *) shift ;; + esac +done +case "$pid:$field" in + 89187:comm=) printf '%s\n' claude ;; + 89187:args=) printf '%s\n' 'claude --dangerously-skip-permissions' ;; + 89187:ppid=) printf '%s\n' 82710 ;; + 4242:comm=) printf '%s\n' claude ;; + 4242:args=) printf '%s\n' 'claude' ;; + 4242:ppid=) printf '%s\n' 1 ;; + 5150:comm=) printf '%s\n' bash ;; + 5150:args=) printf '%s\n' bash ;; + 5150:ppid=) printf '%s\n' 1 ;; + 27316:comm=) printf '%s\n' 'claude bg-spare' ;; + 27316:args=) printf '%s\n' 'claude bg-spare --bg-spare /tmp/cc/spare/d6bfe5e1.claim.sock' ;; + 27316:ppid=) printf '%s\n' 27305 ;; + 27305:comm=) printf '%s\n' 'claude bg-pty-host' ;; + 27305:args=) printf '%s\n' 'claude bg-pty-host --bg-pty-host /tmp/cc/spare/d6bfe5e1.pty.sock 200 50' ;; + 27305:ppid=) printf '%s\n' 1 ;; + *:comm=) printf '%s\n' bash ;; + *:args=) printf '%s\n' 'bash /repo/bin/fm-session-start.sh' ;; + *:ppid=) printf '%s\n' 27316 ;; +esac +SH + chmod +x "$1/ps" +} + +# Run one library expression with shadowing ps, under session id $3 +# and served-session pid $4 (default 27316, the pool process this fixture's +# ancestry really passes through). kill is stubbed so liveness is decided by the +# process table alone. +lib_eval() { # [session-id] [claude-pid] + local fakebin=$1 expr=$2 session=${3-} claude_pid=${4-27316} + CLAUDE_CODE_SESSION_ID="$session" CLAUDE_PID="$claude_pid" PATH="$fakebin:$PATH" bash -c " + . \"\$0\" + kill() { return 0; } + $expr + " "$LIB" +} + +# A Stop payload naming session $1, in the shape Claude Code actually delivers. +stop_payload() { # + printf '{"session_id":"%s","hook_event_name":"Stop","stop_hook_active":false}' "$1" +} + +new_home() { # + local dir="$TMP_ROOT/$1" + mkdir -p "$dir/state" + printf '%s' "$dir" +} + +bind_identity() { # + printf 'pid=%s\nsession=%s\n' "$2" "$3" > "$1/.lock.session" +} + +SESSION=bebaa84f-3b50-4abc-8d29-6c2751cc722c +OTHER_SESSION=a080589e-8fe8-41c3-bbb5-cd5a8b66a11d + +test_owning_session_is_recognized_from_a_reparented_pool() { + local dir fakebin + dir=$(new_home reparented-owner) + fakebin=$(fm_fakebin "$dir/bin") + write_reparented_pool_ps "$fakebin" + printf '89187\n' > "$dir/state/.lock" + bind_identity "$dir/state" 89187 "$SESSION" + + # The divergence itself: ancestry cannot reach 89187 from this pool at all. + if lib_eval "$fakebin" "fm_session_lock_owned_by_self '$dir/state'" "$SESSION"; then + fail "the fixture is vacuous: ancestry reached the lock owner across a reparented pool" + fi + lib_eval "$fakebin" "fm_harness_ancestry_pid" "$SESSION" | grep -qx 27305 \ + || fail "the fixture is vacuous: ancestry did not stop at the reparented pool host 27305" + + lib_eval "$fakebin" "fm_session_lock_owned_by_session_identity '$dir/state'" "$SESSION" \ + || fail "the session that acquired the lock was not recognized by its recorded identity" + lib_eval "$fakebin" "fm_session_lock_owned_by_current_session '$dir/state'" "$SESSION" \ + || fail "the owning session was refused ownership from a reparented worker pool" + pass "session-lock identity: the session that owns the lock is recognized from a reparented worker pool" +} + +test_foreign_session_still_fails_closed() { + local dir fakebin + dir=$(new_home foreign-session) + fakebin=$(fm_fakebin "$dir/bin") + write_reparented_pool_ps "$fakebin" + printf '89187\n' > "$dir/state/.lock" + bind_identity "$dir/state" 89187 "$SESSION" + + if lib_eval "$fakebin" "fm_session_lock_owned_by_current_session '$dir/state'" "$OTHER_SESSION"; then + fail "a foreign session claimed a home whose lock is bound to another session" + fi + if lib_eval "$fakebin" "fm_session_lock_owned_by_current_session '$dir/state'" ''; then + fail "a process carrying no session identity claimed a bound home" + fi + pass "session-lock identity: a foreign session and an identity-less process still fail closed" +} + +test_inherited_session_environment_never_proves_ownership() { + local dir fakebin + dir=$(new_home inherited-environment) + fakebin=$(fm_fakebin "$dir/bin") + write_reparented_pool_ps "$fakebin" + printf '89187\n' > "$dir/state/.lock" + bind_identity "$dir/state" 89187 "$SESSION" + + # A session identity travels down every child of a tool call, so a process + # launched by the owning session - a crewmate on a harness that does not + # overwrite these variables, or any unrelated command started from the same + # environment - carries the owner's session id without being that session. + # Its own ancestry names its own harness, so the served-session process the + # environment claims is nowhere inside it. + if lib_eval "$fakebin" "fm_session_lock_owned_by_current_session '$dir/state'" "$SESSION" 89187; then + fail "a process that merely inherited the owning session's environment claimed the home" + fi + if lib_eval "$fakebin" "fm_session_lock_owned_by_current_session '$dir/state'" "$SESSION" ''; then + fail "an uncorroborated session identity claimed the home" + fi + if lib_eval "$fakebin" "fm_session_lock_owned_by_current_session '$dir/state'" "$SESSION" 5150; then + fail "a session identity corroborated by a non-harness process claimed the home" + fi + + # And such a process must never BIND the home to the identity it inherited. + command rm -f -- "$dir/state/.lock.session" + lib_eval "$fakebin" "fm_session_lock_publish_identity '$dir/state' 89187" "$SESSION" 89187 \ + || fail "publishing from an inherited environment reported failure instead of declining" + [ -e "$dir/state/.lock.session" ] \ + && fail "a process with an inherited session environment bound the home to that session" + pass "session-lock identity: an inherited session environment neither proves nor binds ownership" +} + +test_absent_malformed_and_unbound_locks_fail_closed() { + local dir fakebin + dir=$(new_home unbound-locks) + fakebin=$(fm_fakebin "$dir/bin") + write_reparented_pool_ps "$fakebin" + + if lib_eval "$fakebin" "fm_session_lock_owned_by_current_session '$dir/state'" "$SESSION"; then + fail "an absent lock was claimed as owned" + fi + + # An OLD lock, carrying only a pid and no binding: today's behavior exactly. + printf '89187\n' > "$dir/state/.lock" + if lib_eval "$fakebin" "fm_session_lock_owned_by_current_session '$dir/state'" "$SESSION"; then + fail "a legacy pid-only lock naming another live session was claimed as owned" + fi + + bind_identity "$dir/state" 89187 "$SESSION" + printf 'not-a-pid\n' > "$dir/state/.lock" + if lib_eval "$fakebin" "fm_session_lock_owned_by_current_session '$dir/state'" "$SESSION"; then + fail "a malformed lock was claimed as owned" + fi + + # A binding left behind by a previous owner cannot speak for the current lock. + printf '4242\n' > "$dir/state/.lock" + if lib_eval "$fakebin" "fm_session_lock_owned_by_current_session '$dir/state'" "$SESSION"; then + fail "a stale binding naming a different pid was accepted for the current lock" + fi + + # A dead owner is never owned; it is reclaimed through the acquire path. + printf '5150\n' > "$dir/state/.lock" + bind_identity "$dir/state" 5150 "$SESSION" + if lib_eval "$fakebin" "fm_session_lock_owned_by_current_session '$dir/state'" "$SESSION"; then + fail "a lock whose owner is not a live harness was claimed as owned" + fi + pass "session-lock identity: absent, malformed, legacy, stale-bound, and dead-owner locks all fail closed" +} + +test_two_homes_on_one_machine_stay_independent() { + local mine theirs fakebin + mine=$(new_home home-mine) + theirs=$(new_home home-theirs) + fakebin=$(fm_fakebin "$mine/bin") + write_reparented_pool_ps "$fakebin" + + printf '89187\n' > "$mine/state/.lock" + bind_identity "$mine/state" 89187 "$SESSION" + printf '4242\n' > "$theirs/state/.lock" + bind_identity "$theirs/state" 4242 "$OTHER_SESSION" + + lib_eval "$fakebin" "fm_session_lock_owned_by_current_session '$mine/state'" "$SESSION" \ + || fail "this session lost ownership of its own home" + if lib_eval "$fakebin" "fm_session_lock_owned_by_current_session '$theirs/state'" "$SESSION"; then + fail "one session claimed a second firstmate home bound to a different session" + fi + pass "session-lock identity: two firstmate homes on one machine stay independent" +} + +test_binding_is_replaced_not_inherited() { + local dir fakebin + dir=$(new_home binding-replaced) + fakebin=$(fm_fakebin "$dir/bin") + write_reparented_pool_ps "$fakebin" + printf '89187\n' > "$dir/state/.lock" + bind_identity "$dir/state" 89187 "$OTHER_SESSION" + + # A harness that exposes no session identity must leave NO binding behind, so + # a later recycled pid can never meet a previous owner's session id. + lib_eval "$fakebin" "fm_session_lock_publish_identity '$dir/state' 89187" '' 27316 \ + || fail "publishing a binding for an identity-less harness reported failure" + [ -e "$dir/state/.lock.session" ] \ + && fail "an identity-less acquisition left a previous owner's binding in place" + + lib_eval "$fakebin" "fm_session_lock_publish_identity '$dir/state' 89187" "$SESSION" \ + || fail "publishing this session's binding failed" + lib_eval "$fakebin" "fm_session_lock_owned_by_current_session '$dir/state'" "$SESSION" \ + || fail "the freshly published binding did not prove ownership" + if lib_eval "$fakebin" "fm_session_lock_owned_by_current_session '$dir/state'" "$OTHER_SESSION"; then + fail "the replaced binding still spoke for the previous owner" + fi + pass "session-lock identity: publishing replaces any previous binding instead of inheriting it" +} + +# --- end-to-end layer: the real bin/fm-lock.sh ------------------------------- + +# Run the REAL acquire path with shadowing ps, under session id $3. +lock_sh() { # [arg] + local home=$1 fakebin=$2 session=$3 claude_pid=$4 + shift 4 + CLAUDE_CODE_SESSION_ID="$session" CLAUDE_PID="$claude_pid" \ + FM_HOME="$home" FM_STATE_OVERRIDE="$home/state" \ + PATH="$fakebin:$PATH" bash "$ROOT/bin/fm-lock.sh" "$@" 2>&1 +} + +# The end-to-end layer runs the REAL bin/fm-lock.sh, which does NOT stub kill, +# so every pid it is asked about must be a genuinely live process on this host. +# These are real background processes whose pids the generated ps then describes +# as the session and its pool; using invented numbers would make the case pass +# or fail on whatever happened to occupy those pids on the machine running it. +# LIVE_PID carries the result rather than stdout: a command substitution would +# run this in a subshell, losing the pid list this file must reap, and would +# also block until the backgrounded process closed the captured stdout. +E2E_PIDS= +LIVE_PID= +spawn_live_pid() { # -> sets LIVE_PID to a fresh long-lived process + sleep 300 >/dev/null 2>&1 & + LIVE_PID=$! + E2E_PIDS="$E2E_PIDS $LIVE_PID" +} +reap_live_pids() { + local pid + for pid in $E2E_PIDS; do kill "$pid" 2>/dev/null || true; done + E2E_PIDS= +} + +# A ps that reports as the lock-owning claude session and every other +# process as an ordinary shell parented by . +write_direct_ps() { # + cat > "$1/ps" < + cat > "$1/ps" < claude bg-spare -> claude bg-pty-host -> init +# and , the lock owner, is live but unreachable from that chain. +write_live_pool_ps() { # + cat > "$1/ps" <&1)" + grep -qx "session=$SESSION" "$dir/state/.lock.session" \ + || fail "the acquisition did not bind this session's identity" + + # Now the same session comes back through the REPARENTED pool. + write_live_pool_ps "$fakebin" "$session_pid" "$spare_pid" "$host_pid" + out=$(lock_sh "$dir" "$fakebin" "$SESSION" "$spare_pid") \ + || fail "the owning session was refused read-only from a reparented pool: $out" + case "$out" in + *"lock acquired: harness pid $session_pid"*) ;; + *) fail "expected ownership of pid $session_pid to be confirmed, got: $out" ;; + esac + [ "$(tr -d '[:space:]' < "$dir/state/.lock")" = "$session_pid" ] \ + || fail "the lock moved off the session onto the reparented pool host" + + # A foreign session through the same pool must still be told to stay read-only. + out=$(lock_sh "$dir" "$fakebin" "$OTHER_SESSION" "$spare_pid") \ + && fail "a foreign session acquired the lock: $out" + case "$out" in + *"another live firstmate session holds the lock (pid $session_pid)"*) ;; + *) fail "expected the read-only refusal for a foreign session, got: $out" ;; + esac + + # And a process that only INHERITED the owner's session id, whose own ancestry + # names its own harness instead of the pool, must stay read-only too. + out=$(lock_sh "$dir" "$fakebin" "$SESSION" "$session_pid") \ + && fail "a process with an inherited session environment acquired the lock: $out" + case "$out" in + *"another live firstmate session holds the lock (pid $session_pid)"*) ;; + *) fail "expected the read-only refusal for an inherited environment, got: $out" ;; + esac + + reap_live_pids + pass "session-lock identity e2e: acquire binds the session, readmits it from a reparented pool, and still refuses a foreign or inherited one" +} + +test_e2e_existing_owned_lock_backfills_only_invalid_binding() { + local dir fakebin out session_pid expected + dir=$(new_home e2e-binding-backfill) + fakebin=$(fm_fakebin "$dir/bin") + spawn_live_pid; session_pid=$LIVE_PID + write_direct_ps "$fakebin" "$session_pid" "$session_pid" + printf '%s\n' "$session_pid" > "$dir/state/.lock" + + out=$(lock_sh "$dir" "$fakebin" "$SESSION" "$session_pid") \ + || fail "the legacy owned lock was refused during binding backfill: $out" + grep -qx "pid=$session_pid" "$dir/state/.lock.session" \ + || fail "the legacy owned lock did not backfill its pid binding" + grep -qx "session=$SESSION" "$dir/state/.lock.session" \ + || fail "the legacy owned lock did not backfill its session identity" + + printf 'pid=4242\nsession=%s\n' "$OTHER_SESSION" > "$dir/state/.lock.session" + out=$(lock_sh "$dir" "$fakebin" "$SESSION" "$session_pid") \ + || fail "the owned lock with an invalid binding was refused: $out" + [ "$(cat "$dir/state/.lock.session")" = "$(printf 'pid=%s\nsession=%s' "$session_pid" "$SESSION")" ] \ + || fail "the invalid binding was not replaced with the proved owner's identity" + + expected=$(printf 'pid=%s\nsession=%s\nupgrade-field=preserve' "$session_pid" "$SESSION") + printf '%s\n' "$expected" > "$dir/state/.lock.session" + out=$(lock_sh "$dir" "$fakebin" "$SESSION" "$session_pid") \ + || fail "the owned lock with a valid binding was refused: $out" + [ "$(cat "$dir/state/.lock.session")" = "$expected" ] \ + || fail "an already-valid session binding was rewritten instead of preserved" + + reap_live_pids + pass "session-lock identity e2e: legacy and invalid bindings backfill while valid bindings remain unchanged" +} + +test_e2e_backfill_reproves_after_claim_lock_wait() { + local fixture_dir fakebin old_pid new_pid holder_pid call_pid i status expected + local lock_file binding_file claim_lock claim_held release_claim call_out call_status + fixture_dir=$(new_home e2e-binding-backfill-takeover) + fakebin=$(fm_fakebin "$fixture_dir/bin") + lock_file="$fixture_dir/state/.lock" + binding_file="$fixture_dir/state/.lock.session" + claim_lock="$fixture_dir/state/.lock.acquire" + claim_held="$fixture_dir/claim-held" + release_claim="$fixture_dir/release-claim" + call_out="$fixture_dir/call.out" + call_status="$fixture_dir/call.status" + spawn_live_pid; old_pid=$LIVE_PID + spawn_live_pid; new_pid=$LIVE_PID + write_two_session_ps "$fakebin" "$old_pid" "$new_pid" + printf '%s\n' "$old_pid" > "$lock_file" + + ( + . "$ROOT/bin/fm-wake-lib.sh" + fm_lock_acquire_wait "$claim_lock" || exit 1 + : > "$claim_held" + while [ ! -e "$release_claim" ]; do sleep 0.05; done + fm_lock_release "$claim_lock" + ) & + holder_pid=$! + for i in $(seq 1 100); do + [ ! -e "$claim_held" ] || break + sleep 0.05 + done + [ -e "$claim_held" ] || fail "binding-backfill-takeover: claim fixture never acquired the lock" + + ( + lock_sh "$fixture_dir" "$fakebin" "$SESSION" "$old_pid" > "$call_out" + printf '%s\n' "$?" > "$call_status" + ) & + call_pid=$! + sleep 0.3 + kill -0 "$call_pid" 2>/dev/null || fail "binding-backfill-takeover: backfill bypassed the claim lock" + + printf '%s\n' "$new_pid" > "$lock_file" + expected=$(printf 'pid=%s\nsession=%s' "$new_pid" "$OTHER_SESSION") + printf '%s\n' "$expected" > "$binding_file" + : > "$release_claim" + wait "$holder_pid" || fail "binding-backfill-takeover: claim fixture failed" + wait "$call_pid" + status=$(cat "$call_status") + [ "$status" -ne 0 ] || fail "binding-backfill-takeover: obsolete owner accepted the replacement lock" + [ "$(cat "$lock_file")" = "$new_pid" ] \ + || fail "binding-backfill-takeover: obsolete owner replaced the new lock pid" + [ "$(cat "$binding_file")" = "$expected" ] \ + || fail "binding-backfill-takeover: obsolete owner replaced the new session binding" + + reap_live_pids + pass "session-lock identity e2e: backfill serializes with takeover and reproves ownership" +} + +# --- hook membership when the exported pid is not the recorded owner ---------- +# The second, pid-shaped proof asks whether the pid the harness exports for this +# event IS the pid the lock records. bin/fm-lock.sh deliberately records the +# OUTERMOST pid of the contiguous harness run that acquired the lock, so for one +# and the same session those can be two different processes, and then ancestry +# cannot reach the recorded owner either. The measured consequence was a Stop +# hook permanently inert on a home whose lock is alive and genuinely its own, +# reporting "live session N owns this home and neither identity proof applies". +# These cases pin the third proof and its refusals, and assert the divergence so +# they cannot pass vacuously. + +test_hook_is_readmitted_when_the_exported_pid_is_not_the_recorded_owner() { + local dir fakebin payload + dir=$(new_home hook-binding-owner) + fakebin=$(fm_fakebin "$dir/bin") + write_reparented_pool_ps "$fakebin" + printf '89187\n' > "$dir/state/.lock" + bind_identity "$dir/state" 89187 "$SESSION" + payload=$(stop_payload "$SESSION") + + # The divergence: with the pool process serving this event, neither existing + # proof can answer, which is exactly the inert-hook shape. + if lib_eval "$fakebin" "fm_session_lock_owned_by_self '$dir/state'" "$SESSION"; then + fail "the fixture is vacuous: ancestry reached the recorded owner from the pool" + fi + if lib_eval "$fakebin" "fm_session_lock_owned_by_claude_hook '$dir/state' '$payload'" "$SESSION"; then + fail "the fixture is vacuous: the exported pid already matched the recorded owner" + fi + + lib_eval "$fakebin" \ + "fm_session_lock_owned_by_claude_hook_binding '$dir/state' '$payload'" "$SESSION" \ + || fail "the delivering session was not recognized through the lock's own binding" + lib_eval "$fakebin" \ + "fm_session_lock_owned_by_this_claude_session '$dir/state' '$payload'" "$SESSION" \ + || fail "the hook disjunction refused the very session the lock is bound to" + [ "$(lib_eval "$fakebin" \ + "fm_session_lock_owned_by_this_claude_session '$dir/state' '$payload' && printf '%s' \"\$FM_SESSION_LOCK_PROOF\"" \ + "$SESSION")" = claude-session-binding ] \ + || fail "the disjunction did not report which proof carried the verdict" + pass "session-lock identity: a Stop hook proves membership through the lock's recorded session when the exported pid is another process of it" +} + +test_hook_binding_proof_refuses_every_unproven_shape() { + local dir fakebin + dir=$(new_home hook-binding-refusals) + fakebin=$(fm_fakebin "$dir/bin") + write_reparented_pool_ps "$fakebin" + printf '89187\n' > "$dir/state/.lock" + bind_identity "$dir/state" 89187 "$SESSION" + + # A foreign session carries its own id in both the payload and the + # environment, so the binding names someone else and it stays out. + if lib_eval "$fakebin" \ + "fm_session_lock_owned_by_this_claude_session '$dir/state' '$(stop_payload "$OTHER_SESSION")'" \ + "$OTHER_SESSION"; then + fail "a foreign session claimed a home bound to another session" + fi + + # An environment inherited from the owning session, replayed against some + # other event, proves nothing: the payload is what describes THIS event. + if lib_eval "$fakebin" \ + "fm_session_lock_owned_by_this_claude_session '$dir/state' '$(stop_payload "$OTHER_SESSION")'" \ + "$SESSION"; then + fail "an inherited environment claimed the home against an unrelated event" + fi + + # No payload session at all, and no environment identity at all. + if lib_eval "$fakebin" \ + "fm_session_lock_owned_by_this_claude_session '$dir/state' '{}'" "$SESSION"; then + fail "a payload naming no session claimed the home" + fi + if lib_eval "$fakebin" \ + "fm_session_lock_owned_by_this_claude_session '$dir/state' '$(stop_payload "$SESSION")'" ''; then + fail "a hook with no exported session identity claimed the home" + fi + + # A binding naming a pid the lock does not record cannot speak for that lock. + bind_identity "$dir/state" 4242 "$SESSION" + if lib_eval "$fakebin" \ + "fm_session_lock_owned_by_this_claude_session '$dir/state' '$(stop_payload "$SESSION")'" \ + "$SESSION"; then + fail "a binding recorded for another pid spoke for the current lock" + fi + + # A home whose lock carries no binding keeps exactly the previous behavior. + rm -f "$dir/state/.lock.session" + if lib_eval "$fakebin" \ + "fm_session_lock_owned_by_this_claude_session '$dir/state' '$(stop_payload "$SESSION")'" \ + "$SESSION"; then + fail "an unbound lock was claimed through a binding that does not exist" + fi + + # A dead recorded owner is not readmitted by its binding either; that path + # belongs to the hook's own guarded stale-lock recovery. + printf '5150\n' > "$dir/state/.lock" + bind_identity "$dir/state" 5150 "$SESSION" + if lib_eval "$fakebin" \ + "fm_session_lock_owned_by_this_claude_session '$dir/state' '$(stop_payload "$SESSION")'" \ + "$SESSION"; then + fail "a lock naming a non-harness process was claimed through its binding" + fi + pass "session-lock identity: the binding proof refuses foreign, inherited, unconfirmed, mismatched, unbound, and dead-owner shapes" +} + +test_owning_session_is_recognized_from_a_reparented_pool +test_foreign_session_still_fails_closed +test_inherited_session_environment_never_proves_ownership +test_absent_malformed_and_unbound_locks_fail_closed +test_two_homes_on_one_machine_stay_independent +test_binding_is_replaced_not_inherited +test_hook_is_readmitted_when_the_exported_pid_is_not_the_recorded_owner +test_hook_binding_proof_refuses_every_unproven_shape +test_e2e_acquire_records_the_binding_and_readmits_the_pool +test_e2e_existing_owned_lock_backfills_only_invalid_binding +test_e2e_backfill_reproves_after_claim_lock_wait diff --git a/tests/fm-sessionstart-hook-live-e2e.test.sh b/tests/fm-sessionstart-hook-live-e2e.test.sh index ba38197a0d2..d3f0010f400 100755 --- a/tests/fm-sessionstart-hook-live-e2e.test.sh +++ b/tests/fm-sessionstart-hook-live-e2e.test.sh @@ -117,7 +117,7 @@ make_lab() { # -> echoes lab dir chmod +x "$lab/bin/$stub" done - # The REAL deferred-network stage plus the two libraries it sources, so fact + # The REAL deferred-network stage plus the libraries it sources, so fact # (c) is proven against the actual detach this ship relies on rather than a # re-creation of it. Its bootstrap child is a stub: what is under test here is # survival across the hook boundary, not the sweeps, which @@ -126,6 +126,7 @@ make_lab() { # -> echoes lab dir ln -sf "$ROOT/bin/fm-timeout-lib.sh" "$lab/bin/fm-timeout-lib.sh" ln -sf "$ROOT/bin/fm-wake-lib.sh" "$lab/bin/fm-wake-lib.sh" ln -sf "$ROOT/bin/fm-session-lock-lib.sh" "$lab/bin/fm-session-lock-lib.sh" + ln -sf "$ROOT/bin/fm-cursor-lib.sh" "$lab/bin/fm-cursor-lib.sh" cat > "$lab/bin/fm-bootstrap.sh" <<'SH' #!/usr/bin/env bash # Outlives the hook on purpose: the marker can only appear if the worker was @@ -143,7 +144,7 @@ SH # supplied and prints a source-stamped token for the model to quote back. set -u record=${FM_LIVE_RECORD:?} -source= +source= payload= while [ $# -gt 0 ]; do case "$1" in --source) source=${2:-}; shift 2 || exit 0 ;; @@ -151,7 +152,8 @@ while [ $# -gt 0 ]; do esac done if [ -z "$source" ]; then - source=$(cat 2>/dev/null | awk ' + payload=$(cat 2>/dev/null || true) + source=$(printf '%s' "$payload" | awk ' BEGIN { RS = "\"" } seen == 2 { print; exit } seen == 1 && $0 ~ /^[[:space:]]*:[[:space:]]*$/ { seen = 2; next } @@ -161,6 +163,24 @@ if [ -z "$source" ]; then fi [ -n "$source" ] || source=none printf '%s\n' "$source" >> "$record" +# The session-identity evidence the fleet lock's durable binding rests on. It is +# vendor-emitted, so only a real harness can confirm it: the payload names a +# session, the harness exported that same session, and the served-session +# process it names is genuinely inside this hook's own harness ancestry. +if [ -n "${FM_LIVE_IDENTITY_RECORD:-}" ]; then + payload_session=$(printf '%s' "$payload" \ + | tr ',{}' '\n' \ + | sed -n 's/^[[:space:]]*"session_id"[[:space:]]*:[[:space:]]*"\([A-Za-z0-9._-]\{1,\}\)"[[:space:]]*$/\1/p' \ + | sed -n '1p') + corroborated=no + if . "$(dirname "$0")/fm-session-lock-lib.sh" 2>/dev/null \ + && fm_harness_session_is_ours 2>/dev/null; then + corroborated=yes + fi + printf '%s|%s|%s|%s|%s\n' "$source" "${payload_session:-none}" \ + "${CLAUDE_CODE_SESSION_ID:-none}" "${CLAUDE_PID:-none}" "$corroborated" \ + >> "$FM_LIVE_IDENTITY_RECORD" +fi # Exactly what bin/fm-session-start.sh does after taking the lock. if [ -n "${FM_LIVE_DETACH_MARKER:-}" ]; then "$(dirname "$0")/fm-startup-network.sh" start --locked 0 --harvest-pid $$ >/dev/null 2>&1 || true @@ -205,6 +225,7 @@ probe_process_opens() { # "$record" rm -f "$marker" "$lab/state/.startup-network."* out=$( cd "$lab" && FM_LIVE_RECORD="$record" FM_LIVE_NONCE="$LIVE_NONCE" FM_ROOT_OVERRIDE="$lab" FM_HOME="$lab" \ + FM_LIVE_IDENTITY_RECORD="$lab/identity" \ FM_LIVE_DETACH_MARKER="$marker" \ "${cold[@]}" "$ASK" < /dev/null 2>&1 ) source=$(head -n 1 "$record") @@ -229,6 +250,7 @@ probe_process_opens() { # "$record" ( cd "$lab" && FM_LIVE_RECORD="$record" FM_LIVE_NONCE="$LIVE_NONCE" FM_ROOT_OVERRIDE="$lab" FM_HOME="$lab" \ + FM_LIVE_IDENTITY_RECORD="$lab/identity" \ "${resume[@]}" 'Say only OK.' < /dev/null >/dev/null 2>&1 ) || true source=$(head -n 1 "$record") [ -n "$source" ] \ @@ -246,6 +268,39 @@ probe_process_opens() { # + local harness=$1 version=$2 lab=$3 + local record="$lab/identity" n=0 + [ -s "$record" ] \ + || fail "$harness $version: no session-open recorded any identity evidence, so this check verified nothing" + while IFS='|' read -r source payload_session env_session claude_pid corroborated; do + [ -n "$source" ] || continue + n=$((n + 1)) + [ "$payload_session" != none ] \ + || fail "$harness $version: the $source session-open payload carried no session_id" + [ "$env_session" = "$payload_session" ] \ + || fail "$harness $version: the $source session-open exported session '$env_session' but its payload named '$payload_session'" + [ "$claude_pid" != none ] \ + || fail "$harness $version: the $source session-open named no served session process" + [ "$corroborated" = yes ] \ + || fail "$harness $version: the $source session-open's served session process $claude_pid was not inside the hook's own harness ancestry" + done < "$record" + [ "$n" -ge 2 ] \ + || fail "$harness $version: only $n session-open(s) were checked for identity; a context reset must be among them" + note "$harness $version: session identity usable on $n session-open(s)" + pass "$harness $version: every session open carries a corroborated session identity for the fleet lock" +} + probe_context_reset() { # local harness=$1 version=$2 lab=$3 clear_cmd=$4 shift 4 @@ -253,7 +308,7 @@ probe_context_reset() { # "$record" tmux -L "$SOCKET" new-session -d -s "$session" -c "$lab" -x 200 -y 50 \ -e FM_LIVE_RECORD="$record" -e FM_ROOT_OVERRIDE="$lab" -e FM_HOME="$lab" \ - -e FM_LIVE_NONCE="$LIVE_NONCE" \ + -e FM_LIVE_NONCE="$LIVE_NONCE" -e FM_LIVE_IDENTITY_RECORD="$lab/identity" \ "$*" \ || fail "$harness $version: could not start an interactive lab session" @@ -606,6 +661,7 @@ for harness in claude codex pi; do -- claude --continue -p --permission-mode bypassPermissions probe_context_reset claude "$version" "$lab" /clear \ claude --permission-mode bypassPermissions + assert_session_identity claude "$version" "$lab" ;; codex) probe_process_opens codex "$version" "$lab" resume \ diff --git a/tests/fm-spawn-pool-base-freshen.test.sh b/tests/fm-spawn-pool-base-freshen.test.sh index 492d4ebeabc..7392e7f7c92 100755 --- a/tests/fm-spawn-pool-base-freshen.test.sh +++ b/tests/fm-spawn-pool-base-freshen.test.sh @@ -424,6 +424,40 @@ test_stale_pin_beside_other_dirt_reports_one_verdict() { "spawn discarded the untracked file while refusing the pool" pass "a stale pin beside other dirt yields the conservative refusal alone, with no stale-pin line" } +test_local_default_ahead_warns_without_changing_the_base() { + local rec id out status current adopted + id='pool-local-ahead-r7' + rec=$(make_case local-ahead "$id") + read_case_record "$rec" + + # The project's local default branch carries an adoption origin never had - + # the fork-that-cannot-merge-upstream shape. The base must stay origin's tip, + # so the branch never carries it into an upstream PR, and the spawn must say so. + git -C "$PROJECT_DIR" fetch --quiet origin + git -C "$PROJECT_DIR" merge --quiet --ff-only "origin/$DEFAULT_BRANCH" + printf 'local adoption that must not reach an upstream PR\n' > "$PROJECT_DIR/adoption.txt" + git -C "$PROJECT_DIR" add adoption.txt + git -C "$PROJECT_DIR" -c user.name='Firstmate Tests' -c user.email='tests@example.invalid' \ + commit -qm 'adopt upstream PR locally' + adopted=$(git -C "$PROJECT_DIR" rev-parse "refs/heads/$DEFAULT_BRANCH") + + out=$(run_spawn "$id" --mode no-mistakes --yolo off) + status=$? + expect_code 0 "$status" "a diverged local default branch must not stop the spawn: $out" + assert_contains "$out" "ahead of origin/$DEFAULT_BRANCH" "spawn did not warn about the local divergence" + assert_contains "$out" "git merge $DEFAULT_BRANCH" "the warning did not name the landing reconciliation" + current=$(git -C "$POOL_DIR" rev-parse "origin/$DEFAULT_BRANCH") + [ "$(git -C "$POOL_DIR" rev-parse HEAD)" = "$current" ] \ + || fail "the warning changed the spawn base away from origin/$DEFAULT_BRANCH" + [ "$current" != "$adopted" ] || fail "fixture did not prove local $DEFAULT_BRANCH is ahead of origin" + [ ! -e "$POOL_DIR/adoption.txt" ] \ + || fail "the worker base carries a local-only adoption that would pollute an upstream PR" + if [ "${FM_TEST_EVIDENCE:-0}" = 1 ]; then + printf '# observed divergence warning: %s\n' "$(printf '%s\n' "$out" | grep 'ahead of origin')" + fi + pass "a local default branch ahead of origin warns at spawn without moving the base off origin's tip" +} + test_stale_pool_base_refreshes_before_branching test_non_main_default_branch_refreshes_before_branching @@ -436,5 +470,6 @@ test_unpushed_submodule_commit_is_still_uncommitted_work test_work_inside_submodule_is_still_uncommitted_work test_stale_pin_carrying_real_work_is_not_called_stale test_stale_pin_beside_other_dirt_reports_one_verdict +test_local_default_ahead_warns_without_changing_the_base echo "# all fm-spawn-pool-base-freshen tests passed" diff --git a/tests/fm-teardown.test.sh b/tests/fm-teardown.test.sh index fe0131ce479..56640c4a914 100755 --- a/tests/fm-teardown.test.sh +++ b/tests/fm-teardown.test.sh @@ -2605,6 +2605,351 @@ EOF pass "the run abort and the leaked-process reap both complete before the destructive worktree return" } +# Replace the treehouse mock with one that logs every invocation to +# $case_dir/treehouse.log and succeeds. Args: case_dir +add_logging_treehouse() { + local case_dir=$1 + : > "$case_dir/treehouse.log" + cat > "$case_dir/fakebin/treehouse" <> "$case_dir/treehouse.log" +exit 0 +SH + chmod +x "$case_dir/fakebin/treehouse" +} + +# Fast-forward the project's local main to the worktree HEAD so the local-only +# landed-work safety check passes. Args: case_dir +merge_wt_into_local_main() { + local case_dir=$1 wt_head + wt_head=$(git -C "$case_dir/wt" rev-parse HEAD) + git -C "$case_dir/project" update-ref refs/heads/main "$wt_head" +} + +# The 2026-08-08 incident end to end (teardown-rerun-reissue): attempt 1 returns +# the pool worktree, then fails on the refused pane close; the freed slot is +# reissued to a NEWER task; the sanctioned rerun must close the stale pane and +# finish cleanup WITHOUT re-returning the slot under the live tenant. +test_rerun_after_return_and_reissue_never_rereturns() { + local case_dir log closed rc + case_dir=$(make_case rerun-reissue) + write_meta "$case_dir" local-only ship + wt_commit "$case_dir" "landed work" + merge_wt_into_local_main "$case_dir" + configure_flat_herdr_teardown_case "$case_dir" + add_logging_treehouse "$case_dir" + log="$case_dir/herdr.log"; : > "$log" + closed="$case_dir/closed" + : > "$case_dir/state/task-x1.status" + + # Attempt 1: the pane close cannot record success (FM_FAKE_HERDR_CLOSED points + # into a missing dir), so the pane stays present and teardown fails AFTER the + # worktree return - the incident's exact failure point. + rc=0 + FM_FAKE_HERDR_LOG="$log" FM_FAKE_HERDR_CLOSED="$case_dir/noexist/closed" \ + run_teardown "$case_dir" > "$case_dir/stdout" 2> "$case_dir/stderr" || rc=$? + [ "$rc" -ne 0 ] || fail "rerun-reissue: attempt 1 reported success although the pane was never closed" + [ "$(wc -l < "$case_dir/treehouse.log")" -eq 1 ] \ + || fail "rerun-reissue: attempt 1 did not return the worktree exactly once: $(cat "$case_dir/treehouse.log")" + [ -e "$case_dir/state/task-x1.worktree-returned" ] \ + || fail "rerun-reissue: the completed return left no durable marker for the rerun" + [ -e "$case_dir/state/task-x1.meta" ] || fail "rerun-reissue: attempt 1 erased the durable metadata" + + # The pool reissues the same slot to a newer task: its meta binds the same + # worktree path and its session occupies the worktree on its own branch with + # uncommitted work. + fm_write_meta "$case_dir/state/task-x2.meta" \ + "window=default:wZ:pZ" \ + "endpoint_task_id=task-x2" \ + "worktree=$case_dir/wt" \ + "project=$case_dir/project" \ + "kind=scout" \ + "mode=no-mistakes" \ + "backend=herdr" + git -C "$case_dir/wt" checkout -q -b fm/task-x2 + printf '%s\n' "tenant work in flight" > "$case_dir/wt/tenant.txt" + + # Attempt 2 (the sanctioned rerun): the pane close now works. + FM_FAKE_HERDR_LOG="$log" FM_FAKE_HERDR_CLOSED="$closed" FM_BACKEND_HERDR_IDLE_SHELL_PROOF_POLLS=1 \ + run_teardown "$case_dir" > "$case_dir/stdout2" 2> "$case_dir/stderr2" \ + || fail "rerun-reissue: the rerun failed: $(cat "$case_dir/stderr2")" + [ "$(wc -l < "$case_dir/treehouse.log")" -eq 1 ] \ + || fail "rerun-reissue: the rerun re-returned the reissued slot: $(cat "$case_dir/treehouse.log")" + assert_grep "skipping worktree return" "$case_dir/stdout2" \ + "rerun-reissue: the rerun did not say it skipped the return" + [ -e "$closed" ] || fail "rerun-reissue: the rerun never closed the stale pane" + [ "$(git -C "$case_dir/wt" rev-parse --abbrev-ref HEAD)" = "fm/task-x2" ] \ + || fail "rerun-reissue: the rerun moved the tenant's branch" + [ -f "$case_dir/wt/tenant.txt" ] || fail "rerun-reissue: the rerun destroyed the tenant's uncommitted work" + [ -e "$case_dir/state/task-x2.meta" ] || fail "rerun-reissue: the rerun removed the tenant's metadata" + [ ! -e "$case_dir/state/task-x1.meta" ] || fail "rerun-reissue: the rerun left the old metadata behind" + [ ! -e "$case_dir/state/task-x1.worktree-returned" ] \ + || fail "rerun-reissue: the rerun left the return marker behind" + grep -q "teardown task-x1 complete" "$case_dir/stdout2" \ + || fail "rerun-reissue: the rerun did not report completion" + pass "a rerun after a completed return and a reissued slot closes the pane and cleans records without re-returning" +} + +test_teardown_and_spawn_share_task_set_first_lock_order() { + local case_dir holder_pid teardown_pid i status + case_dir=$(make_case task-set-lock-order) + write_meta "$case_dir" local-only ship + wt_commit "$case_dir" "landed work" + merge_wt_into_local_main "$case_dir" + add_logging_treehouse "$case_dir" + + ( + . "$ROOT/bin/fm-wake-lib.sh" + set_lock=$(fm_task_set_lock_path "$case_dir/state") || exit 1 + meta_lock=$(fm_meta_lock_path "$case_dir/state/task-x1.meta") || exit 1 + fm_lock_acquire_wait "$set_lock" || exit 1 + : > "$case_dir/task-set-held" + while [ ! -e "$case_dir/acquire-meta" ]; do sleep 0.05; done + fm_lock_acquire_wait "$meta_lock" || exit 1 + : > "$case_dir/meta-held" + while [ ! -e "$case_dir/release-spawn-locks" ]; do sleep 0.05; done + fm_lock_release "$meta_lock" || exit 1 + fm_lock_release "$set_lock" || exit 1 + ) & + holder_pid=$! + for i in $(seq 1 100); do + [ ! -e "$case_dir/task-set-held" ] || break + sleep 0.05 + done + [ -e "$case_dir/task-set-held" ] || fail "task-set-lock-order: spawn fixture never acquired the task-set lock" + + ( + run_teardown "$case_dir" > "$case_dir/stdout" 2> "$case_dir/stderr" + printf '%s\n' "$?" > "$case_dir/teardown.status" + ) & + teardown_pid=$! + sleep 0.3 + : > "$case_dir/acquire-meta" + for i in $(seq 1 100); do + [ ! -e "$case_dir/meta-held" ] || break + sleep 0.05 + done + if [ ! -e "$case_dir/meta-held" ]; then + kill "$holder_pid" "$teardown_pid" 2>/dev/null || true + wait "$holder_pid" 2>/dev/null || true + wait "$teardown_pid" 2>/dev/null || true + fail "task-set-lock-order: spawn and teardown deadlocked on opposite task-set/meta lock order" + fi + : > "$case_dir/release-spawn-locks" + wait "$holder_pid" || fail "task-set-lock-order: spawn fixture failed while releasing locks" + wait "$teardown_pid" + status=$(cat "$case_dir/teardown.status") + [ "$status" -eq 0 ] || fail "task-set-lock-order: teardown failed after the spawn released its locks: $(cat "$case_dir/stderr")" + [ "$(wc -l < "$case_dir/treehouse.log")" -eq 1 ] \ + || fail "task-set-lock-order: teardown did not return the worktree exactly once" + pass "spawn and teardown acquire the task-set lock before task metadata" +} + +test_marker_failure_reissue_before_rerun_preserves_new_tenant() { + local case_dir log closed rc spawn_pid rerun_pid i rerun_status + case_dir=$(make_case marker-failure-reissue-race) + write_meta "$case_dir" local-only ship + wt_commit "$case_dir" "landed work" + merge_wt_into_local_main "$case_dir" + configure_flat_herdr_teardown_case "$case_dir" + log="$case_dir/herdr.log"; : > "$log" + closed="$case_dir/closed" + : > "$case_dir/state/task-x1.status" + : > "$case_dir/treehouse.log" + cat > "$case_dir/fakebin/treehouse" <> "$case_dir/treehouse.log" +ln -s "$case_dir/missing/marker" "$case_dir/state/task-x1.worktree-returned" +exit 0 +SH + chmod +x "$case_dir/fakebin/treehouse" + + rc=0 + FM_FAKE_HERDR_LOG="$log" FM_FAKE_HERDR_CLOSED="$case_dir/noexist/closed" \ + run_teardown "$case_dir" > "$case_dir/stdout" 2> "$case_dir/stderr" || rc=$? + [ "$rc" -ne 0 ] || fail "marker-failure-race: attempt 1 succeeded although its pane stayed open" + assert_grep "could not record the completed worktree return" "$case_dir/stderr" \ + "marker-failure-race: the fixture did not force marker publication to fail" + rm -f "$case_dir/state/task-x1.worktree-returned" + + ( + . "$ROOT/bin/fm-wake-lib.sh" + lock=$(fm_task_set_lock_path "$case_dir/state") || exit 1 + fm_lock_acquire_wait "$lock" || exit 1 + git -C "$case_dir/wt" checkout -q -b fm/task-x2 || exit 1 + printf '%s\n' "tenant work in flight" > "$case_dir/wt/tenant.txt" || exit 1 + : > "$case_dir/reissue-acquired" + while [ ! -e "$case_dir/allow-meta-publish" ]; do sleep 0.05; done + cat > "$case_dir/state/.task-x2.meta.spawn" < "$case_dir/stdout2" 2> "$case_dir/stderr2" + printf '%s\n' "$?" > "$case_dir/rerun.status" + ) & + rerun_pid=$! + sleep 0.3 + kill -0 "$rerun_pid" 2>/dev/null \ + || fail "marker-failure-race: rerun did not wait for the replacement binding publication" + [ "$(wc -l < "$case_dir/treehouse.log")" -eq 1 ] \ + || fail "marker-failure-race: rerun returned the slot before replacement metadata publication" + + : > "$case_dir/allow-meta-publish" + wait "$spawn_pid" || fail "marker-failure-race: replacement spawn failed" + wait "$rerun_pid" + rerun_status=$(cat "$case_dir/rerun.status") + [ "$rerun_status" -eq 0 ] || fail "marker-failure-race: rerun failed: $(cat "$case_dir/stderr2")" + [ "$(wc -l < "$case_dir/treehouse.log")" -eq 1 ] \ + || fail "marker-failure-race: rerun re-returned the replacement tenant's slot" + [ "$(git -C "$case_dir/wt" rev-parse --abbrev-ref HEAD)" = "fm/task-x2" ] \ + || fail "marker-failure-race: rerun moved the replacement tenant's branch" + [ -f "$case_dir/wt/tenant.txt" ] || fail "marker-failure-race: rerun destroyed replacement tenant work" + [ -e "$case_dir/state/task-x2.meta" ] || fail "marker-failure-race: rerun removed replacement tenant metadata" + [ ! -e "$case_dir/state/task-x1.meta" ] || fail "marker-failure-race: rerun retained predecessor metadata" + pass "marker failure and pre-publication slot reissue leave the replacement tenant untouched" +} + +# A plain first-run teardown of a task that still owns its worktree must return +# it exactly as before the guard existed. +test_first_run_still_returns_worktree() { + local case_dir + case_dir=$(make_case first-run-return) + write_meta "$case_dir" local-only ship + wt_commit "$case_dir" "landed work" + merge_wt_into_local_main "$case_dir" + add_logging_treehouse "$case_dir" + + run_teardown "$case_dir" > "$case_dir/stdout" 2> "$case_dir/stderr" \ + || fail "first-run-return: teardown failed: $(cat "$case_dir/stderr")" + grep -q "return --force $case_dir/wt" "$case_dir/treehouse.log" \ + || fail "first-run-return: the worktree was never returned: $(cat "$case_dir/treehouse.log")" + grep -q "skipping worktree return" "$case_dir/stdout" \ + && fail "first-run-return: a first run wrongly skipped the return" + [ ! -e "$case_dir/state/task-x1.worktree-returned" ] \ + || fail "first-run-return: the return marker survived a completed teardown" + pass "a first-run teardown of an owned worktree still returns it to the pool" +} + +# A rerun whose worktree is still bound to this task and un-returned (the first +# attempt failed BEFORE or AT the return) must perform the return. +test_rerun_still_bound_unreturned_returns() { + local case_dir rc + case_dir=$(make_case rerun-unreturned) + write_meta "$case_dir" local-only ship + wt_commit "$case_dir" "landed work" + merge_wt_into_local_main "$case_dir" + + # Attempt 1: the return itself fails with a non-lock error, so no marker may + # be written and the rerun must retry the return. + cat > "$case_dir/fakebin/treehouse" <<'SH' +#!/usr/bin/env bash +echo "error: pool server unavailable" >&2 +exit 1 +SH + chmod +x "$case_dir/fakebin/treehouse" + rc=0 + run_teardown "$case_dir" > "$case_dir/stdout" 2> "$case_dir/stderr" || rc=$? + [ "$rc" -ne 0 ] || fail "rerun-unreturned: attempt 1 reported success although the return failed" + [ ! -e "$case_dir/state/task-x1.worktree-returned" ] \ + || fail "rerun-unreturned: a failed return still wrote the returned marker" + + add_logging_treehouse "$case_dir" + run_teardown "$case_dir" > "$case_dir/stdout2" 2> "$case_dir/stderr2" \ + || fail "rerun-unreturned: the rerun failed: $(cat "$case_dir/stderr2")" + grep -q "return --force $case_dir/wt" "$case_dir/treehouse.log" \ + || fail "rerun-unreturned: the rerun never returned the still-bound worktree" + grep -q "teardown task-x1 complete" "$case_dir/stdout2" \ + || fail "rerun-unreturned: the rerun did not report completion" + pass "a rerun with the worktree still bound to this task and un-returned performs the return" +} + +# Second signal: even with the durable marker lost, another task's meta binding +# the same worktree path proves reissue and must skip every worktree step. +test_reissued_slot_without_marker_skips_return() { + local case_dir + case_dir=$(make_case reissue-no-marker) + write_meta "$case_dir" local-only ship + wt_commit "$case_dir" "landed work" + merge_wt_into_local_main "$case_dir" + add_logging_treehouse "$case_dir" + fm_write_meta "$case_dir/state/task-x2.meta" \ + "window=firstmate:fm-task-x2" \ + "endpoint_task_id=task-x2" \ + "worktree=$case_dir/wt" \ + "project=$case_dir/project" \ + "kind=ship" \ + "mode=no-mistakes" + + run_teardown "$case_dir" > "$case_dir/stdout" 2> "$case_dir/stderr" \ + || fail "reissue-no-marker: teardown failed: $(cat "$case_dir/stderr")" + [ ! -s "$case_dir/treehouse.log" ] \ + || fail "reissue-no-marker: the reissued slot was still returned: $(cat "$case_dir/treehouse.log")" + grep -q "recorded for task task-x2" "$case_dir/stdout" \ + || fail "reissue-no-marker: the skip line did not name the tenant" + [ "$(git -C "$case_dir/wt" rev-parse --abbrev-ref HEAD)" = "fm/task-x1" ] \ + || fail "reissue-no-marker: the tenant's checked-out branch was dropped" + [ -e "$case_dir/state/task-x2.meta" ] || fail "reissue-no-marker: the tenant's metadata was removed" + [ ! -e "$case_dir/state/task-x1.meta" ] || fail "reissue-no-marker: the old metadata survived" + grep -q "teardown task-x1 complete" "$case_dir/stdout" \ + || fail "reissue-no-marker: teardown did not report completion" + pass "a reissued slot is never re-returned even when the durable marker is lost" +} + +# Mirror of the incident window: a PREDECESSOR of this slot already returned it +# (its own worktree-returned marker proves that) but awaits its sanctioned +# rerun, so its stale meta still binds the same worktree path. The CURRENT +# tenant's teardown must ignore that past-tenancy binding and still perform +# its own return. +test_returned_predecessor_binding_does_not_skip_current_return() { + local case_dir + case_dir=$(make_case predecessor-returned) + write_meta "$case_dir" local-only ship + wt_commit "$case_dir" "landed work" + merge_wt_into_local_main "$case_dir" + add_logging_treehouse "$case_dir" + fm_write_meta "$case_dir/state/task-x0.meta" \ + "window=firstmate:fm-task-x0" \ + "endpoint_task_id=task-x0" \ + "worktree=$case_dir/wt" \ + "project=$case_dir/project" \ + "kind=ship" \ + "mode=no-mistakes" + : > "$case_dir/state/task-x0.worktree-returned" + + run_teardown "$case_dir" > "$case_dir/stdout" 2> "$case_dir/stderr" \ + || fail "predecessor-returned: teardown failed: $(cat "$case_dir/stderr")" + grep -q "return --force $case_dir/wt" "$case_dir/treehouse.log" \ + || fail "predecessor-returned: the owned slot was never returned: $(cat "$case_dir/treehouse.log")" + grep -q "skipping worktree return" "$case_dir/stdout" \ + && fail "predecessor-returned: the predecessor's stale binding wrongly skipped the return" + [ -e "$case_dir/state/task-x0.worktree-returned" ] \ + || fail "predecessor-returned: the predecessor's own return marker was removed" + [ -e "$case_dir/state/task-x0.meta" ] \ + || fail "predecessor-returned: the predecessor's metadata was removed" + [ ! -e "$case_dir/state/task-x1.worktree-returned" ] \ + || fail "predecessor-returned: the return marker survived a completed teardown" + grep -q "teardown task-x1 complete" "$case_dir/stdout" \ + || fail "predecessor-returned: teardown did not report completion" + pass "a returned predecessor's stale binding does not block the current tenant's own return" +} + test_local_only_fork_remote_allows test_teardown_closes_the_backlog_item_itself test_teardown_manual_backend_leaves_the_backlog_to_the_operator @@ -2663,3 +3008,10 @@ test_process_spawned_during_grace_is_reaped_on_later_pass test_persistent_scan_refuses_after_bounded_retries test_process_exit_during_identity_lookup_does_not_refuse test_run_abort_precedes_process_reap_precedes_worktree_removal +test_rerun_after_return_and_reissue_never_rereturns +test_teardown_and_spawn_share_task_set_first_lock_order +test_marker_failure_reissue_before_rerun_preserves_new_tenant +test_first_run_still_returns_worktree +test_rerun_still_bound_unreturned_returns +test_reissued_slot_without_marker_skips_return +test_returned_predecessor_binding_does_not_skip_current_return diff --git a/tests/fm-turnend-guard.test.sh b/tests/fm-turnend-guard.test.sh index 54cfcdae861..c0a3eb0244e 100755 --- a/tests/fm-turnend-guard.test.sh +++ b/tests/fm-turnend-guard.test.sh @@ -1448,6 +1448,284 @@ test_hook_claude_mode_terminal_fail_open_clears_abandoned_claim() { pass "fm-turnend-guard --claude: the terminal path clears an abandoned claim instead of stepping aside silently" } +# --- bounding consecutive blocks when the automatic path never runs ----------- +# The measured 2026-08-27 lapse: the Stop-owned auto-arm never claimed the home +# after an away-mode episode, so the epoch ledger never changed. COUNT is keyed +# on that epoch identity, so it froze at 1, and the one attended fail-open also +# requires a VERIFIED failure episode - evidence only an auto-arm that RUNS can +# produce. The result was 172 consecutive blocked stops over five hours with no +# bound and no alarm. A run of blocked stops is itself evidence that supervision +# is down, so it opens the same one alarm on its own. +seed_frozen_ledger() { # + printf 'epoch=918 owner_pid=999 outcome=afk updated_at=1\n' > "$1/state/.claude-autoarm-epoch" + touch -t 202001010000 "$1/state/.claude-autoarm-epoch" +} + +budget_field() { # + sed -n "s/^$2=//p" "$1/state/.turnend-claude-blocks" 2>/dev/null || true +} + +test_hook_claude_mode_bounds_blocks_when_the_auto_arm_never_claims() { + local dir out status i + dir=$(make_primary_dir "$TMP_ROOT/hook-claude-stall-bound") + : > "$dir/state/task1.meta" + seed_frozen_ledger "$dir" + + for i in 1 2 3; do + out=$(FM_CLAUDE_TURNEND_STALL_BUDGET=3 FM_CLAUDE_AUTOARM_SYNC_WAIT_MS=100 \ + run_hook_claude "$dir" true); status=$? + expect_code 2 "$status" "block $i must still refuse a blind turn end" + assert_contains "$out" "TURN WOULD END BLIND" "block $i lost the blind-turn banner" + done + # The divergence this bound exists for: the epoch-keyed count is still frozen + # at its first value, so on its own it could never reach any budget. + [ "$(budget_field "$dir" count)" = 1 ] \ + || fail "the fixture is vacuous: the epoch-keyed count advanced on a frozen ledger ($(budget_field "$dir" count))" + [ "$(budget_field "$dir" stalled)" = 3 ] \ + || fail "consecutive blocked stops were not counted: $(budget_field "$dir" stalled)" + + out=$(FM_CLAUDE_TURNEND_STALL_BUDGET=3 FM_CLAUDE_AUTOARM_SYNC_WAIT_MS=100 \ + run_hook_claude "$dir" true); status=$? + expect_code 0 "$status" "the bound must open the one attended fail-open once the run of blocks passes it" + assert_contains "$out" 'FIRSTMATE SUPERVISION IS GENUINELY DOWN' "the bounded fail-open produced no captain-visible alarm" + assert_contains "$out" 'without the Stop-owned auto-arm ever claiming this home' \ + "the alarm did not name the automatic path that never ran" + # It must NOT borrow the failure-alarm marker: that marker also tells the + # auto-arm to stop creating exit-2 continuations, which would swallow the first + # real wake of an auto-arm that comes back. + assert_absent "$dir/state/.claude-autoarm-failure-alarmed" \ + "the stalled fail-open consumed the auto-arm's failure-episode marker" + [ "$(budget_field "$dir" stalled)" = 0 ] \ + || fail "the attended alarm did not end its blocked-stop series" + + for i in 1 2 3; do + out=$(FM_CLAUDE_TURNEND_STALL_BUDGET=3 FM_CLAUDE_AUTOARM_SYNC_WAIT_MS=100 \ + run_hook_claude "$dir" true); status=$? + expect_code 2 "$status" "new episode block $i must refuse the blind turn end" + assert_contains "$out" "TURN WOULD END BLIND" "new episode block $i lost the blind-turn banner" + assert_not_contains "$out" 'GENUINELY DOWN' "the next episode alarmed before its own budget" + done + out=$(FM_CLAUDE_TURNEND_STALL_BUDGET=3 FM_CLAUDE_AUTOARM_SYNC_WAIT_MS=100 \ + run_hook_claude "$dir" true); status=$? + expect_code 0 "$status" "the next uninterrupted episode must retain its own attended alarm" + assert_contains "$out" 'FIRSTMATE SUPERVISION IS GENUINELY DOWN' \ + "the next episode never reached its attended alarm" + pass "fm-turnend-guard --claude: consecutive blocked episodes each retain one bounded alarm" +} + +test_hook_claude_mode_marker_failure_preserves_stall_fallback() { + local dir out status + dir=$(make_primary_dir "$TMP_ROOT/hook-claude-alarm-marker-failure") + : > "$dir/state/task1.meta" + seed_claude_failure "$dir" + printf 'session=sess-claude-mode\ncount=4\nepoch=3\nstalled=6\n' > "$dir/state/.turnend-claude-blocks" + ln -s "$dir/missing/alarm" "$dir/state/.claude-autoarm-failure-alarmed" + + out=$(FM_CLAUDE_AUTOARM_SYNC_WAIT_MS=100 run_hook_claude "$dir" true); status=$? + expect_code 2 "$status" "failed failure-alarm publication must keep blocking" + assert_not_contains "$out" 'FIRSTMATE SUPERVISION IS GENUINELY DOWN' \ + "failed marker publication reported an alarm it did not commit" + [ "$(budget_field "$dir" stalled)" = 7 ] \ + || fail "failed marker publication did not restore the pre-reset stall progression" + + out=$(FM_CLAUDE_AUTOARM_SYNC_WAIT_MS=100 run_hook_claude "$dir" true); status=$? + expect_code 0 "$status" "the preserved series must reach its marker-independent stall alarm" + assert_contains "$out" 'FIRSTMATE SUPERVISION IS GENUINELY DOWN' \ + "persistent marker failure prevented the bounded stall alarm" + assert_contains "$out" 'without the Stop-owned auto-arm ever claiming this home' \ + "the fallback alarm did not identify the stalled automatic path" + [ "$(budget_field "$dir" stalled)" = 0 ] \ + || fail "the successful fallback alarm did not reset its series" + pass "fm-turnend-guard --claude: marker failure preserves the bounded stall fallback" +} + +test_hook_claude_mode_alarm_commits_stall_reset_before_unlock() { + local dir fakebin real_mv out status + dir=$(make_primary_dir "$TMP_ROOT/hook-claude-alarm-lock-contention") + : > "$dir/state/task1.meta" + seed_claude_failure "$dir" + printf 'session=sess-claude-mode\ncount=4\nepoch=3\nstalled=2\n' > "$dir/state/.turnend-claude-blocks" + fakebin="$dir/fakebin" + real_mv=$(command -v mv) + mkdir -p "$fakebin" + cat > "$fakebin/mv" </dev/null \ + && [ ! -d "\$FM_RACE_OWNER_LOCK" ]; then + exit 1 +fi +exec "$real_mv" "\$@" +SH + chmod +x "$fakebin/mv" + + out=$(printf '{"stop_hook_active":true,"session_id":"sess-claude-mode"}' \ + | PATH="$fakebin:$PATH" FM_RACE_OWNER_LOCK="$dir/state/.claude-autoarm.lock" \ + FM_RACE_BUDGET="$dir/state/.turnend-claude-blocks" \ + FM_CLAUDE_AUTOARM_SYNC_WAIT_MS=100 CLAUDECODE=1 FM_HOME="$dir" \ + bash "$dir/bin/fm-turnend-guard.sh" --claude 2>&1); status=$? + expect_code 0 "$status" "the attended alarm must commit its stall reset inside the terminal lock boundary" + assert_contains "$out" 'FIRSTMATE SUPERVISION IS GENUINELY DOWN' \ + "a post-unlock reset failure swallowed the committed attended alarm" + assert_present "$dir/state/.claude-autoarm-failure-alarmed" \ + "the successful failure alarm did not retain its one-shot marker" + [ "$(budget_field "$dir" stalled)" = 0 ] \ + || fail "the terminal decision did not clear stalled state before releasing its locks" + pass "fm-turnend-guard --claude: attended decisions clear stalled state before unlock" +} + +test_hook_claude_mode_default_stall_alarm_precedes_hard_override() { + local dir out status i + dir=$(make_primary_dir "$TMP_ROOT/hook-claude-default-stall-bound") + : > "$dir/state/task1.meta" + seed_frozen_ledger "$dir" + + for i in 1 2 3 4 5 6 7; do + out=$(FM_CLAUDE_AUTOARM_SYNC_WAIT_MS=100 run_hook_claude "$dir" true); status=$? + expect_code 2 "$status" "default stall block $i must refuse the blind turn end" + assert_not_contains "$out" 'GENUINELY DOWN' "the default stall alarm fired before seven blocks" + done + out=$(FM_CLAUDE_AUTOARM_SYNC_WAIT_MS=100 run_hook_claude "$dir" true); status=$? + expect_code 0 "$status" "the eighth Stop attempt must alarm rather than become an eighth blocked Stop" + assert_contains "$out" 'FIRSTMATE SUPERVISION IS GENUINELY DOWN' "the pre-override attended alarm is missing" + [ "$(budget_field "$dir" stalled)" = 0 ] \ + || fail "the default attended alarm did not reset its completed series" + pass "fm-turnend-guard --claude: the default attended alarm precedes Claude's eight-block override" +} + +test_hook_claude_mode_clamps_unsafe_stall_override() { + local dir out status i + dir=$(make_primary_dir "$TMP_ROOT/hook-claude-clamped-stall-bound") + : > "$dir/state/task1.meta" + seed_frozen_ledger "$dir" + + for i in 1 2 3 4 5 6 7; do + out=$(FM_CLAUDE_TURNEND_STALL_BUDGET=99 FM_CLAUDE_AUTOARM_SYNC_WAIT_MS=100 \ + run_hook_claude "$dir" true); status=$? + expect_code 2 "$status" "clamped stall block $i must refuse the blind turn end" + done + out=$(FM_CLAUDE_TURNEND_STALL_BUDGET=99 FM_CLAUDE_AUTOARM_SYNC_WAIT_MS=100 \ + run_hook_claude "$dir" true); status=$? + expect_code 0 "$status" "an unsafe stall override must alarm before the hard override" + assert_contains "$out" 'FIRSTMATE SUPERVISION IS GENUINELY DOWN' "the clamped override produced no attended alarm" + [ "$(budget_field "$dir" stalled)" = 0 ] \ + || fail "the clamped attended alarm did not reset its completed series" + pass "fm-turnend-guard --claude: unsafe stall overrides clamp below Claude's hard limit" +} + +test_hook_claude_mode_terminal_recovery_clears_stall_series() { + local dir fakebin ready release once real_mv guard_pid out status pid identity i + dir=$(make_primary_dir "$TMP_ROOT/hook-claude-terminal-stall-recovery") + : > "$dir/state/task1.meta" + seed_frozen_ledger "$dir" + printf 'session=sess-claude-mode\ncount=1\nepoch=918\nstalled=7\n' > "$dir/state/.turnend-claude-blocks" + fakebin="$dir/fakebin" + ready="$dir/budget-ready" + release="$dir/budget-release" + once="$dir/budget-once" + real_mv=$(command -v mv) + mkdir -p "$fakebin" + cat > "$fakebin/mv" < "\$FM_RACE_ONCE") 2>/dev/null; then + : > "\$FM_RACE_READY" + while [ ! -e "\$FM_RACE_RELEASE" ]; do sleep 0.05; done +fi +exec "$real_mv" "\$@" +SH + chmod +x "$fakebin/mv" + + ( + printf '{"stop_hook_active":true,"session_id":"sess-claude-mode"}' \ + | PATH="$fakebin:$PATH" FM_RACE_BUDGET="$dir/state/.turnend-claude-blocks" \ + FM_RACE_ONCE="$once" FM_RACE_READY="$ready" FM_RACE_RELEASE="$release" \ + FM_CLAUDE_AUTOARM_SYNC_WAIT_MS=100 CLAUDECODE=1 FM_HOME="$dir" \ + bash "$dir/bin/fm-turnend-guard.sh" --claude > "$dir/guard.out" 2>&1 + printf '%s\n' "$?" > "$dir/guard.status" + ) & + guard_pid=$! + for i in $(seq 1 100); do + [ ! -e "$ready" ] || break + sleep 0.05 + done + [ -e "$ready" ] || fail "terminal-stall-recovery: guard never reached the threshold boundary" + + sleep 60 & + pid=$! + identity=$(fm_test_pid_identity "$pid") || fail "terminal-stall-recovery: could not identify claim owner" + printf 'epoch=919 owner_pid=%s outcome=arming updated_at=%s\n%s\n' "$pid" "$(date +%s)" "$identity" \ + > "$dir/state/.claude-autoarm-epoch" + : > "$dir/state/.last-watcher-beat" + : > "$release" + wait "$guard_pid" + status=$(cat "$dir/guard.status") + expect_code 0 "$status" "a claim opening at the terminal boundary must own recovery" + [ "$(budget_field "$dir" stalled)" = 0 ] \ + || fail "terminal-stall-recovery: recovery-owned allow retained the exhausted stall counter" + + kill "$pid" 2>/dev/null || true + wait "$pid" 2>/dev/null || true + seed_frozen_ledger "$dir" + out=$(FM_CLAUDE_AUTOARM_SYNC_WAIT_MS=100 run_hook_claude "$dir" true); status=$? + expect_code 2 "$status" "a fault after terminal recovery must start a new blocked series" + [ "$(budget_field "$dir" stalled)" = 1 ] \ + || fail "terminal-stall-recovery: later fault did not restart the stall series at one" + pass "fm-turnend-guard --claude: terminal recovery clears exhausted stall progression" +} + +test_hook_claude_mode_stall_bound_resets_on_a_turn_it_lets_through() { + local dir out status i + dir=$(make_primary_dir "$TMP_ROOT/hook-claude-stall-reset") + : > "$dir/state/task1.meta" + seed_frozen_ledger "$dir" + + for i in 1 2 3; do + FM_CLAUDE_TURNEND_STALL_BUDGET=3 FM_CLAUDE_AUTOARM_SYNC_WAIT_MS=100 \ + run_hook_claude "$dir" true >/dev/null 2>&1 || true + done + [ "$(budget_field "$dir" stalled)" = 3 ] || fail "the fixture did not accumulate a run of blocks" + + # The auto-arm claims this event's recovery: that is real progress, so the run + # of consecutive blocks ends here rather than carrying into the next fault. + printf 'epoch=919 owner_pid=999 outcome=rewake updated_at=%s\n' "$(date +%s)" \ + > "$dir/state/.claude-autoarm-epoch" + out=$(FM_CLAUDE_TURNEND_STALL_BUDGET=3 run_hook_claude "$dir" true); status=$? + expect_code 0 "$status" "a rewake the auto-arm owns must still allow the stop" + [ "$(budget_field "$dir" stalled)" = 0 ] \ + || fail "an allowed turn did not end the run of consecutive blocks: $(budget_field "$dir" stalled)" + + seed_frozen_ledger "$dir" + out=$(FM_CLAUDE_TURNEND_STALL_BUDGET=3 FM_CLAUDE_AUTOARM_SYNC_WAIT_MS=100 \ + run_hook_claude "$dir" true); status=$? + expect_code 2 "$status" "the next fault must start a fresh run of blocks, not inherit the old one" + assert_not_contains "$out" 'GENUINELY DOWN' "a reset run alarmed on its first block instead of counting again" + [ "$(budget_field "$dir" stalled)" = 1 ] \ + || fail "the fresh run did not start counting from one: $(budget_field "$dir" stalled)" + pass "fm-turnend-guard --claude: a turn the guard lets through ends the run of consecutive blocks" +} + +test_hook_claude_mode_stall_bound_stays_shut_in_away_mode() { + local dir out status i + dir=$(make_primary_dir "$TMP_ROOT/hook-claude-stall-afk") + : > "$dir/state/task1.meta" + : > "$dir/state/.afk" + seed_frozen_ledger "$dir" + + for i in 1 2 3 4 5; do + out=$(FM_CLAUDE_TURNEND_STALL_BUDGET=3 FM_CLAUDE_AUTOARM_SYNC_WAIT_MS=100 \ + run_hook_claude "$dir" true); status=$? + expect_code 2 "$status" "away mode must keep refusing blind stops rather than opening the attended alarm" + done + assert_absent "$dir/state/.claude-autoarm-failure-alarmed" "away mode spent the attended alarm with nobody attending" + pass "fm-turnend-guard --claude: the consecutive-block bound never opens while the away daemon owns supervision" +} + test_hook_claude_mode_preserves_fresh_failed_progression() { local dir out status count dir=$(make_primary_dir "$TMP_ROOT/hook-claude-failed-epoch") @@ -1808,6 +2086,14 @@ test_hook_claude_mode_blocks_on_stuck_arming_claim test_hook_claude_mode_allows_on_open_generation_claim test_hook_claude_mode_blocks_on_stuck_generation_claim test_hook_claude_mode_terminal_fail_open_clears_abandoned_claim +test_hook_claude_mode_bounds_blocks_when_the_auto_arm_never_claims +test_hook_claude_mode_marker_failure_preserves_stall_fallback +test_hook_claude_mode_alarm_commits_stall_reset_before_unlock +test_hook_claude_mode_default_stall_alarm_precedes_hard_override +test_hook_claude_mode_clamps_unsafe_stall_override +test_hook_claude_mode_terminal_recovery_clears_stall_series +test_hook_claude_mode_stall_bound_resets_on_a_turn_it_lets_through +test_hook_claude_mode_stall_bound_stays_shut_in_away_mode test_hook_claude_mode_preserves_fresh_failed_progression test_hook_claude_mode_integrated_monotonic_fail_open test_hook_claude_mode_recovery_contention_is_not_ordinary_allow diff --git a/tests/fm-watch-triage.test.sh b/tests/fm-watch-triage.test.sh index 04a8caaea9e..121c2a81ae6 100755 --- a/tests/fm-watch-triage.test.sh +++ b/tests/fm-watch-triage.test.sh @@ -2054,6 +2054,61 @@ test_exited_declared_pause_is_bounded_but_live_gate_surfaces() { pass "exited declared-pause and captain-held panes use bounded pause cadence while a live decision gate still surfaces once" } +# A declared pause surfaces once on first sight under the long-cadence semantics, +# but a declared pause must never mask an agent that subsequently dies: endpoint- +# mort detection keeps priority over the long cadence. So when the same pane goes +# dead after surfacing as a live pause, it must still re-surface on the bounded +# pause cadence (the death is detected for a recheck), never stay hidden behind +# the pause cadence and never be mislabeled a wedge. +test_live_paused_agent_death_still_rechecks() { + local dir state fakebin out capture_file statusf window key pane_hash sig pid back + dir=$(make_case live-paused-then-dead); state="$dir/state"; fakebin="$dir/fakebin" + out="$dir/watch.out"; capture_file="$dir/pane.txt"; statusf="$state/gate.status" + window="test:fm-gate" + printf 'idle external-decision gate\n' > "$capture_file" + printf 'window=%s\nkind=ship\nharness=grok\nbackend=tmux\n' "$window" > "$state/gate.meta" + printf 'paused: waiting at an active external-decision gate\n' > "$statusf" + sig=$(seen_sig "$statusf"); printf '%s' "$sig" > "$state/.seen-gate_status" + key=$(printf '%s' "$window" | tr ':/.' '___') + pane_hash=$(hash_text "idle external-decision gate") + printf '%s' "$pane_hash" > "$state/.hash-$key" + printf '1\n' > "$state/.count-$key" + + # Phase A: the live agent's declared pause surfaces once on first sight under + # the long-cadence semantics, recording its pause marker, then is acknowledged. + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \ + FM_FAKE_TMUX_CURRENT_COMMAND=grok FM_FAKE_CREW_STATE='state: paused · source: status-log · waiting at an active external-decision gate' \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_PAUSE_RESURFACE_SECS=999 FM_POLL=1 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + wait_for_exit "$pid" 100 || fail "live paused agent did not surface once on first sight: $(cat "$out")" + [ -e "$state/.paused-$key" ] || fail "live paused surface did not record its pause marker" + ack_stopped_cycle "$state" || fail "could not acknowledge the live-pause surface" + + # Phase B: the agent dies (stopped, bare shell) while the pause status remains. + # The death must still be detected on the bounded pause cadence: age the status + # and the re-surface throttle past PAUSE_RESURFACE_SECS and confirm the pane + # re-surfaces as a paused recheck, not a wedge and not hidden behind the pause + # cadence. (Phase A's first-sight surface set the resurface throttle, so the + # re-surface is due once that cadence elapses, not immediately.) + printf 'idle bare shell after agent death\n' > "$capture_file" + back=$(( $(date +%s) - 500 )) + if [ "$(uname)" = Darwin ]; then touch -mt "$(date -r "$back" '+%Y%m%d%H%M.%S')" "$statusf" "$state/.paused-resurfaced-$key" + else touch -m -d "@$back" "$statusf" "$state/.paused-resurfaced-$key"; fi + sig=$(seen_sig "$statusf"); printf '%s' "$sig" > "$state/.seen-gate_status" + : > "$out" + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \ + FM_FAKE_TMUX_CURRENT_COMMAND=zsh FM_FAKE_CREW_STATE='state: stopped · source: pane · bare shell' \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_STALE_ESCALATE_SECS=240 FM_PAUSE_RESURFACE_SECS=240 FM_POLL=1 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + wait_for_exit "$pid" 100 || fail "dead-after-pause agent did not re-surface on the bounded cadence: $(cat "$out")" + grep -F "stale: $window" "$out" >/dev/null || fail "dead-after-pause agent did not re-surface with a stale wake" + grep -F "awaiting external" "$out" >/dev/null || fail "dead-after-pause re-surface was not labeled a paused recheck" + grep -F "possible wedge" "$out" >/dev/null && fail "dead-after-pause agent was mislabeled a possible wedge" + pass "a live paused agent that subsequently dies still re-surfaces on the bounded pause cadence (death not masked)" +} + test_secondmate_paused_resurfaces_in_normal_mode() { local dir state fakebin out capture_file statusf window key pane_hash sig pid back dir=$(make_case secondmate-paused-resurface); state="$dir/state"; fakebin="$dir/fakebin" @@ -3861,6 +3916,7 @@ test_afk_busy_declared_pause_ticking_pane_hands_off_once test_nonterminal_stale_not_working_surfaced test_nonterminal_stale_paused_absorbed_then_resurfaced test_exited_declared_pause_is_bounded_but_live_gate_surfaces +test_live_paused_agent_death_still_rechecks test_secondmate_paused_resurfaces_in_normal_mode test_secondmate_captain_held_resurfaces_in_normal_mode test_secondmate_nonpaused_stale_remains_suppressed