diff --git a/bin/backends/cmux.sh b/bin/backends/cmux.sh index 0d9791216a3..c0f5f8d1b09 100644 --- a/bin/backends/cmux.sh +++ b/bin/backends/cmux.sh @@ -444,7 +444,8 @@ fm_backend_cmux_target_ready() { # [expected-label] # polling cannot solve this here any more than it could for zellij. Active # probe instead: print the surface's `$PWD` with a unique marker (atomically # submitted via send_text_line), briefly settle, then capture and read only -# that marker line. Scoped to fm-spawn.sh's own worktree-discovery poll loop. +# that marker line. Scoped to fm-spawn.sh's own worktree-cwd confirmation poll +# loop. fm_backend_cmux_current_path() { # [expected-label] local target=$1 expected_label=${2:-} out line marker_begin="__FM_CMUX_CWD_BEGIN__" marker_end="__FM_CMUX_CWD_END__" in_block=0 chunk="" last="" fm_backend_cmux_target_ready "$target" "$expected_label" || return 0 diff --git a/bin/backends/herdr.sh b/bin/backends/herdr.sh index 8728b356cc0..713622449e3 100644 --- a/bin/backends/herdr.sh +++ b/bin/backends/herdr.sh @@ -2520,17 +2520,15 @@ fm_backend_herdr_target_ready() { # } # fm_backend_herdr_current_path: the live FOREGROUND process's cwd, or empty on -# any error. Mirrors tmux's pane_current_path poll used for worktree-path -# discovery after `treehouse get`. +# any error. Mirrors tmux's pane_current_path poll fm-spawn.sh uses to confirm +# the pane's shell sits in the leased worktree it was created in. # # Verified pitfall: `pane get`'s `.result.pane.cwd` is the pane's cwd AT # CREATION TIME - the top-level shell's cwd - and does NOT update when that -# shell `cd`s or enters a subshell (as `treehouse get` does). Reading it here -# would make fm-spawn.sh's worktree-discovery poll never see the pane "leave" -# the project directory, since `cwd` stays frozen at the original path forever. -# `.result.pane.foreground_cwd` tracks the ACTUALLY RUNNING foreground -# process's cwd instead, which is what changes when `treehouse get` enters its -# worktree subshell - confirmed live against a real treehouse acquisition. +# shell `cd`s or enters a subshell. `.result.pane.foreground_cwd` tracks the +# ACTUALLY RUNNING foreground process's cwd instead - confirmed live against a +# real treehouse subshell - so it is the read that reports where the shell +# really is rather than where the pane was opened. fm_backend_herdr_current_path() { # fm_backend_herdr_target_ready "$1" || return 0 fm_backend_herdr_cli "$FM_BACKEND_HERDR_SESSION" pane get "$FM_BACKEND_HERDR_PANE" 2>/dev/null \ @@ -2539,8 +2537,8 @@ fm_backend_herdr_current_path() { # # fm_backend_herdr_send_text_line: send one line of TEXT then submit, # ATOMICALLY - mirrors tmux's `send-keys -t T text Enter`. Used for the fixed -# spawn-time commands (treehouse get, the GOTMPDIR export). `pane run` types -# the command and submits it in one call (verified). +# spawn-time commands (the GOTMPDIR export). `pane run` types the command and +# submits it in one call (verified). fm_backend_herdr_send_text_line() { # fm_backend_herdr_target_ready "$1" || return 1 fm_backend_herdr_cli "$FM_BACKEND_HERDR_SESSION" pane run "$FM_BACKEND_HERDR_PANE" "$2" >/dev/null 2>&1 diff --git a/bin/backends/tmux.sh b/bin/backends/tmux.sh index 9eed5f3ec3e..b13ad81f40d 100644 --- a/bin/backends/tmux.sh +++ b/bin/backends/tmux.sh @@ -8,10 +8,10 @@ # default (tmux, `backend=` absent) path stays byte-identical. Sourced only # through bin/fm-backend.sh's fm_backend_source, never directly. # -# Worktree acquisition (running `treehouse get` inside the pane, and polling -# its cwd) is unchanged by this extraction: P1 scopes only the session -# provider, not the worktree provider, so fm-spawn.sh still drives that part -# inline with these same send/current-path primitives. +# Worktree acquisition is not this adapter's concern: bin/fm-spawn.sh leases +# and refreshes the copy itself, creates the window with that path as its +# starting directory, and confirms the pane's cwd through the current-path +# primitive here (fm-spawn.sh's header owns that order). # # The verified composer/busy-detection and verify-and-retry-submit primitives # already live in bin/fm-tmux-lib.sh, shared with the away-mode daemon @@ -82,11 +82,13 @@ fm_backend_tmux_container_ensure() { # at the next free index by targeting the session with a trailing colon # ("$ses:"), so a non-default base-index (e.g. base-index 1) cannot collide. # - PIN the window name by disabling automatic-rename and allow-rename on the -# new window: the captain's tmux may rename the window away from fm- once -# treehouse cd's into the worktree, which would break name-based targeting. +# new window: the captain's tmux may rename the window away from fm- as +# soon as its shell starts in the worktree, which would break name-based +# targeting. # The returned window id lets callers target the window even if its name is ever -# lost, so worktree discovery cannot fall back to the active client's window. -fm_backend_tmux_create_task() { # -> prints window id +# lost, so worktree cwd confirmation cannot fall back to the active client's +# window. +fm_backend_tmux_create_task() { # -> prints window id local ses=$1 wname=$2 proj_abs=$3 wid if tmux list-windows -t "$ses" -F '#{window_name}' | grep -qx "$wname"; then echo "error: window $ses:$wname already exists" >&2 @@ -99,16 +101,16 @@ fm_backend_tmux_create_task() { # -> prints } # fm_backend_tmux_current_path: the live pane's current working directory, or -# empty on any tmux error. Mirrors fm-spawn.sh's worktree-discovery poll: +# empty on any tmux error. Mirrors fm-spawn.sh's worktree-cwd confirmation poll: # `tmux display-message -p -t "$T" '#{pane_current_path}'`. fm_backend_tmux_current_path() { # tmux display-message -p -t "$1" '#{pane_current_path}' 2>/dev/null } # fm_backend_tmux_send_text_line: send one line of TEXT then Enter, with no -# composer verification - used for the fixed spawn-time commands -# (`treehouse get`, the GOTMPDIR export) that already ran this exact sequence -# inline in fm-spawn.sh. Mirrors `tmux send-keys -t "$T" "" Enter`. +# composer verification - used for the fixed spawn-time commands (the GOTMPDIR +# export) that already ran this exact sequence inline in fm-spawn.sh. Mirrors +# `tmux send-keys -t "$T" "" Enter`. fm_backend_tmux_send_text_line() { # tmux send-keys -t "$1" "$2" Enter } diff --git a/bin/backends/zellij.sh b/bin/backends/zellij.sh index 56478f7db35..2648e7b6644 100644 --- a/bin/backends/zellij.sh +++ b/bin/backends/zellij.sh @@ -62,7 +62,7 @@ # contradicts the design report's assumption ("acceptable for tmux and # zellij") and required a different implementation strategy - see # fm_backend_zellij_current_path below and docs/zellij-backend.md -# "Worktree-path discovery: pane_cwd does not track a subshell". +# "Current operation and safety". # 5. `new-tab` DOES steal focus from an attached client with NO flag to # suppress it (unlike herdr's --no-focus and tmux's new-window -d). # Mitigated (fm_backend_zellij_create_task): capture the previously @@ -387,24 +387,24 @@ fm_backend_zellij_target_ready() { # [expected-label] } # fm_backend_zellij_current_path: the live pane's cwd, or empty on any error. -# Mirrors tmux's pane_current_path poll used for worktree-path discovery after -# `treehouse get`. +# Mirrors tmux's pane_current_path poll fm-spawn.sh uses to confirm the pane's +# shell sits in the leased worktree it was created in. # -# Verified pitfall (docs/zellij-backend.md "Worktree-path discovery: pane_cwd -# does not track a subshell"): `list-panes --json`'s `pane_cwd` DOES reflect a +# Verified pitfall (docs/zellij-backend.md "Current operation and safety"): +# `list-panes --json`'s `pane_cwd` DOES reflect a # `cd` run directly in the pane's own top-level shell, but stays FROZEN at -# whatever directory the pane's shell was in when it launched `treehouse get` -# as a foreground command - it never follows that command's own internal `cd` -# into the acquired worktree, even after the subshell is fully interactive and +# whatever directory the pane's shell was in when it launched a foreground +# command such as `treehouse get` - it never follows that command's own internal +# `cd` into the acquired worktree, even after the subshell is fully interactive and # a `pwd` typed into it prints the correct live path on screen. Zellij's CLI # exposes no per-pane pid and no live-process cwd field to read instead # (unlike herdr's `foreground_cwd`), so passive JSON polling cannot solve # this. Active probe instead: print the pane's `$PWD` with a unique marker # (atomically submitted, mirroring send_text_line), briefly settle, then capture -# and read only that marker line. Scoped to fm-spawn.sh's own worktree-discovery -# poll loop (the only caller of this op), where injecting a harmless extra -# command before the harness ever launches is an acceptable trade for a reliable -# answer. +# and read only that marker line. Scoped to fm-spawn.sh's own worktree-cwd +# confirmation poll loop (the only caller of this op), where injecting a +# harmless extra command before the harness ever launches is an acceptable +# trade for a reliable answer. fm_backend_zellij_current_path() { # [expected-label] local target=$1 expected_label=${2:-} out line marker_begin="__FM_ZELLIJ_CWD_BEGIN__" marker_end="__FM_ZELLIJ_CWD_END__" in_block=0 chunk="" last="" fm_backend_zellij_target_ready "$target" "$expected_label" || return 0 diff --git a/bin/fm-spawn.sh b/bin/fm-spawn.sh index 9158fce64df..006d5f3088e 100755 --- a/bin/fm-spawn.sh +++ b/bin/fm-spawn.sh @@ -48,8 +48,8 @@ # then tmux. # Spawn-capable backends are the reference tmux adapter and experimental # herdr, zellij, orca, and cmux. Orca owns both the task worktree and -# terminal, so ship/scout Orca spawns do not run treehouse get; cmux is a -# session provider only, exactly like herdr/zellij, so it does. An +# terminal, so ship/scout Orca spawns never lease a treehouse copy; cmux is a +# session provider only, exactly like tmux/herdr/zellij, so it does. An # auto-detected herdr or cmux spawn prints a loud stderr notice; # auto-detected tmux stays silent; zellij and orca are never auto-detected. # codex-app is not a known backend yet; docs/codex-app-backend.md owns that @@ -138,6 +138,40 @@ # origin, resolves the current remote default branch, and resets to its tip. # An unreachable origin, unresolved default branch, or non-clean worktree # refuses the spawn rather than risking a PR based on stale history. +# This script is the one owner of the ORDER of that sequence on every +# session-provider backend (tmux, herdr, zellij, cmux): the copy is leased +# from this process with `treehouse get --lease` (no pane, no shell), the +# refresh above runs on that leased path, and only then is the task pane +# created with the leased path as its starting directory. No `treehouse get` +# is ever typed into the pane; the one exception is a reclaimed Herdr pane +# that already existed, which is sent a plain `cd` into the copy only after +# the refresh. A fresh spawn whose durable record still names a pooled copy +# (a quarantined Herdr recovery) refreshes that same copy instead of leasing +# another, and fetches and prunes its remote refs before refusing when its +# HEAD holds commits no remote has, so a retry never grows the pool or resets +# a prior worker's unlanded commits. +# Recovery first classifies the recorded endpoint and moves an agent-free +# shell back to the primary project before inspecting or refreshing its +# recorded copy. The reclaimed replacement likewise starts outside the copy +# and enters it only after refresh. +# The old order (create the pane, enter the copy +# from its shell, then refresh) raced a fresh shell's git prompt against the +# refresh for the worktree's index.lock under the shared .git/worktrees/, and +# refused the spawn every time on a large repository; the worker shell is +# simply not in the copy yet when the refresh runs. The pane is then polled +# once a second until its live cwd resolves to the leased path before launch; +# FM_SPAWN_SETTLE_POLLS bounds that wait (default 60). +# When a fresh spawn stops after the lease but BEFORE any backend endpoint +# creation has begun with the leased path as its cwd, a clean leased copy is +# returned to the pool so a refused launch cannot shrink the pool; a copy +# holding uncommitted work stays leased, untouched, and its path is reported. +# From the first create call onward the lease is retained: `treehouse return` +# terminates every process still inside the copy, so returning it after a +# pane, tab, or workspace was opened there would kill a shell the refusal +# deliberately left in place (a quarantined Herdr projection, an +# inspectable endpoint). A refusal in that window reports the retained path +# and its release command, and bin/fm-teardown.sh's `treehouse return` +# releases a launched task's lease. # A slot whose only deviation is a stale submodule gitlink is refused by that # same clean check, but is reported as a stale checkout naming each submodule # and both pins; nothing is converged or removed, and no remedy is suggested. @@ -727,6 +761,13 @@ RELAUNCH_REPLACEMENT_STATE= RELAUNCH_REPLACEMENT_WT= CONFIG_INHERIT_LOCK= CONFIG_INHERIT_LOCK_HELD=0 +SPAWN_LEASE_RETURN_PENDING=0 +SPAWN_LEASE_WT= +SPAWN_PRIOR_RECORD_WT= +HERDR_RECLAIMED_PANE=0 +HERDR_RECOVERY_PREFLIGHTED=0 +HERDR_RECOVERY_ENDPOINT_STATE= +HERDR_RECOVERY_TARGET= spawn_fresh_commit_rollback() { if fm_backlog_atomic_transition rollback "$STATE/$ID.meta" \ @@ -838,6 +879,12 @@ spawn_abort_cleanup() { fi fi fi + if [ "$SPAWN_LEASE_RETURN_PENDING" = 1 ]; then + SPAWN_LEASE_RETURN_PENDING=0 + spawn_lease_return_on_abort "$SPAWN_LEASE_WT" || true + elif [ "$status" -ne 0 ] && [ -n "$SPAWN_LEASE_WT" ]; then + echo "warning: leased copy '$SPAWN_LEASE_WT' is retained because an endpoint may already sit inside it; inspect it, then release it with: (cd '$PROJ_ABS' && treehouse return --force '$SPAWN_LEASE_WT')" >&2 + fi if [ "$SPAWN_TASK_LOCK_HELD" = 1 ]; then SPAWN_TASK_LOCK_HELD=0 fm_lock_release "$SPAWN_TASK_LOCK" || true @@ -1892,8 +1939,20 @@ EOF printf '%s' "$lines" >&2 } -freshen_spawn_worktree_base() { # - local worktree=$1 default target expected actual status +spawn_refuse_unpushed_head_after_fetch() { # + local worktree=$1 unpushed + unpushed=$(git -C "$worktree" log --format=%H --max-count=1 HEAD --not --remotes -- 2>/dev/null) || { + echo "error: could not inspect prior copy '$worktree' for unpushed commits; refusing to refresh it" >&2 + return 1 + } + if [ -n "$unpushed" ]; then + echo "error: prior copy '$worktree' for $ID holds commits not on any remote; refusing to refresh over that work - land or discard it first" >&2 + return 1 + fi +} + +freshen_spawn_worktree_base() { # [protect-unpushed-head] + local worktree=$1 protect_unpushed_head=${2:-0} default target expected actual status if ! git -C "$worktree" fetch --quiet origin; then echo "error: could not fetch origin for pooled worktree '$worktree'; refusing to launch from a potentially stale base" >&2 return 1 @@ -1927,6 +1986,9 @@ freshen_spawn_worktree_base() { # fi return 1 fi + if [ "$protect_unpushed_head" = 1 ]; then + spawn_refuse_unpushed_head_after_fetch "$worktree" || return 1 + fi if ! git -C "$worktree" reset --hard "$target" >/dev/null; then echo "error: could not reset pooled worktree '$worktree' to '$target'; refusing to launch from a potentially stale base" >&2 return 1 @@ -1938,6 +2000,129 @@ freshen_spawn_worktree_base() { # fi } +# Lease a pooled copy of the project from THIS process, with no pane and no +# shell anywhere near it. `treehouse get --lease` prints exactly the absolute +# path on stdout; its banners go to stderr and are shown only on failure. +spawn_lease_pool_worktree() { # -> prints the leased worktree path + local out err_file lines + err_file=$(mktemp "${TMPDIR:-/tmp}/fm-spawn-lease.XXXXXX") || return 1 + if ! out=$(cd "$PROJ_ABS" && treehouse get --lease --lease-holder "$W" 2>"$err_file"); then + echo "error: treehouse get --lease failed for project '$PROJ_ABS'; no worker was started" >&2 + cat "$err_file" >&2 2>/dev/null || true + rm -f "$err_file" + return 1 + fi + rm -f "$err_file" + lines=$(printf '%s\n' "$out" | grep -c . || true) + case "$out" in + /*) ;; + *) lines=0 ;; + esac + if [ "$lines" != 1 ] || [ ! -d "$out" ]; then + echo "error: treehouse get --lease did not report exactly one existing absolute worktree path for project '$PROJ_ABS' (got '${out:-nothing}'); no worker was started" >&2 + return 1 + fi + printf '%s\n' "$out" +} + +# A fresh spawn that stops after leasing but before any endpoint exists hands +# a CLEAN copy straight back so a refused launch cannot shrink the pool. A copy +# holding uncommitted work is never reset here: it stays leased (prune cannot +# recycle it) and its path is reported. The primary checkout is never returned. +spawn_lease_return_on_abort() { # + local worktree=$1 wt_real status + [ -n "$worktree" ] || return 0 + wt_real=$(real_path_or_raw "$worktree") + if [ "$wt_real" = "$PROJ_ABS_REAL" ]; then + echo "warning: leased path '$worktree' resolves to the primary checkout; not returning it" >&2 + return 1 + fi + status=$(git -C "$worktree" -c core.quotePath=false status --porcelain 2>/dev/null) || { + echo "warning: could not inspect leased copy '$worktree'; leaving it leased - release it with: (cd '$PROJ_ABS' && treehouse return '$worktree')" >&2 + return 1 + } + if [ -n "$status" ]; then + echo "warning: leased copy '$worktree' holds uncommitted work; leaving it leased and untouched - inspect it, then release it with: (cd '$PROJ_ABS' && treehouse return '$worktree')" >&2 + return 1 + fi + if ! (cd "$PROJ_ABS" && treehouse return --force "$worktree") >/dev/null 2>&1; then + echo "warning: could not return leased copy '$worktree' to the pool; release it with: (cd '$PROJ_ABS' && treehouse return --force '$worktree')" >&2 + return 1 + fi +} + +# A fresh Herdr recovery reuses a durable record's copy only when the record +# proves the same task, project, kind, backend, endpoint, and projection +# journal. An absent record means ordinary leasing; any existing record that +# cannot establish that identity refuses rather than leasing a second copy or +# overwriting foreign metadata. +spawn_prior_record_worktree() { + local meta="$STATE/$ID.meta" journal="$STATE/$ID.herdr-presentation" + local prior prior_project prior_project_real prior_kind kind_count + SPAWN_PRIOR_RECORD_WT= + if [ ! -e "$meta" ] && [ ! -L "$meta" ]; then + return 1 + fi + fm_backlog_record_present "$meta" "task record" "$STATE" || { + echo "error: existing task record for $ID is unsafe: $FM_BACKLOG_TRANSITION_ERROR" >&2 + return 2 + } + fm_backend_validate_task_endpoint "$meta" "$ID" || return 2 + prior=$(herdr_projection_meta_field_exact "$meta" worktree 2>/dev/null) || { + echo "error: existing task record for $ID has no exact worktree; refusing recovery" >&2 + return 2 + } + [ -d "$prior" ] && [ "$(real_path_or_raw "$prior")" != "$PROJ_ABS_REAL" ] || { + echo "error: existing task record for $ID names an unusable isolated copy '$prior'; refusing recovery" >&2 + return 2 + } + prior_project=$(herdr_projection_meta_field_exact "$meta" project 2>/dev/null) || { + echo "error: existing task record for $ID has no exact project; refusing recovery" >&2 + return 2 + } + prior_project_real=$(real_path_or_raw "$prior_project") + if [ "$prior_project_real" != "$PROJ_ABS_REAL" ]; then + echo "error: existing task record for $ID belongs to project '$prior_project', not '$PROJ_ABS'; refusing to reuse or replace its copy" >&2 + return 2 + fi + kind_count=$(grep -c '^kind=' "$meta" 2>/dev/null || true) + case "$kind_count" in + 0) prior_kind=ship ;; + 1) prior_kind=$(herdr_projection_meta_field_exact "$meta" kind 2>/dev/null) || prior_kind= ;; + *) prior_kind= ;; + esac + if [ -z "$prior_kind" ] || [ "$prior_kind" != "$KIND" ]; then + echo "error: existing task record for $ID has kind '${prior_kind:-ambiguous}', not '$KIND'; refusing recovery" >&2 + return 2 + fi + if [ "$BACKEND" != herdr ] || [ "$FM_BACKEND_VALIDATED_BACKEND" != herdr ] \ + || [ ! -f "$journal" ] || [ -L "$journal" ]; then + echo "error: existing task record for $ID is not an exact Herdr presentation recovery; refusing to reuse or replace its copy" >&2 + return 2 + fi + SPAWN_PRIOR_RECORD_WT=$prior +} + +# A reused copy is refreshed under the same clean check as a leased one, and +# additionally refuses when its HEAD carries commits that no remote holds: +# the refresh would reset those away, and they may be the prior worker's +# unlanded work. +spawn_refuse_unpushed_head() { # + local worktree=$1 + if ! git -C "$worktree" fetch --quiet --prune origin; then + echo "error: could not fetch and prune origin for prior copy '$worktree'; refusing to inspect or refresh it" >&2 + return 1 + fi + spawn_refuse_unpushed_head_after_fetch "$worktree" +} + +# The first backend call that opens anything with the leased path as its cwd +# ends the return-on-abort window (header): from here a refusal retains the +# lease rather than terminating whatever that call left inside the copy. +spawn_lease_endpoint_begins() { + SPAWN_LEASE_RETURN_PENDING=0 +} + herdr_projection_meta_field_exact() { # local meta=$1 key=$2 count [ -f "$meta" ] && [ ! -L "$meta" ] || return 1 @@ -1956,6 +2141,8 @@ herdr_projection_existing_meta_allows_flat() { # HERDR_RECOVERY_WORKSPACE_ID="" HERDR_RECOVERY_TAB_ID="" HERDR_RECOVERY_PANE_ID="" + HERDR_RECOVERY_ENDPOINT_STATE="" + HERDR_RECOVERY_TARGET="" old_backend=$(fm_backend_of_meta "$meta") old_target=$(fm_backend_target_of_meta "$meta") [ -n "$old_target" ] || { @@ -1996,6 +2183,8 @@ herdr_projection_existing_meta_allows_flat() { # return 1 } old_state=$(fm_backend_herdr_pane_agent_state "$old_session" "$old_pane") + HERDR_RECOVERY_ENDPOINT_STATE=$old_state + HERDR_RECOVERY_TARGET=$old_target case "$old_state" in dead|no-agent) return 0 ;; live|unknown) @@ -2014,6 +2203,45 @@ herdr_projection_existing_meta_allows_flat() { # esac } +herdr_projection_recovery_preflight_before_refresh() { + local seen settled=0 + HERDR_LABEL_HOME=$FM_HOME + HERDR_LAUNCHER_RELATIONSHIP=launcher-home + HERDR_PRESENTATION_JOURNAL=$(fm_backend_herdr_projection_journal_path "$STATE" "$ID") + HERDR_SES=$(fm_backend_herdr_session) + HERDR_PARENT_LABEL=$(FM_HOME="$HERDR_LABEL_HOME" fm_backend_herdr_workspace_label) + fm_backend_herdr_server_ensure "$HERDR_SES" || { + echo "error: herdr presentation recovery could not ensure its exact named session" >&2 + return 1 + } + spawn_herdr_presentation_order_lock_acquire "$HERDR_SES" || { + echo "error: herdr presentation recovery could not acquire its session lock; refusing a concurrent resume" >&2 + return 1 + } + herdr_projection_existing_meta_allows_flat "$STATE/$ID.meta" || return 1 + fm_backend_herdr_projection_recovery_allows_flat \ + "$HERDR_SES" "$HERDR_PRESENTATION_JOURNAL" "$ID" || return 1 + HERDR_RECOVERY_PREFLIGHTED=1 + [ "$HERDR_RECOVERY_ENDPOINT_STATE" = no-agent ] || return 0 + fm_backend_herdr_send_text_line "$HERDR_RECOVERY_TARGET" \ + "cd $(shell_quote "$PROJ_ABS")" || { + echo "error: existing Herdr shell for $ID could not be moved out of its recorded copy before refresh" >&2 + return 1 + } + for _ in $(seq 1 "${FM_SPAWN_SETTLE_POLLS:-60}"); do + seen=$(fm_backend_herdr_current_path "$HERDR_RECOVERY_TARGET" || true) + if [ -n "$seen" ] && [ "$(real_path_or_raw "$seen")" = "$PROJ_ABS_REAL" ]; then + settled=1 + break + fi + sleep 1 + done + if [ "$settled" -ne 1 ]; then + echo "error: existing Herdr shell for $ID did not leave its recorded copy before refresh (last read '${seen:-none}')" >&2 + return 1 + fi +} + # Backlog preflight (bin/fm-backlog-transition-lib.sh). This spawn is about to # become the sole owner of the row's In-flight transition, so prove the row is # transitionable BEFORE any endpoint, worktree, or record exists: a refusal here @@ -2067,17 +2295,46 @@ if [ "$RELAUNCH" -eq 1 ]; then WT_TARGET=$T SES=${T%%:*} else +# Lease and refresh the task copy BEFORE any endpoint exists (header: the pane +# is created already inside it; nothing else may enter the copy first). Orca +# owns its own worktree and a secondmate launches in its home, so both keep +# the project or home path as the pane's starting directory. +PANE_CWD=$PROJ_ABS +FRESHEN_PROTECT_UNPUSHED_HEAD=0 +if [ "$KIND" != secondmate ] && [ "$BACKEND" != orca ]; then + if spawn_prior_record_worktree; then + WT=$SPAWN_PRIOR_RECORD_WT + FRESHEN_PROTECT_UNPUSHED_HEAD=1 + # The record's own copy: never returned by this spawn, refreshed in place. + herdr_projection_recovery_preflight_before_refresh || exit 1 + validate_spawn_worktree "prior record" "(no endpoint created yet)" + spawn_refuse_unpushed_head "$WT" || exit 1 + else + prior_status=$? + if [ "$prior_status" -eq 1 ]; then + WT=$(spawn_lease_pool_worktree) || exit 1 + SPAWN_LEASE_WT=$WT + SPAWN_LEASE_RETURN_PENDING=1 + validate_spawn_worktree "treehouse get --lease" "(no endpoint created yet)" + else + exit 1 + fi + fi + freshen_spawn_worktree_base "$WT" "$FRESHEN_PROTECT_UNPUSHED_HEAD" || exit 1 + PANE_CWD=$WT +fi case "$BACKEND" in tmux) SES=$(fm_backend_tmux_container_ensure) T="$SES:$W" # #134 robustness (tmux): fm_backend_tmux_create_task captures a stable window # id and pins the window name (automatic-rename/allow-rename off) so a captain's - # non-default tmux config cannot rename the window away from fm- once - # treehouse cd's into the worktree. WT_TARGET carries that stable id for the - # rename-critical worktree-detection steps below; the persisted window= handle + # non-default tmux config cannot rename the window away from fm- as soon + # as its shell starts in the worktree. WT_TARGET carries that stable id for the + # rename-critical cwd-confirmation steps below; the persisted window= handle # stays $T (the name form), which is safe now that rename is disabled. - WID=$(fm_backend_tmux_create_task "$SES" "$W" "$PROJ_ABS") || exit 1 + spawn_lease_endpoint_begins + WID=$(fm_backend_tmux_create_task "$SES" "$W" "$PANE_CWD") || exit 1 WT_TARGET="$WID" ;; herdr) @@ -2111,19 +2368,21 @@ case "$BACKEND" in HERDR_SES=$(fm_backend_herdr_session) HERDR_PARENT_LABEL=$(FM_HOME="$HERDR_LABEL_HOME" fm_backend_herdr_workspace_label) if [ -e "$HERDR_PRESENTATION_JOURNAL" ] || [ -L "$HERDR_PRESENTATION_JOURNAL" ]; then - fm_backend_herdr_server_ensure "$HERDR_SES" || { - echo "error: herdr presentation recovery could not ensure its exact named session" >&2 - exit 1 - } - spawn_herdr_presentation_order_lock_acquire "$HERDR_SES" || { - echo "error: herdr presentation recovery could not acquire its session lock; refusing a concurrent resume" >&2 - exit 1 - } - if [ -e "$STATE/$ID.meta" ] || [ -L "$STATE/$ID.meta" ]; then - herdr_projection_existing_meta_allows_flat "$STATE/$ID.meta" || exit 1 + if [ "$HERDR_RECOVERY_PREFLIGHTED" != 1 ]; then + fm_backend_herdr_server_ensure "$HERDR_SES" || { + echo "error: herdr presentation recovery could not ensure its exact named session" >&2 + exit 1 + } + spawn_herdr_presentation_order_lock_acquire "$HERDR_SES" || { + echo "error: herdr presentation recovery could not acquire its session lock; refusing a concurrent resume" >&2 + exit 1 + } + if [ -e "$STATE/$ID.meta" ] || [ -L "$STATE/$ID.meta" ]; then + herdr_projection_existing_meta_allows_flat "$STATE/$ID.meta" || exit 1 + fi + fm_backend_herdr_projection_recovery_allows_flat \ + "$HERDR_SES" "$HERDR_PRESENTATION_JOURNAL" "$ID" || exit 1 fi - fm_backend_herdr_projection_recovery_allows_flat \ - "$HERDR_SES" "$HERDR_PRESENTATION_JOURNAL" "$ID" || exit 1 if [ "${HERDR_RECOVERY_BACKEND:-}" = herdr ]; then set +e FM_HOME="$HERDR_LABEL_HOME" fm_backend_herdr_projection_reclaim_task \ @@ -2134,6 +2393,10 @@ case "$BACKEND" in set -e case "$HERDR_RECLAIM_STATUS" in 0) + # The reclaimed pane already existed, so it was not created + # inside the copy; the settle step below sends it one plain + # `cd` there after the refresh (header). + HERDR_RECLAIMED_PANE=1 HERDR_PROJECTED=1 HERDR_WORKSPACE_ID=$HERDR_RECOVERY_WORKSPACE_ID HERDR_SEEDED_DEFAULT_TAB_ID="" @@ -2182,8 +2445,9 @@ case "$BACKEND" in else HERDR_PROJECTION_ID=$(fm_backend_herdr_projection_journal_create "$STATE" "$ID") || exit 1 HERDR_PROJECTION_LABEL=$(fm_backend_herdr_projection_workspace_label "$ID" "$HERDR_PROJECTION_ID") + spawn_lease_endpoint_begins if ! FM_HOME="$HERDR_LABEL_HOME" fm_backend_herdr_projection_create_task \ - "$PROJ_ABS" "$HERDR_PROJECTION_LABEL" "$W"; then + "$PANE_CWD" "$HERDR_PROJECTION_LABEL" "$W"; then if [ "${FM_BACKEND_HERDR_PROJECTION_CLEANUP_SAFE:-0}" = 1 ]; then HERDR_PROJECTION_ABORT_CLEANUP=1 HERDR_PROJECTION_ABORT_SESSION=$FM_BACKEND_HERDR_PROJECTION_SESSION @@ -2236,7 +2500,8 @@ case "$BACKEND" in HERDR_SEEDED_DEFAULT_TAB_ID=${HERDR_CONTAINER_RAW#*$'\t'} HERDR_SES=${CONTAINER%%:*} HERDR_WORKSPACE_ID=${CONTAINER#*:} - HERDR_TASK_IDS=$(FM_HOME="$HERDR_LABEL_HOME" fm_backend_herdr_create_task "$CONTAINER" "$W" "$PROJ_ABS" "$HERDR_SEEDED_DEFAULT_TAB_ID") || exit 1 + spawn_lease_endpoint_begins + HERDR_TASK_IDS=$(FM_HOME="$HERDR_LABEL_HOME" fm_backend_herdr_create_task "$CONTAINER" "$W" "$PANE_CWD" "$HERDR_SEEDED_DEFAULT_TAB_ID") || exit 1 read -r HERDR_TAB_ID HERDR_PANE_ID <&2 fi -# #134 robustness: only tmux needs a worktree-detection target distinct from $T - +# #134 robustness: only tmux needs a cwd-confirmation target distinct from $T - # its rename-safe stable window id, set as WT_TARGET=$WID in the tmux branch above. # Every other backend addresses its pane/surface by the id already in $T, so default -# WT_TARGET to $T for them (and for any future backend) - the shared treehouse-get + -# worktree-detection steps below must never reference an unbound WT_TARGET under set -u. +# WT_TARGET to $T for them (and for any future backend) - the shared worktree +# cwd-confirmation steps below must never reference an unbound WT_TARGET under set -u. : "${WT_TARGET:=$T}" spawn_send_text_line() { # case "$BACKEND" in @@ -2421,55 +2692,47 @@ if [ "$RELAUNCH" -eq 1 ]; then fi [ "$KIND" = secondmate ] || validate_spawn_worktree "relaunch" "$T" elif [ "$KIND" != secondmate ] && [ "$BACKEND" != orca ]; then - spawn_send_text_line "$WT_TARGET" 'treehouse get' - - # Wait for the treehouse subshell: the pane's cwd moves from the project to the worktree. + # The copy was leased and refreshed before this pane existed, and the pane + # was created with that path as its starting directory. What remains is to + # prove the live shell actually sits there before anything is launched. # Target the stable window id, not the name: if the name is ever lost (e.g. an # automatic-rename slips through), display-message -t falls back to the # active client's window, which would misread firstmate's OWN pane path as the # worktree and tangle a hook into the primary checkout. The window id never lies. - # Compare against PROJ_ABS_REAL (physical), not PROJ_ABS: a symlinked project - # prefix would otherwise make the pane's OS-level cwd read differ from - # PROJ_ABS on the very first poll, before the pane has actually moved. + # Compare physically resolved paths: a symlinked prefix would otherwise make + # the pane's OS-level cwd read differ from the leased path string forever. # - # A single read that already differs from PROJ_ABS_REAL is not proof the pane - # settled there: on some tmux/WSL setups a brand-new window's pane_current_path - # transiently reports an unrelated stale path (seen live as another real git - # checkout entirely) before the shell catches up with treehouse get's cd. That - # stale path still passes the PROJ_ABS_REAL comparison and validate_spawn_worktree - # below (it resolves to a real, distinct worktree top-level too), so accepting it - # on one read alone silently records the wrong worktree= in state/.meta. Require - # two consecutive reads to agree on the same non-project path before accepting it; - # a mismatch just becomes the new candidate rather than resetting the wait, so a - # pane that is already settled by the first real read only costs the one existing - # inter-poll sleep as confirmation, not a whole extra cycle on top. - candidate="" - for _ in $(seq 1 60); do + # A brand-new pane can transiently report an unrelated stale path (seen live + # on some tmux/WSL setups as another real git checkout entirely) before its + # shell settles. Only a read equal to the leased path is accepted, so a stale + # read costs one more poll rather than ever being recorded as the worktree. + # A reclaimed Herdr pane predates the lease, so it is moved into the copy + # with one plain `cd` now that the refresh is already complete; the copy is + # still never entered before its refresh. + if [ "$HERDR_RECLAIMED_PANE" = 1 ]; then + spawn_send_text_line "$WT_TARGET" "cd $(shell_quote "$WT")" + fi + WT_REAL=$(real_path_or_raw "$WT") + settled=0 + p= + for _ in $(seq 1 "${FM_SPAWN_SETTLE_POLLS:-60}"); do p=$(spawn_current_path "$WT_TARGET" || true) - if [ -n "$p" ]; then - p_real=$(real_path_or_raw "$p") - if [ "$p_real" != "$PROJ_ABS_REAL" ]; then - if [ -n "$candidate" ] && [ "$p_real" = "$candidate" ]; then - WT="$p" - break - fi - candidate="$p_real" - else - candidate="" - fi - else - candidate="" + if [ -n "$p" ] && [ "$(real_path_or_raw "$p")" = "$WT_REAL" ]; then + settled=1 + break fi sleep 1 done - if [ -z "$WT" ]; then - echo "error: treehouse get did not enter a worktree within 60s; inspect window $T" >&2 + if [ "$settled" -ne 1 ]; then + echo "error: endpoint $T did not settle in the leased worktree '$WT' (last read '${p:-none}'); refusing to launch outside the refreshed copy. Inspect window $T; the lease is retained - release it with: (cd '$PROJ_ABS' && treehouse return --force '$WT')" >&2 exit 1 fi - validate_spawn_worktree "treehouse get" "$T" + validate_spawn_worktree "leased worktree" "$T" fi -if [ "$RELAUNCH" -eq 0 ] && [ "$KIND" != secondmate ]; then +if [ "$RELAUNCH" -eq 0 ] && [ "$KIND" != secondmate ] && [ "$BACKEND" = orca ]; then + # Orca creates its own worktree and terminal; its refresh keeps its existing + # place after terminal creation. freshen_spawn_worktree_base "$WT" || exit 1 fi diff --git a/bin/fm-teardown.sh b/bin/fm-teardown.sh index ad9e042ba11..7d6d520c9d5 100755 --- a/bin/fm-teardown.sh +++ b/bin/fm-teardown.sh @@ -108,10 +108,10 @@ # checks before any destructive return. Teardown output notes every wait, retry, and # removal so the operator can see what happened. # -# Pre-teardown cleanup sequence (runs once every landed/discard-work safety -# refusal above has already passed, and BEFORE any worktree return, branch -# delete, or backend kill below - a still-active run or a leaked process may -# own live work in that worktree): +# Pre-teardown cleanup sequence runs once every landed/discard-work safety +# refusal above has already passed. The task's parked run is concluded before +# its endpoint closes; leaked processes are reaped after that close but before +# any worktree return or branch delete. # Fix 1 - conclude the task's own no-mistakes run. A ship task's worktree can # be torn down while its no-mistakes pipeline run is still PARKED at a gate # (awaiting_approval/fix_review/any awaiting_agent field), with no worker @@ -142,6 +142,9 @@ # roots are unique per task and never # shared, so this can never reach another task's or the primary's # processes. Idempotent: nothing left to find is a silent no-op. +# When lsof is unavailable, teardown captures a verified tmux pane process +# group before closing the endpoint, then reaps that captured group after +# the close removes the pane lookup target. # Fix 3 - sweep abandoned remote job workers. A remote job worker started # from a worktree's own bin/ outlives that worktree's removal without # being reachable by Fix 2, because its working directory is wherever it @@ -1598,17 +1601,22 @@ $out EOF } -task_process_identity() { # - local pid=$1 proc_root stat_line starttime value +task_process_birth_identity() { # + local pid=$1 proc_root stat_line starttime local -a stat_fields proc_root=${FM_PROC_ROOT_OVERRIDE:-/proc} - if [ -r "$proc_root/$pid/stat" ]; then - stat_line=$(cat "$proc_root/$pid/stat" 2>/dev/null) || return 1 - read -r -a stat_fields <<< "${stat_line##*)}" - [ "${#stat_fields[@]}" -ge 20 ] || return 1 - starttime=${stat_fields[19]} - case "$starttime" in ''|*[!0-9]*) return 1 ;; esac - printf 'starttime=%s\n' "$starttime" + [ -r "$proc_root/$pid/stat" ] || return 1 + stat_line=$(cat "$proc_root/$pid/stat" 2>/dev/null) || return 1 + read -r -a stat_fields <<< "${stat_line##*)}" + [ "${#stat_fields[@]}" -ge 20 ] || return 1 + starttime=${stat_fields[19]} + case "$starttime" in ''|*[!0-9]*) return 1 ;; esac + printf 'starttime=%s\n' "$starttime" +} + +task_process_identity() { # + local pid=$1 value + if task_process_birth_identity "$pid"; then return 0 fi value=$(LC_ALL=C ps -p "$pid" -o lstart= 2>/dev/null) || return 1 @@ -1624,10 +1632,52 @@ task_process_identity_matches() { # [ "$current" = "$2" ] } +task_process_birth_identity_matches() { # + local current + current=$(task_process_birth_identity "$1") || return 1 + [ "$current" = "$2" ] +} + task_pid_list_contains() { # printf '%s\n' "$1" | grep -Fxq "$2" } +task_process_group_pids() { # + local pgid=$1 rows pid current_pgid extra + rows=$(LC_ALL=C ps -axo pid=,pgid= 2>/dev/null) || return 1 + while read -r pid current_pgid extra; do + [ -z "$extra" ] || return 1 + case "$pid:$current_pgid" in + *[!0-9:]*) return 1 ;; + esac + [ "$current_pgid" != "$pgid" ] || printf '%s\n' "$pid" + done < + local current_pgid + current_pgid=$(ps -o pgid= -p "$1" 2>/dev/null) || return 1 + current_pgid=$(printf '%s' "$current_pgid" | tr -d '[:space:]') + [ "$current_pgid" = "$2" ] +} + +task_backend_process_group_has_captured_member() { + local pgid=$TASK_BACKEND_PGID i pid identity + i=0 + while [ "$i" -lt "${#TASK_BACKEND_MEMBER_PIDS[@]}" ]; do + pid=${TASK_BACKEND_MEMBER_PIDS[$i]} + identity=${TASK_BACKEND_MEMBER_IDENTITIES[$i]} + if task_process_birth_identity_matches "$pid" "$identity" \ + && task_process_group_matches "$pid" "$pgid"; then + return 0 + fi + i=$((i + 1)) + done + return 1 +} + task_pids_under_roots() { # ... TASK_PIDS= TASK_PIDS_FAILED_DIR= @@ -1644,8 +1694,12 @@ $dir_pids" TASK_PIDS=$(printf '%s\n' "$pids" | grep -E '^[0-9]+$' | sort -un || true) } -reap_task_backend_process_group() { #