diff --git a/.agents/skills/harness-adapters/references/harness/claude.md b/.agents/skills/harness-adapters/references/harness/claude.md index 44324e467a8..9c2e8d18458 100644 --- a/.agents/skills/harness-adapters/references/harness/claude.md +++ b/.agents/skills/harness-adapters/references/harness/claude.md @@ -13,8 +13,10 @@ Busy hooks verified 2026-07-28 on Claude Code 2.1.220. | Model | `--model `; discover through the interactive `/model` picker, with alias or full-name shape documented by `claude --help`. | | Effort | `--effort `, verified on 2.1.196. | -Fresh-worktree or first-machine launch may show trust or bypass-permissions confirmation. -Inspect within about 20 seconds, accept the required choice with `FM_HOME= ../../../bin/fm-send.sh --key Enter` unless already bound, and verify instructions started. +Workspace trust confirms on every worktree path Claude has not seen, which is every task, and the launch flags do not change that: it measured identically under `--dangerously-skip-permissions` and under today's `--permission-mode auto`. +`../../../docs/verification/runtime-backends.md` owns that measurement under "Crewmate autonomy and the status-file write contract". +Inspect within about 20 seconds; the preselected row is `No, exit`, so a bare Enter DECLINES and quits Claude, while the accept `Yes, I trust this folder` sits on the row below. +Accept with `FM_HOME= ../../../bin/fm-send.sh --key Down` then `--key Enter` unless already bound, and verify instructions started. ## Composer ghost diff --git a/.agents/skills/harness-adapters/references/harness/codex.md b/.agents/skills/harness-adapters/references/harness/codex.md index 5fb95b8e494..6213fe685c3 100644 --- a/.agents/skills/harness-adapters/references/harness/codex.md +++ b/.agents/skills/harness-adapters/references/harness/codex.md @@ -10,6 +10,8 @@ Verified on 2026-06-11 with codex-cli 0.139.0 unless a fact gives a newer versio | Exit command | `/quit`; its slash popup needs about one second between text and Enter, which the shared submit path used by the control plane handles. | | Interrupt | Single Escape. | | Skill invocation | `$`, for example `$no-mistakes`; `/` is Claude-only and Codex rejects it as "Unrecognized command". | +| Autonomy | `-s workspace-write -a never`: sandboxed to the worktree, with no approval path to escalate out of it; `../../../bin/fm-spawn.sh` owns the exact flags. | +| Sandboxed writes | `workspace-write` normally refuses writes outside the worktree, and `-a never` leaves the model no way to ask, so the paths the brief permits outside it are granted with `--add-dir`; `../../../docs/verification/runtime-backends.md` owns the measurement under "Crewmate autonomy and the status-file write contract". `workspace-write` grants `/tmp` and `$TMPDIR` by default, so a write measured there proves nothing. | | Resume | `codex resume `, using the id printed on quit. | | Model flag | `--model `. | | Effort flag | `-c 'model_reasoning_effort=""'`, verified on codex-cli 0.142.1 whose installed schema contains `model_reasoning_effort`, active config uses it, and bundled catalog advertises only these four values while omitting `max`. | @@ -18,6 +20,7 @@ Verified on 2026-06-11 with codex-cli 0.139.0 unless a fact gives a newer versio A directory trust dialog appears on the first run for a repository root: "Do you trust the contents of this directory?" Accept it with Enter and verify the instructions begin processing. The decision persists for the repository, so later worktrees of the same project skip it. +A second gate, `Hooks need review`, was observed standing between that dialog and the composer whenever Codex hooks are configured, so a Codex spawn can need two answers before it reads its brief; `../../../docs/verification/runtime-backends.md` owns that measurement under "Crewmate autonomy and the status-file write contract". ## Skill popup diff --git a/.agents/skills/harness-adapters/references/harness/cursor.md b/.agents/skills/harness-adapters/references/harness/cursor.md index 3048a0a8347..6c4f37590e6 100644 --- a/.agents/skills/harness-adapters/references/harness/cursor.md +++ b/.agents/skills/harness-adapters/references/harness/cursor.md @@ -8,14 +8,15 @@ Cross-harness provider and credential identity is owned by `references/common/mo | Fact | Value | |---|---| | Binary | `fm_cursor_resolve_binary` in `../../../bin/fm-cursor-lib.sh` resolves stable launcher `cursor-agent` or legacy `agent`, never `cursor`; both symlink into `~/.local/share/cursor-agent/versions//cursor-agent`, whose target auto-update replaces. | -| Launch | Positional instructions with `--trust`, `--yolo`, optional `--model `, and `--workspace `, after clearing foreign primary markers. | +| Launch | Positional instructions with `--trust`, the autonomy flags `../../../bin/fm-spawn.sh` owns, optional `--model `, and `--workspace `, after clearing foreign primary markers. | | Models | Use current-account `cursor-agent --list-models` or legacy `agent --list-models`; the drifting observed list had only `cursor-grok-4.5-high` and `cursor-grok-4.5-high-fast` for Grok plus several `xhigh` ids, so choose a returned reasoning id and never assume low or medium Grok. | | Busy state | `../../../bin/fm-busy-lib.sh` folds the per-conversation transcript as `cursor-transcript`: `role:user` opens and typed `turn_ended` closes success or abort, covering manual interrupt; nothing is armed or seeded, and this backend-agnostic source was identical on tmux and Herdr. | | Exit command | `/exit`. | | Interrupt | Single Escape returns the placeholder with no clear key; control makes no cancellation claim because an aborted transcript close appeared within seconds in some runs and not within twenty in others. | | Skill invocation | `/`, for example `/no-mistakes`; Cursor discovers Firstmate's user skills. | | Resume | No verified native pane resume; use deterministic relaunch. | -| Autonomy | `--yolo`, documented alias for `--force`; footer `Run Everything`. | +| Autonomy | `--auto-review --sandbox enabled`, so a crewmate runs under Cursor's own review and sandbox controls, plus the `--add-dir` writable roots below; `../../../bin/fm-spawn.sh` owns the exact flags. `--yolo`, the documented alias for `--force` whose footer reads `Run Everything`, switches both off and is not used. | +| Sandboxed writes | The sandbox normally refuses writes outside the workspace, and `--auto-review` then parks the turn on a manual approval prompt an unattended crewmate cannot clear, so the paths the brief permits outside the worktree are granted with `--add-dir`; `../../../docs/verification/runtime-backends.md` owns the measurement under "Crewmate autonomy and the status-file write contract". | | Trust | `--trust` suppresses the dialog; `--yolo` does not, and every task has a fresh path. | | Marker | `CURSOR_INVOKED_AS=cursor-agent` on agent and children, plus `CURSOR_AGENT=1` on child or tool processes; other `CURSOR_*` variables are not identity markers. | | Effort | No verified flag; `references/common/model-and-effort.md` owns unsupported-value handling. | @@ -61,7 +62,9 @@ Refresh with `FM_HARNESS_LIVENESS_DRIFT=1 ../../../bin/fm-test-run.sh ../../../t Firstmate enters its acquired worktree and passes the same absolute path through `--workspace`. Never pass Cursor `-w` or `--worktree`, which allocates a second copy under `~/.cursor/worktrees` and breaks isolation. -The CLI supports repeatable `--add-dir`, but the adapter adds none; positional instructions need no grant to their private directory. +The adapter also passes repeatable `--add-dir ` for the writable roots the sandbox posture needs; `../../../bin/fm-spawn.sh` owns which ones, and they are the only paths a worker may write outside its worktree. +The brief rides inline as the positional prompt, so the private brief directory needs no grant. +`--add-dir` adds writable roots only: Cursor still records the `--workspace` path as its project `workspacePath`, the exact identity `state/.cursor-session` binds the busy fold to. Example: `../../../bin/fm-spawn.sh --scout --harness cursor --model cursor-grok-4.5-high`. ## Primary integration diff --git a/bin/backends/herdr.sh b/bin/backends/herdr.sh index 8728b356cc0..b64f335bf14 100644 --- a/bin/backends/herdr.sh +++ b/bin/backends/herdr.sh @@ -1203,8 +1203,13 @@ fm_backend_herdr_pane_idle_shell_pid() { # } # fm_backend_herdr_pane_idle_shell_sample: one strict instantaneous -# observation for fm_backend_herdr_pane_idle_shell_pid, which owns the proof -# contract and the settle retry. +# observation of the idle-shell shape. +# fm_backend_herdr_pane_idle_shell_pid wraps it with the settle retry and owns +# the proof contract for the destructive close paths. +# fm_backend_herdr_pane_agent_state instead takes a single sample directly, on +# purpose: that read runs in poll loops, the retry budget would slow every +# healthy `live` answer, and its corroboration is positive-only, so a sample +# lost to a transient prompt helper simply keeps `live` until the next poll. fm_backend_herdr_pane_idle_shell_sample() { # local session=$1 pane=$2 info shell_pid foreground_pgid count local process_pid name argv0 shell_name rows stat ps_bin @@ -1874,25 +1879,55 @@ fm_backend_herdr_explicit_close_pane_confirmed() { # # reaped it - verified empirically: killing a pane's shell pid # on a live server makes herdr immediately drop both the pane # and its tab from `pane get`/`tab list`). -# no-agent - `pane get` succeeds (the pane structurally exists) but `agent -# get` responds with error code agent_not_found: nothing is -# registered in it - exactly what a herdr session-layout restore -# produces (verified empirically: `session stop` + fresh `herdr -# server` restart leaves the pane alive, agent_status "unknown", -# agent get -> agent_not_found - docs/herdr-backend.md "ID -# stability across a server restart"), and what a future -# `resume_agents_on_restore = false` restore would produce too -# (a plain shell, never an agent). -# live - `agent get` succeeds and reports a real agent_status (working, -# idle, done, or blocked - any registered value). An idle or -# blocked agent is still a genuine, still-registered agent, not -# a restored husk, so it is never a close-and-replace candidate. +# no-agent - the pane structurally exists but confidently hosts no agent. +# Two independent positive grounds reach this verdict: +# (a) `agent get` responds with error code agent_not_found: +# nothing is registered in it - exactly what a herdr +# session-layout restore produces (verified empirically: +# `session stop` + fresh `herdr server` restart leaves the pane +# alive, agent_status "unknown", agent get -> agent_not_found - +# docs/herdr-backend.md "ID stability across a server restart"), +# and what a future `resume_agents_on_restore = false` restore +# would produce too (a plain shell, never an agent); +# (b) a registration IS reported, but the pane's own process +# inventory positively proves a lone bare idle shell, so the +# registration outlived the process it describes. +# live - `agent get` reports a real agent_status (working, idle, done, +# or blocked - any registered value) and the pane's process +# inventory does not contradict it. An idle or blocked agent is +# still a genuine, still-registered agent, not a restored husk, +# so it is never a close-and-replace candidate. # unknown - anything else: an unparseable/unexpected response from either # call, or a `pane get` success whose own echoed pane_id does not # round-trip (guards against misreading a herdr response shape # change as "the pane exists"). The caller must fail safe toward # refusal here, never toward closing - this is the conservative # backstop the husk check depends on. +# +# Why a reported registration is corroborated at all. Herdr's agent registry +# is written by whatever reports into it - each harness's herdr integration +# extension, or any other source - and a report is not withdrawn when the +# process that made it goes away. Verified empirically on herdr 0.8.2: a +# `pane report-agent` from a source herdr does not itself own stays registered +# on a pane running nothing but its shell, so `agent get` keeps answering +# `idle` with no agent alive anywhere. Trusting that alone made a task whose +# worker had already exited to its shell read `alive` forever: every lifecycle +# verb then typed the harness's exit command into a shell and refused to +# relaunch, with no state that could ever clear it. The tmux classifier never +# had this hole because it reads the foreground process group and lets a +# group that is nothing but shells settle the negative verdict; this is the +# same rule expressed through herdr's own `pane process-info` inventory. +# +# The corroboration is positive-only and one strict instantaneous sample, so +# it is the cheapest possible addition to a read that runs in poll loops and +# can only ever move a verdict from `live` toward `no-agent`, never the other +# way. Anything short of proof - a live harness process in the foreground, an +# extra foreground process, a shell with a child, an unreadable inventory, or +# a response about a different pane - leaves `live` exactly as before, so a +# genuinely live, ambiguous, unreadable, or contradicting endpoint is never +# reclassified as recoverable. An idle shell transiently hosting a prompt +# helper therefore keeps `live` for that sample; every caller that acts on the +# negative verdict polls, so the next sample settles it. fm_backend_herdr_pane_agent_state() { # local session=$1 pane_id=$2 out code presence status presence=$(fm_backend_herdr_pane_presence_state "$session" "$pane_id") @@ -1911,7 +1946,13 @@ fm_backend_herdr_pane_agent_state() { # fi status=$(printf '%s' "$out" | jq -r '.result.agent.agent_status // empty' 2>/dev/null) case "$status" in - working|idle|done|blocked) printf 'live' ;; + working|idle|done|blocked) + if fm_backend_herdr_pane_idle_shell_sample "$session" "$pane_id" >/dev/null 2>&1; then + printf 'no-agent' + else + printf 'live' + fi + ;; *) printf 'unknown' ;; esac } @@ -1920,7 +1961,9 @@ fm_backend_herdr_pane_agent_state() { # # states (dead, no-agent) fm_backend_herdr_pane_agent_state can positively # confirm; live and unknown both refuse (1), so an inconclusive read never # licenses closing anything. Restored-layout recovery depends on this -# fail-safe-toward-refusal behavior. +# fail-safe-toward-refusal behavior. A husk is defined by what the pane IS - +# gone, or a plain shell - so a pane whose registration outlived its agent is +# a husk on the same terms as a restored one; the classifier owns that proof. fm_backend_herdr_tab_is_husk() { # case "$(fm_backend_herdr_pane_agent_state "$1" "$2")" in dead|no-agent) return 0 ;; @@ -1931,8 +1974,10 @@ fm_backend_herdr_tab_is_husk() { # # fm_backend_herdr_agent_state: recovery-grade state for the same session-start # sweep as the tmux classifier. It reuses the husk classifier rather than # creating a second Herdr state machine: a structurally gone pane is `missing`, -# a confirmed agent-less pane is `dead`, a registered agent is `alive`, and an -# unexpected or failed API read is `unreadable`. +# a confirmed agent-less pane is `dead` - including one whose reported +# registration its own process inventory proves stale - a registered agent +# that inventory does not contradict is `alive`, and an unexpected or failed +# API read is `unreadable`. fm_backend_herdr_agent_state() { # local target=$1 fm_backend_herdr_parse_target "$target" || { printf 'unreadable'; return 0; } diff --git a/bin/fm-spawn.sh b/bin/fm-spawn.sh index 9158fce64df..fd0b156e56c 100755 --- a/bin/fm-spawn.sh +++ b/bin/fm-spawn.sh @@ -167,6 +167,10 @@ # written by this script; outside the worktree to avoid pi's trust gate) # __PITURNEND__ absolute path to .pi/extensions/fm-primary-turnend-guard.ts in a pi secondmate home # __PIWATCH__ absolute path to .pi/extensions/fm-primary-pi-watch.ts in a pi secondmate home +# __ADDDIRS__ sandbox grants for the ONLY paths the brief lets a worker write +# outside its worktree: state/ (the status file every kind appends) +# plus data// for a scout's report. Trailing space +# included, empty when the harness needs no grant. # __OPINPUT__ absolute path to the canonical operational-input encoder # __WORKTREE__ absolute path to the task worktree # __CURSORBIN__ resolved, cursor-verified executable for a cursor launch @@ -1231,12 +1235,19 @@ launch_template() { # does NOT suppress the interactive ghost text (verified empirically), so the env # var is the correct control. The dim-aware composer reader in fm-tmux-lib.sh is # the defense-in-depth backstop for any pane this flag cannot reach. - claude) printf '%s' 'CLAUDE_CODE_ENABLE_PROMPT_SUGGESTION=false claude --dangerously-skip-permissions __MODELFLAG____EFFORTFLAG__"$(__OPINPUT__ encode launch-brief < __BRIEF__)"' ;; + claude) printf '%s' 'CLAUDE_CODE_ENABLE_PROMPT_SUGGESTION=false claude --permission-mode auto __MODELFLAG____EFFORTFLAG__"$(__OPINPUT__ encode launch-brief < __BRIEF__)"' ;; + # codex runs sandboxed (-s workspace-write) with no escalation path (-a never), + # so every write the brief asks for outside the worktree must be granted + # explicitly or the model just sees "operation not permitted" and cannot ask. + # __ADDDIRS__ carries exactly the brief-permitted paths and nothing else; see + # the placeholder note above. Measured: without it a codex crewmate cannot + # append its own status file, which is the ONLY way it can report done, + # blocked, or needs-decision, so it is mute to the control plane. codex) if [ "$kind" = secondmate ]; then - printf '%s' 'codex __MODELFLAG____EFFORTFLAG__--dangerously-bypass-approvals-and-sandbox "$(__OPINPUT__ encode launch-brief < __BRIEF__)"' + printf '%s' 'codex __MODELFLAG____EFFORTFLAG__-s workspace-write -a never __ADDDIRS__"$(__OPINPUT__ encode launch-brief < __BRIEF__)"' else - printf '%s' 'codex __MODELFLAG____EFFORTFLAG__--dangerously-bypass-approvals-and-sandbox -c "notify=[\"bash\",\"-c\",\"touch __TURNEND__\"]" "$(__OPINPUT__ encode launch-brief < __BRIEF__)"' + printf '%s' 'codex __MODELFLAG____EFFORTFLAG__-s workspace-write -a never __ADDDIRS__-c "notify=[\"bash\",\"-c\",\"touch __TURNEND__\"]" "$(__OPINPUT__ encode launch-brief < __BRIEF__)"' fi ;; opencode) printf '%s' 'OPENCODE_CONFIG_CONTENT='\''{"permission":{"*":"allow"}}'\'' opencode __MODELFLAG__--prompt "$(__OPINPUT__ encode launch-brief < __BRIEF__)"' ;; @@ -1252,14 +1263,20 @@ launch_template() { # session. --always-approve auto-approves every tool execution (verified: the # crewmate runs fully autonomously, no permission gate), which an unattended # crewmate needs; it is the targeted equivalent of claude's - # --dangerously-skip-permissions. grok's turn-end signal does NOT ride the + # --permission-mode auto. grok's turn-end signal does NOT ride the # launch command - it is a Stop-event hook installed below (global hook + # per-task pointer), so the template is identical for ship/scout/secondmate. grok) printf '%s' 'grok --always-approve __MODELFLAG____EFFORTFLAG__"$(__OPINPUT__ encode launch-brief < __BRIEF__)"' ;; - # Cursor Agent CLI. --trust suppresses the workspace-trust prompt, which - # --yolo does NOT cover and which would otherwise block every spawn, since - # each task gets a fresh worktree path cursor has never seen. --yolo is the - # --force alias whose TUI label is "Run Everything". --workspace pins the + # Cursor Agent CLI. --trust suppresses the workspace-trust prompt that + # would otherwise block every spawn, since each task gets a fresh worktree + # path cursor has never seen, and it is also what loads the project hooks. + # --auto-review --sandbox enabled replaces the former blanket --yolo bypass + # with a sandboxed, auto-reviewing autonomy posture, so cursor needs the same + # __ADDDIRS__ grant codex does: measured without it, cursor's sandbox refused + # the status-file append and its auto-review classifier then parked the turn + # on a manual approval prompt, which an unattended crewmate can never clear. + # --add-dir adds writable roots; --workspace still pins the single project + # root that state/.cursor-session binds the busy fold to. --workspace pins the # exact worktree. -w/--worktree is deliberately never passed: it allocates a # SECOND worktree under ~/.cursor/worktrees and would break firstmate's # isolation contract. The binary is resolved rather than named because @@ -1268,7 +1285,7 @@ launch_template() { # inherited CLAUDECODE cannot outrank cursor's own marker in a process that # only reads the environment. Cursor exposes no effort flag, so the shared # effort axis is deliberately omitted and stays in task metadata only. - cursor) printf '%s' 'env -u CLAUDECODE -u PI_CODING_AGENT -u GROK_AGENT -u FM_PI_HARNESS -u CURSOR_INVOKED_AS __CURSORBIN__ --trust --yolo __MODELFLAG__--workspace __WORKTREE__ "$(__OPINPUT__ encode launch-brief < __BRIEF__)"' ;; + cursor) printf '%s' 'env -u CLAUDECODE -u PI_CODING_AGENT -u GROK_AGENT -u FM_PI_HARNESS -u CURSOR_INVOKED_AS __CURSORBIN__ --trust --auto-review --sandbox enabled __ADDDIRS____MODELFLAG__--workspace __WORKTREE__ "$(__OPINPUT__ encode launch-brief < __BRIEF__)"' ;; # Kimi Code rejects a positional prompt, so it launches bare and receives # only an absolute brief pointer after the TUI readiness gate below. # Its turn-end signal is a globally configured Stop hook plus a guarded @@ -2955,6 +2972,15 @@ sq_piturnend=$(shell_quote "$PROJ_ABS/.pi/extensions/fm-primary-turnend-guard.ts sq_piwatch=$(shell_quote "$PROJ_ABS/.pi/extensions/fm-primary-pi-watch.ts") sq_opinput=$(shell_quote "$FM_ROOT/bin/fm-operational-input.sh") sq_worktree=$(shell_quote "$WT") +# The sandbox grant is derived from the brief, not from convenience: bin/fm-brief.sh +# permits a worker exactly two writes outside its worktree, the status file at +# state/.status (every kind) and, for a scout only, data//report.md. Grant +# those two directories and nothing else - never $FM_HOME itself, which also holds +# every other task's worktree, record, and config. +ADDDIRS="--add-dir $(shell_quote "$STATE") " +if [ "$KIND" = scout ]; then + ADDDIRS="$ADDDIRS--add-dir $(shell_quote "$DATA/$ID") " +fi MODELFLAG=$(model_flag_for_harness "$HARNESS" "$MODEL") EFFORTFLAG=$(effort_flag_for_harness "$HARNESS" "$EFFORT") LAUNCH=${LAUNCH//__MODELFLAG__/$MODELFLAG} @@ -2965,6 +2991,7 @@ LAUNCH=${LAUNCH//__PIEXT__/$sq_piext} LAUNCH=${LAUNCH//__PITURNEND__/$sq_piturnend} LAUNCH=${LAUNCH//__PIWATCH__/$sq_piwatch} LAUNCH=${LAUNCH//__OPINPUT__/$sq_opinput} +LAUNCH=${LAUNCH//__ADDDIRS__/$ADDDIRS} case "$HARNESS" in pi|pi-signed) LAUNCH=${LAUNCH//__PIBIN__/"$(shell_quote "$PI_BIN")"} ;; cursor) LAUNCH=${LAUNCH//__CURSORBIN__/"$(shell_quote "$CURSOR_BIN")"} ;; diff --git a/bin/fm-teardown.sh b/bin/fm-teardown.sh index ad9e042ba11..2d9f9618d9f 100755 --- a/bin/fm-teardown.sh +++ b/bin/fm-teardown.sh @@ -683,6 +683,10 @@ remote_secondmate_teardown() { grep -vE "^- $ID( |$)" "$SECONDMATE_REG" > "$tmp" || true mv -f -- "$tmp" "$SECONDMATE_REG" status_retire_presentation_task "$STATE" "$ID" || return 1 +# Best-effort fleet usage harvest runs while the task's state files still +# exist; a harvest failure must never block teardown. +"$FM_ROOT/bin/fm-usage-harvest.sh" "$ID" >/dev/null \ + || echo "warning: usage harvest for $ID failed; continuing teardown" >&2 fm_backlog_atomic_transition remove "$STATE/$ID.meta" "task record" "$STATE" || return 1 rm -f -- "$STATE/$ID.turn-ended" printf 'teardown %s complete (remote %s:%s)\n' "$ID" "$remote_host" "$remote_home" @@ -2872,6 +2876,10 @@ fm_backend_clear_transition "$BACKEND" "$STATE" "$T" || true remove_pr_poll_artifacts "$STATE" "$ID" || exit 1 retire_busy_state "$STATE" "$ID" "$BUSY_GEN" || exit 1 status_retire_presentation_task "$STATE" "$ID" || exit 1 +# Best-effort fleet usage harvest runs while the task's state files still +# exist; a harvest failure must never block teardown. +"$FM_ROOT/bin/fm-usage-harvest.sh" "$ID" >/dev/null \ + || echo "warning: usage harvest for $ID failed; continuing teardown" >&2 rm -f "$STATE/$ID.turn-ended" \ "$STATE/$ID.pi-ext.ts" "$STATE/$ID.grok-turnend-token" \ "$STATE/$ID.kimi-turnend-token" "$STATE/$ID.muse-session" \ diff --git a/bin/fm-test-run.sh b/bin/fm-test-run.sh index a4ef94500fe..5625adf8d25 100755 --- a/bin/fm-test-run.sh +++ b/bin/fm-test-run.sh @@ -264,10 +264,12 @@ family_for_basename() { fm-cmux-claude-composer-live-e2e.test.sh|\ fm-composer-matrix-live-e2e.test.sh|\ fm-codex-continuity-live-e2e.test.sh|fm-grok-continuity-live-e2e.test.sh|\ + fm-crewmate-autonomy-live-e2e.test.sh|\ fm-cursor-primary-live-e2e.test.sh|\ fm-grok-stop-live-e2e.test.sh|fm-harness-adapter-instructions-live-e2e.test.sh|\ fm-harness-liveness-drift-live-e2e.test.sh|\ fm-muse-signals-live-e2e.test.sh|\ + fm-herdr-agent-free-proof-live-e2e.test.sh|\ fm-herdr-version-floor-live-e2e.test.sh|\ fm-opencode-primary-live-e2e.test.sh|fm-pi-branch-live-e2e.test.sh|\ fm-pi-primary-live-e2e.test.sh|\ @@ -1258,7 +1260,15 @@ families_for_changed_path() { printf '%s\n' pure-contract-unit printf '%s\n' live-harness-optin ;; - bin/fm-spawn.sh|bin/fm-send.sh|bin/fm-harness.sh|\ + bin/fm-spawn.sh) + # The launch templates carry each harness's autonomy and sandbox posture, + # which only a real harness can answer for; a template edit re-selects the + # live crewmate-autonomy guard alongside the portable families. + printf '%s\n' backend-dispatch + printf '%s\n' pure-contract-unit + printf '%s\n' live-harness-optin + ;; + bin/fm-send.sh|bin/fm-harness.sh|\ bin/fm-peek.sh|bin/fm-composer*) printf '%s\n' backend-dispatch printf '%s\n' pure-contract-unit diff --git a/bin/fm-usage-harvest.sh b/bin/fm-usage-harvest.sh new file mode 100755 index 00000000000..5180f32f0d3 --- /dev/null +++ b/bin/fm-usage-harvest.sh @@ -0,0 +1,322 @@ +#!/usr/bin/env bash +# fm-usage-harvest.sh - append one fleet usage-ledger row for a finished task. +# +# Usage: fm-usage-harvest.sh +# +# Reads state/.meta (harness=, model=, effort=, worktree=; the +# backend window= line is intentionally not consumed because the ledger has no +# window field) plus the task's state/.status timestamps for the task +# window, then sums the worker's own session-log usage into exactly one JSON +# line appended to data/usage-ledger.jsonl. data/usage-ledger.jsonl is +# gitignored runtime data. +# +# Ledger line schema (this file is the single owner of that schema; the +# report script is a consumer): +# {"task":,"harness":,"model":,"effort":, +# "spawned_at":,"completed_at":, +# "wall_secs":,"turns":, +# "input_tokens":,"cached_input_tokens":, +# "output_tokens":,"reasoning_tokens":, +# "source":} +# +# Wall clock: status-file birth epoch -> status-file mtime epoch; the meta +# file's mtime is the fallback end when the status file is absent, and a +# missing birth timestamp falls back to the meta-file mtime (the spawn-time +# marker, portable where birth time is unavailable) and then the status mtime +# as the start. +# Turn estimate: count of "^working:" lines in the status file. +# +# Per-request usage sources: +# harness=claude: / '-'>/*.jsonl in +# the task window. Each assistant message carries one API request's usage +# at .message.usage (input_tokens, cache_read_input_tokens, +# cache_creation_input_tokens, output_tokens) and Claude logs one entry +# per content block, so requests are deduped by .message.id before +# summing. cached_input_tokens folds cache_read + cache_creation (both +# billed on top of input_tokens); reasoning_tokens captures +# output_tokens_details.thinking_tokens when present (a subset of +# output_tokens). +# harness=codex: /**/*.jsonl in the task window whose +# session_meta cwd equals the meta worktree. Per-turn token_count events +# carry one request's delta at .payload.info.last_token_usage +# (input_tokens, cached_input_tokens, cache_write_input_tokens, +# output_tokens, reasoning_output_tokens); summing those deltas equals the +# final cumulative total. cached_input_tokens folds cached + cache_write +# (subsets of input_tokens); the model comes from the turn_context. +# harness=cursor, a task with a recorded remote_host (its worker ran on +# another machine, so its logs are not on this filesystem), an absent log +# tree, or no in-window match: token fields are null with source +# "unavailable". +# A corrupt log line is skipped best-effort by the parser. +# +# Idempotent: if the ledger already contains a line whose "task" is +# , the command exits 0 without appending. +# +# Overrides (test seams and alternate homes): +# FM_ROOT_OVERRIDE, FM_HOME, FM_STATE_OVERRIDE, FM_DATA_OVERRIDE as usual +# FM_USAGE_CLAUDE_DIR default $HOME/.claude/projects +# FM_USAGE_CODEX_DIR default $HOME/.codex/sessions +# +# Exit status: 0 on a successful or already-present harvest, 1 on a missing +# task record, missing jq, or an unwritable ledger. Callers that must not +# block (teardown) own their own guard. +set -eu + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +FM_ROOT="${FM_ROOT_OVERRIDE:-$(cd "$SCRIPT_DIR/.." && pwd)}" +FM_HOME="${FM_HOME:-${FM_ROOT_OVERRIDE:-$FM_ROOT}}" +STATE="${FM_STATE_OVERRIDE:-$FM_HOME/state}" +DATA="${FM_DATA_OVERRIDE:-$FM_HOME/data}" +CLAUDE_DIR="${FM_USAGE_CLAUDE_DIR:-${HOME:-}/.claude/projects}" +CODEX_DIR="${FM_USAGE_CODEX_DIR:-${HOME:-}/.codex/sessions}" + +# Portable directory-lock helpers (fm_lock_try_acquire / fm_lock_release) let +# the idempotent check-and-append below run as one critical section, so two +# concurrent harvests of the same task cannot both pass the existence check and +# each append a duplicate row. The acquire is bounded (see below) so it never +# blocks the synchronous teardown caller indefinitely. +# shellcheck source=bin/fm-wake-lib.sh +. "$SCRIPT_DIR/fm-wake-lib.sh" + +err() { printf 'error: %s\n' "$1" >&2; } + +if [ "$#" -ne 1 ] || [ -z "$1" ] || case "$1" in *[!A-Za-z0-9._-]*) true ;; *) false ;; esac; then + err "usage: fm-usage-harvest.sh " + exit 1 +fi +command -v jq >/dev/null 2>&1 || { err "jq is required"; exit 1; } + +ID=$1 +META="$STATE/$ID.meta" +STATUS="$STATE/$ID.status" +[ -f "$META" ] || { err "no task record: $META"; exit 1; } + +meta_get() { # + grep "^$1=" "$META" 2>/dev/null | tail -1 | cut -d= -f2- || true +} +HARNESS=$(meta_get harness) +WORKTREE=$(meta_get worktree) +MODEL_META=$(meta_get model) +EFFORT_META=$(meta_get effort) +# A remote secondmate's worker ran on another machine, so its session logs are +# not on this filesystem. Harvesting the local claude/codex trees for such a +# task would at best find nothing and at worst misattribute an unrelated local +# session that happens to match the worktree path, so a task with a recorded +# remote_host is recorded as source=unavailable without a local scan. +REMOTE_HOST=$(meta_get remote_host) + +file_mtime_epoch() { # + local t + t=$(stat -f %m -- "$1" 2>/dev/null) || t=$(stat -c %Y -- "$1" 2>/dev/null) || return 1 + case "$t" in ''|*[!0-9]*) return 1 ;; esac + printf '%s' "$t" +} +file_birth_epoch() { # + local t + t=$(stat -f %B -- "$1" 2>/dev/null) || t=$(stat -c %W -- "$1" 2>/dev/null) || return 1 + # The plausible-epoch guard keeps GNU stat's filesystem-mode %B output + # (block size) from being misread as a birth time. + case "$t" in ''|*[!0-9]*) return 1 ;; esac + [ "$t" -ge 1000000000 ] 2>/dev/null || return 1 + printf '%s' "$t" +} +iso_from_epoch() { # + date -r "$1" -u +%Y-%m-%dT%H:%M:%SZ 2>/dev/null \ + || date -u -d "@$1" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null \ + || return 1 +} + +END_EPOCH=$(file_mtime_epoch "$STATUS" 2>/dev/null || file_mtime_epoch "$META") +START_EPOCH=$(file_birth_epoch "$STATUS" 2>/dev/null || file_mtime_epoch "$META" 2>/dev/null || file_mtime_epoch "$STATUS" 2>/dev/null || printf '%s' "$END_EPOCH") +WALL=$((END_EPOCH - START_EPOCH)) +[ "$WALL" -ge 0 ] || WALL=0 +TURNS=$(grep -c '^working:' "$STATUS" 2>/dev/null || true) +case "$TURNS" in ''|*[!0-9]*) TURNS=0 ;; esac + +# Ref files pin find's mtime window portably (BSD and GNU find both compare +# against -newer file mtimes, and touch -t exists on both). +REFDIR=$(mktemp -d "${TMPDIR:-/tmp}/fm-usage-harvest.XXXXXX") +LEDGER_LOCK= +LEDGER_LOCK_HELD=0 +# Set once the staging file for the ledger append exists; cleanup owns its +# removal so a failed append (an unwritable ledger) cannot leave the staging +# file behind in the fleet data directory. +LEDGER_TMP= +harvest_cleanup() { + local rc=$? + [ "$LEDGER_LOCK_HELD" != 1 ] || fm_lock_release "$LEDGER_LOCK" || true + [ -z "$LEDGER_TMP" ] || rm -f -- "$LEDGER_TMP" + rm -rf -- "$REFDIR" + return "$rc" +} +trap harvest_cleanup EXIT +epoch_to_touch() { # + date -r "$1" +%Y%m%d%H%M.%S 2>/dev/null || date -u -d "@$1" +%Y%m%d%H%M.%S +} +# find -newer compares sub-second mtimes, so the refs only narrow to +# [START-1, END+1]; the per-file epoch filter below then applies the true +# inclusive whole-second window [START_EPOCH, END_EPOCH]. +touch -t "$(epoch_to_touch "$((START_EPOCH - 1))")" "$REFDIR/start" +touch -t "$(epoch_to_touch "$((END_EPOCH + 1))")" "$REFDIR/end" + +LEDGER="$DATA/usage-ledger.jsonl" + +SRC=unavailable +MODEL_LOG= +matched_files() { # : print in-window *.jsonl paths + local dir=$1 depthargs=() f m + [ -d "$dir" ] || return 0 + if [ -n "$2" ]; then + depthargs=(-maxdepth "$2") + fi + while IFS= read -r f; do + m=$(file_mtime_epoch "$f") || continue + if [ "$m" -ge "$START_EPOCH" ] && [ "$m" -le "$END_EPOCH" ]; then + printf '%s\n' "$f" + fi + done < <(find "$dir" ${depthargs[@]+"${depthargs[@]}"} -type f -name '*.jsonl' \ + -newer "$REFDIR/start" ! -newer "$REFDIR/end" -print 2>/dev/null) || true +} + +IT=null; CT=null; OT=null; RT=null +case "$HARNESS" in + claude) + if [ -z "$REMOTE_HOST" ] && [ -n "$WORKTREE" ]; then + encoded=${WORKTREE//\//-} + encoded=${encoded//./-} + files=$(matched_files "$CLAUDE_DIR/$encoded" 1) + if [ -n "$files" ]; then + SRC=claude-projects + IT=0; CT=0; OT=0; RT=0 + # One entry per content block repeats one request's usage; dedupe on + # .message.id so every request is counted exactly once. + while IFS= read -r f; do + row=$(jq -rn ' + reduce inputs as $l ({seen:{},m:null,it:0,ct:0,ot:0,rt:0}; + if $l.type == "assistant" and ($l.message.usage // null) != null then + ($l.message.id // "no-id") as $id + | if .seen[$id] then . else + .seen[$id] = 1 + | .it += ($l.message.usage.input_tokens // 0) + | .ct += (($l.message.usage.cache_read_input_tokens // 0) + + ($l.message.usage.cache_creation_input_tokens // 0)) + | .ot += ($l.message.usage.output_tokens // 0) + | .rt += ($l.message.usage.output_tokens_details.thinking_tokens // 0) + | (if .m == null then .m = ($l.message.model // null) else . end) + end + elif $l.type == "assistant" and ($l.message.model // null) != null and .m == null then + .m = $l.message.model + else . end) + | [.m, .it, .ct, .ot, .rt] | @tsv' "$f" 2>/dev/null || true) + [ -n "$row" ] || continue + IFS=$'\t' read -r m it ct ot rt <<<"$row" + [ -n "$m" ] && [ -z "$MODEL_LOG" ] && MODEL_LOG=$m + IT=$((IT + ${it:-0})) + CT=$((CT + ${ct:-0})) + OT=$((OT + ${ot:-0})) + RT=$((RT + ${rt:-0})) + done </dev/null || true) + [ -n "$row" ] || continue + IFS=$'\t' read -r cwd m it ct ot rt <<<"$row" + [ "$cwd" = "$WORKTREE" ] || continue + found=1 + SRC=codex-sessions + [ -n "$m" ] && [ -z "$MODEL_LOG" ] && MODEL_LOG=$m + IT=$((IT + ${it:-0})) + CT=$((CT + ${ct:-0})) + OT=$((OT + ${ot:-0})) + RT=$((RT + ${rt:-0})) + done </dev/null; then + exit 0 +fi +# Test seam: widen the check-to-append window so a concurrency regression (a +# removed lock) is observable deterministically; unset in production. +[ -z "${FM_USAGE_HARVEST_APPEND_DELAY:-}" ] || sleep "$FM_USAGE_HARVEST_APPEND_DELAY" +LEDGER_TMP="$DATA/usage-ledger.jsonl.tmp.$$" +jq -cn \ + --arg task "$ID" --arg harness "$HARNESS" \ + --arg model "$MODEL" --arg effort "$EFFORT" \ + --arg spawned "$SPAWNED" --arg completed "$COMPLETED" \ + --argjson wall "$WALL" --argjson turns "$TURNS" \ + --argjson it "$IT" --argjson ct "$CT" --argjson ot "$OT" --argjson rt "$RT" \ + --arg source "$SRC" \ + '{task:$task, harness:$harness, + model:(if ($model == "" or $model == "default") then null else $model end), + effort:(if ($effort == "" or $effort == "default") then null else $effort end), + spawned_at:(if $spawned == "" then null else $spawned end), + completed_at:(if $completed == "" then null else $completed end), + wall_secs:$wall, turns:$turns, + input_tokens:$it, cached_input_tokens:$ct, output_tokens:$ot, + reasoning_tokens:$rt, source:$source}' >> "$LEDGER_TMP" +cat "$LEDGER_TMP" >> "$LEDGER" diff --git a/bin/fm-usage-report.sh b/bin/fm-usage-report.sh new file mode 100755 index 00000000000..3821e0c1c41 --- /dev/null +++ b/bin/fm-usage-report.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# fm-usage-report.sh - plain-text reader for data/usage-ledger.jsonl. +# +# Usage: fm-usage-report.sh [ledger-path] +# +# Prints per-model totals and one row per task from the usage ledger written +# by fm-usage-harvest.sh (that file owns the line schema; this script only +# consumes it). With no argument the ledger resolves from the operational +# home exactly as the harvester does. +# +# Output is aligned plain text; unavailable token fields render as "-". +# Exit status: 0 including when the ledger is absent or empty. +set -eu + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +FM_ROOT="${FM_ROOT_OVERRIDE:-$(cd "$SCRIPT_DIR/.." && pwd)}" +FM_HOME="${FM_HOME:-${FM_ROOT_OVERRIDE:-$FM_ROOT}}" +DATA="${FM_DATA_OVERRIDE:-$FM_HOME/data}" + +command -v jq >/dev/null 2>&1 || { printf 'error: jq is required\n' >&2; exit 1; } + +LEDGER=${1:-$DATA/usage-ledger.jsonl} +if [ ! -s "$LEDGER" ]; then + printf 'no usage ledger at %s\n' "$LEDGER" + exit 0 +fi + +printf 'usage ledger: %s (%s rows)\n' "$LEDGER" "$(wc -l < "$LEDGER" | tr -d ' ')" +echo +echo "per-model totals (source-available rows):" +printf '%-24s %6s %12s %12s %12s %12s %10s\n' \ + model tasks input cached output reasoning wall_secs +jq -rs ' + [.[] | select(.source != "unavailable")] | sort_by(.model // "?") | group_by(.model // "?")[] + | [ (.[0].model // "?"), (length | tostring), + (map(.input_tokens // 0) | add | tostring), + (map(.cached_input_tokens // 0) | add | tostring), + (map(.output_tokens // 0) | add | tostring), + (map(.reasoning_tokens // 0) | add | tostring), + (map(.wall_secs // 0) | add | tostring) ] | @tsv' "$LEDGER" | +while IFS=$'\t' read -r model tasks it ct ot rt wall; do + printf '%-24s %6s %12s %12s %12s %12s %10s\n' \ + "$model" "$tasks" "$it" "$ct" "$ot" "$rt" "$wall" +done +echo +echo "per-task rows:" +printf '%-24s %-8s %-20s %-8s %12s %12s %12s %12s %10s %-16s\n' \ + task harness model effort input cached output reasoning wall_secs source +jq -r ' + [ .task, .harness, + (.model // "-"), (.effort // "-"), + (.input_tokens // "-"), (.cached_input_tokens // "-"), + (.output_tokens // "-"), (.reasoning_tokens // "-"), + (.wall_secs // "-"), .source ] | @tsv' "$LEDGER" | +while IFS=$'\t' read -r task harness model effort it ct ot rt wall source; do + printf '%-24s %-8s %-20s %-8s %12s %12s %12s %12s %10s %-16s\n' \ + "$task" "$harness" "$model" "$effort" "$it" "$ct" "$ot" "$rt" "$wall" "$source" +done diff --git a/docs/herdr-backend.md b/docs/herdr-backend.md index 390e8f15172..c6867b5e996 100644 --- a/docs/herdr-backend.md +++ b/docs/herdr-backend.md @@ -269,8 +269,18 @@ Create replaces only a confidently dead or no-agent husk, creates the replacemen This prevents closing the workspace's last tab before a replacement exists. The generic Herdr agent-liveness probe reuses the same classifier. -A structurally gone pane becomes `missing`, a restored agent-less shell becomes `dead`, a registered agent becomes `alive`, and an unexpected read becomes `unreadable`. -Unlike tmux process-name inspection, native registration can classify Pi without guessing from a generic interpreter name. +A structurally gone pane becomes `missing`, an agent-free shell becomes `dead`, a registration the pane's processes do not contradict becomes `alive`, and an unexpected read becomes `unreadable`. +Native registration classifies a harness without guessing from a generic interpreter name, which tmux process-name inspection cannot do. + +A reported registration is not evidence on its own. +Herdr's agent registry is written by whatever reports into it, and a report is not withdrawn when the process that made it goes away, so a registration can outlive the agent it describes. +The classifier therefore corroborates a reported registration against the pane's own `pane process-info` inventory and treats the registration as stale when that inventory positively proves a lone bare idle shell. +This is the same rule the tmux classifier already applies from the foreground process group, expressed through Herdr's own inventory. + +The corroboration is positive-only and reads one strict instantaneous sample, so it can only move a verdict from `alive` toward `dead`, never the other way. +A live harness process, an extra foreground process, a shell with a child of its own, an unreadable inventory, and an inventory answering about a different pane all leave the registration standing. +An idle shell transiently hosting a prompt helper therefore stays `alive` for that sample, and the next poll settles it. +A pane whose registration outlived its agent is a husk on the same terms as a restored one, so create-time replacement and presentation reclaim treat both identically. The session-start sweep uses this probe. Mid-session secondmate agent-process liveness is not implemented because idle secondmates are deliberately exempt from stale-pane escalation and need a separate periodic identity signal. @@ -344,7 +354,11 @@ tests/fm-herdr-session-cleanup.test.sh tests/fm-herdr-session-cleanup-e2e.test.sh tests/fm-afk-inject-herdr-e2e.test.sh tests/fm-afk-pi-herdr-return-e2e.test.sh +tests/fm-herdr-agent-free-proof-live-e2e.test.sh ``` +`tests/fm-herdr-agent-free-proof-live-e2e.test.sh` is the opt-in drift guard for the agent-free proof: it launches every installed harness for real and fails naming the harness and version if a running one ever proves a bare idle shell. +Run it after any harness upgrade and refresh the per-harness evidence it prints. + Real Herdr tests use the named lab helper and default-session tripwire. [`verification/runtime-backends.md`](verification/runtime-backends.md#herdr) records the active version, CLI, projection, event, and lifecycle evidence without task-specific chronology. diff --git a/docs/scripts.md b/docs/scripts.md index d23aafca7cc..99da86f6101 100644 --- a/docs/scripts.md +++ b/docs/scripts.md @@ -128,6 +128,8 @@ The shared no-mistakes gate refusal for fleet lifecycle entrypoints is summarize | `fm-merge-outcome-lib.sh` | Publish a confirmed merge's durable, role-routed supervision outcome | | `fm-promote.sh` | Promote a scout task in place to a protected ship task with an explicit delivery mode, and write the ship instructions carrying that mode's definition of done | | `fm-teardown.sh` | Fail-closed teardown: return landed ship worktrees, require completed scout deliverables, retire secondmate homes | +| `fm-usage-harvest.sh` | Append one finished task's token, wall-clock, and turn cost to the gitignored usage ledger (sole owner of the ledger line schema) | +| `fm-usage-report.sh` | Read the usage ledger into a plain-text per-model and per-task cost report | | `fm-harness.sh` | Detect the running harness and resolve crew or secondmate harness, model, and effort | | `fm-lock.sh` | Per-home firstmate session lock | | `fm-x-lib.sh` | Shared Relay config, relay, and reply-threading helpers | diff --git a/docs/verification/runtime-backends.md b/docs/verification/runtime-backends.md index 024ca44c018..a4de19f8991 100644 --- a/docs/verification/runtime-backends.md +++ b/docs/verification/runtime-backends.md @@ -267,7 +267,7 @@ This guard is the refresh command after any harness upgrade; it spends a small n ## Herdr The compatibility floor is protocol 14. -The whole real-Herdr lane's latest active verification uses both Herdr 0.7.4 protocol 16 and Herdr 0.8.0 protocol 19 on macOS aarch64, while focused Herdr 0.7.5 protocol 17, earlier protocol-16, protocol-14, and 0.7.3 evidence is retained where it defines current behavior or fallbacks. +The whole real-Herdr lane's latest active verification uses both Herdr 0.7.4 protocol 16 and Herdr 0.8.0 protocol 19 on macOS aarch64, with agent lifecycle control and the agent-free proof measured on Herdr 0.8.2, while focused Herdr 0.7.5 protocol 17, earlier protocol-16, protocol-14, and 0.7.3 evidence is retained where it defines current behavior or fallbacks. Protocol 17 keeps every protocol-16 feature gate satisfied; the event and workspace-move floors remain 16. Default-on presentation projection has its own floor at Herdr 0.8.0, protocol 19, verified below. @@ -635,24 +635,71 @@ Polling remained active and is covered as the fallback for capability, connect, ### Agent lifecycle control -Herdr is one of the two backends whose recovery-grade agent-state classifier the control plane may trust ([agent-control.md](../agent-control.md)), so its lifecycle gating is measured against the real binary; reverified 2026-08-08 on Herdr 0.8.0, and first measured 2026-08-02 on Herdr 0.7.5 with identical results: +Herdr is one of the two backends whose recovery-grade agent-state classifier the control plane may trust ([agent-control.md](../agent-control.md)), so its lifecycle gating is measured against the real binary; reverified 2026-09-01 on Herdr 0.8.2, and first measured 2026-08-02 on Herdr 0.7.5: ```sh tests/fm-control-herdr-smoke.test.sh ``` -Observed output: +Observed output on Herdr 0.8.2: ```text ok - real herdr: exit on a pane with no registered agent is idempotent success ok - real herdr: interrupt refuses when herdr's own agent registry reports no agent +ok - real herdr: a reported registration the pane's own process inventory contradicts reads agent-free +ok - real herdr: an exited worker's pane is positively eligible for its replacement instead of being typed into +ok - real herdr: interrupt still refuses on a pane whose registration outlived its agent +ok - real herdr: the same registration over a running foreground process stays alive ok - real herdr: interrupt delivers the harness's key and proves the agent survived it -ok - real herdr: no control verb removed the endpoint or the task's local copy ok - real herdr: an agent that does not stop fails closed instead of being reported as stopped +ok - real herdr: no control verb removed the endpoint, the task's local copy, or its branch +``` + +The registry written through `herdr pane report-agent` is the same source `fm_backend_herdr_agent_state` classifies, so the case drives the registry and the pane's processes apart deliberately and runs the identical registered reading twice: once over a plain shell and once over a real foreground process, with no real agent launched. + +Herdr 0.8.2 keeps such a report registered even though the pane runs nothing but its shell, measured in an isolated `fm-lab-` session: + +```text +herdr pane report-agent --source fm-control-smoke --agent fm-control-smoke-agent --state idle +herdr agent get -> {"agent_status":"idle"} +herdr pane process-info -> foreground_processes [{"name":"zsh","argv0":"zsh"}] +``` + +That is why the classifier corroborates the registry against the pane's own inventory rather than trusting a reported registration alone. +Herdr does validate a report claiming a source it owns itself: an otherwise identical `--source herdr:pi --agent pi` report on the same shell-only pane was rejected and `agent get` still answered `agent_not_found`. + +`tests/fm-control-herdr-smoke.test.sh` is the guard that refreshes this record; run it after every Herdr upgrade rather than trusting the version above. + +### Agent-free proof across installed harnesses + +The classifier's negative verdict rests on `pane process-info` naming a lone bare idle shell, and both that name and its argv0 come from the harness vendor, so the proof is measured against every installed harness rather than a stub. +Measured 2026-09-01 on Herdr 0.8.2, macOS aarch64, in an isolated `fm-lab-` session: + +```sh +FM_HERDR_AGENT_FREE_PROOF=1 tests/fm-herdr-agent-free-proof-live-e2e.test.sh ``` -The registry read through `herdr pane report-agent` is the same source `fm_backend_herdr_agent_state` classifies, so registering and not registering an agent on a plain shell pane exercises exactly the gate every lifecycle verb depends on, with no real agent launched. -That command is the guard that refreshes this record; run it after every Herdr upgrade rather than trusting the version above. +Observed output: + +```text +# herdr: herdr 0.8.2 +# claude 2.1.252 (Claude Code): foreground=[2.1.252/claude] state=alive +ok - herdr agent-free proof: claude 2.1.252 (Claude Code) running in a Herdr pane never proves a bare idle shell +# codex codex-cli 0.150.1: foreground=[codex/codex] state=alive +ok - herdr agent-free proof: codex codex-cli 0.150.1 running in a Herdr pane never proves a bare idle shell +# opencode 1.17.11: foreground=[opencode/opencode] state=alive +ok - herdr agent-free proof: opencode 1.17.11 running in a Herdr pane never proves a bare idle shell +# pi 0.84.4: foreground=[node/pi] state=alive +ok - herdr agent-free proof: pi 0.84.4 running in a Herdr pane never proves a bare idle shell +# cursor 2026.08.31-4057e58: foreground=[node/cursor-agent] state=alive +ok - herdr agent-free proof: cursor 2026.08.31-4057e58 running in a Herdr pane never proves a bare idle shell +# unverified on this machine (not installed): pi-signed grok kimi muse +# checked 5 installed harness(es) on herdr herdr 0.8.2 in workspace w1 +``` + +Every installed harness owns the pane's foreground under its own argv0, so none can satisfy the shell-only proof, and Herdr registered each of them within the guard's wait. +`pi-signed`, `grok`, `kimi`, and `muse` were not installed on that machine and remain unverified here; the guard reports them explicitly and refuses a pass that checked nothing. +This is the command that refreshes this record; run it after every harness upgrade. ### Away-mode transport @@ -918,6 +965,70 @@ This row is a delivery guard for submit acknowledgement only; recorded worker st | Skill invocation | `/`; cursor discovers firstmate's user-level skills, and `/no-mistakes` autocompleted with firstmate's own description and invoked the skill | | Slash popup | real: the first Enter closes the popup and a SECOND Enter submits, the same hazard as grok, covered by the submit core's retried Enter | +The `Workspace trust` and `Autonomy` rows record what the `--yolo` flag itself does, measured when that flag was still the launch default. +`bin/fm-spawn.sh` now launches cursor with `--trust --auto-review --sandbox enabled` plus the brief-permitted `--add-dir` roots instead, and the [Crewmate autonomy and the status-file write contract](#crewmate-autonomy-and-the-status-file-write-contract) record owns the readings for that posture. + +### Crewmate autonomy and the status-file write contract + +`tests/fm-crewmate-autonomy-live-e2e.test.sh` is the opt-in real-harness guard for the launch and write contract. +Run it after a harness upgrade or a `bin/fm-spawn.sh` launch-template change: + +```sh +FM_CREWMATE_AUTONOMY_LIVE=1 tests/fm-crewmate-autonomy-live-e2e.test.sh +``` + +The guard creates a private `fm-lab-` Herdr session and a lab `FM_HOME` below `$HOME`, never the default session or `/tmp`. +It first sends no key and reports whether the harness reached its composer without a dialog, then submits one prompt asking the crewmate itself to append its status file and a scout report while attempting one ungranted sibling path. +The lab home matters because Codex's `workspace-write` sandbox permits `/tmp` and `$TMPDIR` by default, which would make a status-file write there vacuous. + +#### Recorded readings, 2026-09-02 + +Measured on macOS aarch64 with Herdr 0.8.2, in an isolated `fm-lab-` session whose lab `FM_HOME` sat under `$HOME`. +Harness versions under measurement: Claude Code 2.1.258, Codex CLI 0.152.1, Cursor Agent 2026.08.31-4057e58. + +Observed output, exit status 0: + +```text +# herdr: herdr 0.8.2 +# lab home: <$HOME>/.fm-crewmate-autonomy-lab-61295 (status files outside the lab worktree) +# claude: accepted-workspace-trust +# claude 2.1.258 (Claude Code): NOT unattended - reached its composer only via accepted-workspace-trust (see the record in docs/verification/runtime-backends.md) +# claude 2.1.258 (Claude Code): ready=accepted-workspace-trust submit=empty reply=landed +ok - crewmate write contract: claude 2.1.258 (Claude Code) appends its status file and writes its scout report, both outside its worktree +# claude 2.1.258 (Claude Code): carries no sandbox flag, and did write outside the brief-permitted directories (/Users/npayette/.fm-crewmate-autonomy-lab-61295/denied/claude-61295.status); its posture is an approval classifier, not a filesystem boundary +# codex: accepted-directory-trust +# codex: declined-hook-trust +# codex codex-cli 0.152.1: NOT unattended - reached its composer only via accepted-directory-trust,declined-hook-trust (see the record in docs/verification/runtime-backends.md) +# codex codex-cli 0.152.1: ready=accepted-directory-trust,declined-hook-trust submit=empty reply=landed +ok - crewmate write contract: codex codex-cli 0.152.1 appends its status file and writes its scout report, both outside its worktree +ok - narrow grant: codex codex-cli 0.152.1 is still refused a write outside the two brief-permitted directories +ok - unattended launch: cursor 2026.08.31-4057e58 reaches an empty composer with no key sent +# cursor 2026.08.31-4057e58: ready=unattended submit=empty reply=landed +ok - crewmate write contract: cursor 2026.08.31-4057e58 appends its status file and writes its scout report, both outside its worktree +ok - workspace binding: cursor 2026.08.31-4057e58 still records the task worktree as its exact workspacePath under the added writable roots +ok - narrow grant: cursor 2026.08.31-4057e58 is still refused a write outside the two brief-permitted directories +# checked 3 installed harness(es) on herdr herdr 0.8.2 +``` + +| Harness and version | Unattended launch | The two brief-permitted writes | An ungranted sibling path | +| --- | --- | --- | --- | +| Claude Code 2.1.258 | Not unattended: the workspace-trust dialog stood before the composer and the guard reached it only via `accepted-workspace-trust`. An earlier reading in this work saw the identical dialog under the former `--dangerously-skip-permissions`, so it is not a consequence of `--permission-mode auto`. | Both landed. | Also landed, and is recorded rather than failed: `--permission-mode auto` is an approval classifier, not a filesystem sandbox, so this posture has no grant to keep narrow. | +| Codex CLI 0.152.1 | Not unattended: a directory-trust dialog and then `Hooks need review` stood before the composer, cleared as `accepted-directory-trust,declined-hook-trust`. | Both landed under `-s workspace-write -a never` plus the two `--add-dir` grants. | Refused, so the grant is proven narrow rather than merely sufficient. | +| Cursor Agent 2026.08.31-4057e58 | Unattended: an empty composer with no key sent under `--trust --auto-review --sandbox enabled`. | Both landed under the same two `--add-dir` grants. | Refused, so that grant is proven narrow too. | + +The granted set is exactly what `bin/fm-brief.sh` permits a worker outside its worktree: `state/` for the status file every worker appends, plus `data//` for a scout's report. +Cursor still records the task worktree as its exact `workspacePath` under the added writable roots, so the `state/.cursor-session` binding the busy fold depends on survives the grant. + +The `--add-dir` grant is what turns the Codex reading from broken into passing. +Without it the same guard measured a Codex crewmate's status append failing with `operation not permitted`, and `-a never` left the model no way to ask, so such a worker could not report done, blocked, or needs-decision at all. + +An earlier guard iteration accidentally installed Codex CLI 0.152.1 over 0.150.1 when Enter landed on its update modal. +Version 0.150.1 remains on disk, and the guard now waits for an empty composer to stay empty for ten seconds and dismisses an update offer with Escape rather than Enter before it can submit anything. +The run above is also the Codex 0.152.1 compatibility evidence: it launched through the adapter's own posture, the shared composer classifier read `empty` and held it for the full settle window, and the shared submit path returned `empty` with the crewmate's reply landing in the pane. + +Not established here: the guard measures claude, codex, and cursor, so neither axis was read for opencode, pi, grok, kimi, or muse. +Neither is the captain's own fleet covered, because every reading was taken in a throwaway lab home and lab session rather than a real task. + ### End-to-end A throwaway scout was spawned through `bin/fm-spawn.sh --scout --backend tmux` on a real cursor worker and driven to completion: diff --git a/tests/fm-backend-herdr-presentation-e2e.test.sh b/tests/fm-backend-herdr-presentation-e2e.test.sh index ac62abb4275..144e5548cf6 100755 --- a/tests/fm-backend-herdr-presentation-e2e.test.sh +++ b/tests/fm-backend-herdr-presentation-e2e.test.sh @@ -1401,6 +1401,8 @@ assert_no_projection_mutation_since "$START" "agent-free duplicate-token recover lab workspace get "$DUP1_WSID" >/dev/null 2>&1 || fail "duplicate-token recovery removed the first quarantined workspace" lab workspace get "$DUP2_WSID" >/dev/null 2>&1 || fail "duplicate-token recovery removed the second quarantined workspace" +herdr_pane_run_foreground "$HERDR_LAB_SESSION" "$DUP1_PANE" \ + || fail "could not give the duplicate-live-agent risk fixture a running foreground process" lab pane report-agent "$DUP1_PANE" --source fm-projection-e2e --agent test-agent --state idle >/dev/null \ || fail "could not register the duplicate-live-agent risk fixture" START=$(log_line_count) diff --git a/tests/fm-backend-herdr-respawn-idem-e2e.test.sh b/tests/fm-backend-herdr-respawn-idem-e2e.test.sh index 382fb72d979..b0e78e6c092 100755 --- a/tests/fm-backend-herdr-respawn-idem-e2e.test.sh +++ b/tests/fm-backend-herdr-respawn-idem-e2e.test.sh @@ -159,11 +159,16 @@ WS_COUNT=$(printf '%s' "$WS_TABS" | jq -r '.result.tabs? // [] | length') pass "fixed: the workspace holds exactly the 2 replacement tabs after both respawns - no leaked husk tabs, no destroyed workspace" # --- 4. a GENUINELY live duplicate still refuses, unchanged ----------------- -# Register a real agent (herdr's own native registration primitive) on one of -# the freshly-respawned panes, then confirm a further same-labeled spawn -# attempt refuses exactly as before - the husk fix must never touch a pane -# that actually has something registered in it. - +# Give one of the freshly-respawned panes a real running foreground process and +# register a real agent on it (herdr's own native registration primitive), then +# confirm a further same-labeled spawn attempt refuses exactly as before - the +# husk fix must never touch a pane that is actually running something. +# Both halves matter: a registration standing over a bare idle shell is the +# husk shape, because a report is not withdrawn when the process that made it +# goes away (docs/herdr-backend.md "Restart and liveness behavior"). + +herdr_pane_run_foreground "$SESSION" "$NEW_CREW_PANE_ID" \ + || fail "could not give the respawned crewmate-shaped pane a running foreground process" herdr pane report-agent "$NEW_CREW_PANE_ID" --source fm-respawn-e2e --agent fm-respawn-live-agent --state idle --session "$SESSION" >/dev/null 2>&1 \ || fail "could not register a live agent on the respawned crewmate-shaped pane" diff --git a/tests/fm-backend-herdr-smoke.test.sh b/tests/fm-backend-herdr-smoke.test.sh index 98f1db2e974..9157bd70aa0 100755 --- a/tests/fm-backend-herdr-smoke.test.sh +++ b/tests/fm-backend-herdr-smoke.test.sh @@ -121,6 +121,11 @@ pass "real herdr: create_task prunes the freshly-created workspace's seeded defa # 1. A genuinely LIVE duplicate (a real registered agent, via herdr's own # `pane report-agent`) must still refuse exactly as before. +# "Genuinely live" means both halves the classifier reads: a registration +# AND a pane that is actually running something. A registration standing +# over a bare idle shell is the husk shape instead, because a report is not +# withdrawn when the process that made it goes away +# (docs/herdr-backend.md "Restart and liveness behavior"). LIVE_DUP_LABEL="fm-smoke-livedup" LIVE_DUP_IDS=$(fm_backend_herdr_create_task "$CONTAINER" "$LIVE_DUP_LABEL" /tmp) || fail "could not create the live-duplicate scenario's tab" read -r LIVE_DUP_TAB_ID LIVE_DUP_PANE_ID </dev/null 2>&1 \ || fail "could not register a live agent on the live-duplicate scenario's pane" if fm_backend_herdr_create_task "$CONTAINER" "$LIVE_DUP_LABEL" /tmp >/dev/null 2>&1; then diff --git a/tests/fm-backend-herdr.test.sh b/tests/fm-backend-herdr.test.sh index 426d7ceac26..70537de0ba0 100755 --- a/tests/fm-backend-herdr.test.sh +++ b/tests/fm-backend-herdr.test.sh @@ -835,6 +835,192 @@ test_create_task_creates_and_parses_ids() { pass "fm_backend_herdr_create_task: creates a tab and parses tab_id/pane_id from the JSON response, prunes nothing when no seeded tab id is given" } +# --- a reported registration corroborated against the process inventory ------ +# +# Herdr's agent registry is written by whatever reports into it, and a report +# is not withdrawn when the process that made it goes away. A task whose worker +# had exited to its shell therefore read `alive` forever: every lifecycle verb +# typed the harness's exit command into a shell and refused to relaunch, with +# no state that could ever clear it. The classifier now settles the negative +# verdict from the pane's own `pane process-info` inventory, exactly as the +# tmux classifier already settles it from the foreground process group. +# +# These cases drive the two signals apart deliberately with REAL processes and +# no harness: the same registered `idle` reading resolves to `no-agent` or to +# `live` purely from the inventory beside it. Only a positive proof of a lone +# bare idle shell downgrades the verdict; a live process, an extra foreground +# process, a shell with a child, an unreadable inventory, and an inventory +# about a different pane all keep `live`. + +process_info_fixture() { # + printf '{"result":{"type":"pane_process_info","process_info":{"pane_id":"%s","shell_pid":%s,"foreground_process_group_id":%s,"foreground_processes":%s}}}\n' \ + "$1" "$2" "$3" "$4" +} + +# bare_shell_inventory: the inventory herdr reports for a pane whose agent has +# exited - one foreground process, which is the pane's own shell. Backed by a +# REAL pid so the operating-system half of the proof (exactly one row, no +# child, sleeping) is answered by the real `ps`, never a stub. +bare_shell_inventory() { # + process_info_fixture "$1" "$2" "$2" \ + "$(printf '[{"pid":%s,"name":"zsh","argv0":"zsh"}]' "$2")" +} + +# agent_state_case: run both classifier entry points over one canned +# pane-get/agent-get/process-info sequence and echo " ". +agent_state_case() { # [] + local dir=$1 pane=$2 agent_json=$3 proc_json=${4:-} log resp fb + mkdir -p "$dir/responses"; log="$dir/log"; resp="$dir/responses"; : > "$log" + printf '{"result":{"pane":{"pane_id":"%s"}}}\n' "$pane" > "$resp/1.out" + printf '%s' "$agent_json" > "$resp/2.out" + [ -z "$proc_json" ] || printf '%s' "$proc_json" > "$resp/3.out" + printf '{"result":{"pane":{"pane_id":"%s"}}}\n' "$pane" > "$resp/4.out" + printf '%s' "$agent_json" > "$resp/5.out" + [ -z "$proc_json" ] || printf '%s' "$proc_json" > "$resp/6.out" + fb=$(make_herdr_fakebin "$dir") + PATH="$fb:$PATH" FM_HERDR_LOG="$log" FM_HERDR_RESPONSES="$resp" \ + bash -c '. "$0/bin/backends/herdr.sh" + printf "%s %s" \ + "$(fm_backend_herdr_pane_agent_state fmtest "$1")" \ + "$(fm_backend_herdr_agent_state "fmtest:$1")"' "$ROOT" "$pane" +} + +test_stale_registration_over_a_bare_idle_shell_is_agent_free() { + local dir bgpid stale live + dir="$TMP_ROOT/reg-stale" + # A real, bare, idle process with no child of its own: the operating-system + # half of the idle-shell proof is answered by the real `ps`. + sleep 300 & bgpid=$! + stale=$(agent_state_case "$dir/shell" w1:p2 \ + '{"result":{"agent":{"agent_status":"idle"}}}' \ + "$(bare_shell_inventory w1:p2 "$bgpid")") + # The SAME registered reading, with a live harness process in the pane + # instead. Asserting both halves keeps the case from going vacuous: the + # downgrade must come from the inventory, never from the registration. + live=$(agent_state_case "$dir/live" w1:p2 \ + '{"result":{"agent":{"agent_status":"idle"}}}' \ + "$(process_info_fixture w1:p2 "$bgpid" 4242 '[{"pid":4242,"name":"node","argv0":"pi"}]')") + kill "$bgpid" 2>/dev/null || true; wait "$bgpid" 2>/dev/null || true + [ "$stale" = "no-agent dead" ] \ + || fail "a registration whose pane holds only a bare idle shell must classify agent-free, got '$stale'" + [ "$live" = "live alive" ] \ + || fail "the same registration with a live harness process in the pane must stay alive, got '$live'" + pass "herdr agent state: a reported registration contradicted by a lone bare idle shell reads agent-free, while the identical registration over a live process stays alive" +} + +test_registration_over_an_ambiguous_inventory_stays_alive() { + local dir bgpid out + dir="$TMP_ROOT/reg-ambiguous" + sleep 300 & bgpid=$! + # An idle shell transiently hosting a prompt helper: two foreground + # processes, so nothing is proved and the registration stands. + out=$(agent_state_case "$dir" w1:p2 \ + '{"result":{"agent":{"agent_status":"working"}}}' \ + "$(process_info_fixture w1:p2 "$bgpid" "$bgpid" \ + "$(printf '[{"pid":99998,"name":"starship","argv0":"starship"},{"pid":%s,"name":"zsh","argv0":"zsh"}]' "$bgpid")")") + kill "$bgpid" 2>/dev/null || true; wait "$bgpid" 2>/dev/null || true + [ "$out" = "live alive" ] \ + || fail "an ambiguous process inventory must never downgrade a registration, got '$out'" + pass "herdr agent state: an ambiguous process inventory leaves a registered agent alive rather than licensing recovery" +} + +test_registration_over_a_shell_with_a_child_stays_alive() { + local dir bgpid out attempt=0 child= + dir="$TMP_ROOT/reg-child" + # A real shell that is genuinely running something: the trailing `true` keeps + # bash from exec-ing away, so the child row is real. + bash -c 'sleep 300; true' & bgpid=$! + # Give the real child a moment to exist before the proof reads the table. + while [ "$attempt" -lt 200 ]; do + child=$(ps -axo ppid= -o pid= 2>/dev/null | awk -v p="$bgpid" '$1 == p {print $2}') + [ -n "$child" ] && break + sleep 0.05 + attempt=$((attempt + 1)) + done + if [ -z "$child" ]; then + kill "$bgpid" 2>/dev/null || true; wait "$bgpid" 2>/dev/null || true + fail "ps never reported a child of the backgrounded shell $bgpid, so this case cannot exercise a shell that is running something" + fi + out=$(agent_state_case "$dir" w1:p2 \ + '{"result":{"agent":{"agent_status":"idle"}}}' \ + "$(bare_shell_inventory w1:p2 "$bgpid")") + kill "$bgpid" 2>/dev/null || true; wait "$bgpid" 2>/dev/null || true + [ "$out" = "live alive" ] \ + || fail "a shell that still has a child of its own is not idle and must stay alive, got '$out'" + pass "herdr agent state: a shell running a child of its own never proves an agent-free pane" +} + +test_registration_with_an_unreadable_inventory_stays_alive() { + local dir out + dir="$TMP_ROOT/reg-unreadable" + # No process-info response at all: the fake answers empty, exactly as a + # failed or unparseable inventory read would. + out=$(agent_state_case "$dir" w1:p2 '{"result":{"agent":{"agent_status":"idle"}}}') + [ "$out" = "live alive" ] \ + || fail "an unreadable process inventory must leave the registration standing, got '$out'" + pass "herdr agent state: an unreadable process inventory leaves a registered agent alive" +} + +test_registration_with_an_inventory_for_another_pane_stays_alive() { + local dir bgpid out + dir="$TMP_ROOT/reg-other-pane" + sleep 300 & bgpid=$! + # A bare-idle-shell inventory that answers about a DIFFERENT pane: the + # reading belongs to something else and can never settle this pane. + out=$(agent_state_case "$dir" w1:p2 \ + '{"result":{"agent":{"agent_status":"idle"}}}' \ + "$(bare_shell_inventory w9:p9 "$bgpid")") + kill "$bgpid" 2>/dev/null || true; wait "$bgpid" 2>/dev/null || true + [ "$out" = "live alive" ] \ + || fail "an inventory about another pane must never settle this pane's state, got '$out'" + pass "herdr agent state: a process inventory answering about a different pane never downgrades this pane's registration" +} + +test_unregistered_pane_never_reads_the_process_inventory() { + local dir log resp fb out + dir="$TMP_ROOT/reg-absent"; mkdir -p "$dir/responses"; log="$dir/log"; resp="$dir/responses"; : > "$log" + printf '{"result":{"pane":{"pane_id":"w1:p2"}}}\n' > "$resp/1.out" + printf '{"error":{"code":"agent_not_found","message":"agent target w1:p2 not found"}}\n' > "$resp/2.out" + fb=$(make_herdr_fakebin "$dir") + out=$(PATH="$fb:$PATH" FM_HERDR_LOG="$log" FM_HERDR_RESPONSES="$resp" \ + bash -c '. "$0/bin/backends/herdr.sh"; fm_backend_herdr_pane_agent_state fmtest w1:p2' "$ROOT") + [ "$out" = no-agent ] || fail "an unregistered pane must still read no-agent, got '$out'" + assert_not_contains "$(cat "$log")" $'pane\x1fprocess-info' \ + "an already-agent-free pane must not pay for the corroborating inventory read" + pass "herdr agent state: an unregistered pane is agent-free without reading the process inventory at all" +} + +test_create_task_replaces_a_stale_registration_husk() { + local dir log resp fb bgpid out tab pane + dir="$TMP_ROOT/husk-stale-registration"; mkdir -p "$dir/responses" + log="$dir/log"; resp="$dir/responses"; : > "$log" + sleep 300 & bgpid=$! + printf '{"result":{"tabs":[{"tab_id":"w1:t2","label":"fm-husk3","workspace_id":"w1"}]}}\n' > "$resp/1.out" + printf '{"result":{"panes":[{"pane_id":"w1:p2","tab_id":"w1:t2"}]}}\n' > "$resp/2.out" + printf '{"result":{"pane":{"pane_id":"w1:p2"}}}\n' > "$resp/3.out" + # 4: agent get -> still registered, because nothing ever withdrew the report + printf '{"result":{"agent":{"agent_status":"idle"}}}\n' > "$resp/4.out" + # 5: pane process-info -> the pane is a lone bare idle shell + bare_shell_inventory w1:p2 "$bgpid" > "$resp/5.out" + # 6: tab create -> the replacement tab, created BEFORE the husk is closed + printf '{"result":{"tab":{"tab_id":"w1:t3"},"root_pane":{"pane_id":"w1:p3"}}}\n' > "$resp/6.out" + printf '{"result":{"tabs":[{"tab_id":"w1:t3","label":"fm-husk3","workspace_id":"w1"}]}}\n' > "$resp/8.out" + fb=$(make_herdr_fakebin "$dir") + out=$( PATH="$fb:$PATH" FM_HERDR_LOG="$log" FM_HERDR_RESPONSES="$resp" \ + bash -c '. "$0/bin/backends/herdr.sh"; fm_backend_herdr_create_task fmtest:w1 fm-husk3 /tmp/proj' "$ROOT" ) \ + || { kill "$bgpid" 2>/dev/null; fail "create_task should replace a duplicate whose registration outlived its agent"; } + kill "$bgpid" 2>/dev/null || true; wait "$bgpid" 2>/dev/null || true + read -r tab pane <&1 } +register_agent() { # + herdr pane report-agent "$PANE_ID" --source fm-control-smoke \ + --agent fm-control-smoke-agent --state "$1" --session "$SESSION" >/dev/null 2>&1 +} + +# The same read the backend's own capture performs +# (fm_backend_herdr_capture in bin/backends/herdr.sh): --source recent with a +# generous --lines, because a small bound returns nothing at all. The exit +# status is propagated so callers can refuse to treat a failed read as evidence +# that nothing was typed into the pane. +pane_text() { + herdr pane read "$PANE_ID" --session "$SESSION" --source recent --lines 200 +} + +registered_status() { + herdr agent get "$PANE_ID" --session "$SESSION" 2>/dev/null \ + | jq -r '.result.agent.agent_status // empty' 2>/dev/null +} + # --- no registered agent: the endpoint exists but hosts no agent ------------ OUT=$(run_control hsmoke exit) || fail "exit against an agent-free herdr pane should be idempotent success: $OUT" @@ -113,30 +141,78 @@ case "$OUT" in esac pass "real herdr: interrupt refuses when herdr's own agent registry reports no agent" -# --- a registered agent: classification flips, and the verbs follow --------- +# --- a registration the pane's own processes contradict --------------------- +# +# The worker exited to its shell and nothing withdrew its report. Trusting the +# registry alone left the task alive forever, so every replacement attempt +# typed the harness's exit command into a shell and refused to relaunch. -herdr pane report-agent "$PANE_ID" --source fm-control-smoke --agent fm-control-smoke-agent \ - --state idle --session "$SESSION" >/dev/null 2>&1 \ - || fail "could not register a live agent on the task pane" +register_agent idle || fail "could not leave a registration on the task pane" +[ -n "$(registered_status)" ] \ + || fail "herdr did not keep the registration, so this case cannot exercise the contradiction" STATE=$(fm_backend_agent_state herdr "$SESSION:$PANE_ID") -[ "$STATE" = alive ] || fail "herdr should classify a registered agent as alive, got '$STATE'" +[ "$STATE" = dead ] \ + || fail "a registration whose pane holds only its shell must classify agent-free, got '$STATE'" +pass "real herdr: a reported registration the pane's own process inventory contradicts reads agent-free" + +OUT=$(run_control hsmoke exit) \ + || fail "exit should be idempotent success once the pane is proved agent-free: $OUT" +case "$OUT" in + "already-stopped hsmoke"*) : ;; + *) fail "an agent-free pane should report already-stopped, got: $OUT" ;; +esac +PANE_TEXT=$(pane_text) \ + || fail "could not read the task pane, so an absent /quit proves nothing about what exit typed into it" +case "$PANE_TEXT" in + *"/quit"*) fail "exit typed the harness's exit command into a pane that hosts a plain shell" ;; +esac +pass "real herdr: an exited worker's pane is positively eligible for its replacement instead of being typed into" + +if OUT=$(run_control hsmoke interrupt 2>&1); then + fail "interrupt should refuse on a pane proved agent-free: $OUT" +fi +case "$OUT" in + *"nothing to interrupt"*) : ;; + *) fail "the interrupt refusal should say there is no agent, got: $OUT" ;; +esac +pass "real herdr: interrupt still refuses on a pane whose registration outlived its agent" + +# --- the same registration over a genuinely running process ---------------- +# +# The identical registered reading, with a real non-shell foreground process +# in the pane. Nothing here may be reclassified as recoverable. + +fm_backend_herdr_send_text_line "$SESSION:$PANE_ID" 'sleep 600' \ + || fail "could not start a foreground process in the task pane" +ATTEMPT=0 +while [ "$ATTEMPT" -lt 100 ]; do + case "$(herdr pane process-info --pane "$PANE_ID" --session "$SESSION" 2>/dev/null \ + | jq -r '[.result.process_info.foreground_processes[]?.name] | join(",")' 2>/dev/null)" in + *sleep*) break ;; + esac + sleep 0.1 + ATTEMPT=$((ATTEMPT + 1)) +done +[ "$ATTEMPT" -lt 100 ] || fail "the foreground process never appeared in herdr's process inventory" +register_agent idle || fail "could not register an agent over the running process" + +STATE=$(fm_backend_agent_state herdr "$SESSION:$PANE_ID") +[ "$STATE" = alive ] \ + || fail "a registration over a genuinely running foreground process must stay alive, got '$STATE'" +pass "real herdr: the same registration over a running foreground process stays alive" OUT=$(run_control hsmoke interrupt) || fail "interrupt against a registered agent should succeed: $OUT" case "$OUT" in - *"interrupt-delivered hsmoke harness=claude backend=herdr verified=agent-alive cancel=unconfirmed"*) : ;; + *"interrupt-delivered hsmoke harness=pi backend=herdr verified=agent-alive cancel=unconfirmed"*) : ;; *) fail "interrupt should report the agent-alive proof on herdr, got: $OUT" ;; esac pass "real herdr: interrupt delivers the harness's key and proves the agent survived it" -herdr pane get "$PANE_ID" --session "$SESSION" >/dev/null 2>&1 \ - || fail "the control plane must never remove the endpoint it was operating on" -[ -d "$WT" ] || fail "the control plane must never remove the task's local copy" -pass "real herdr: no control verb removed the endpoint or the task's local copy" - -# Last, because it deliberately types a harness command into a pane that hosts -# a plain shell: the registered agent cannot actually be stopped that way, and -# the control plane must say so rather than report a stop it did not achieve. +# Last, because it deliberately types a harness command into a pane whose +# foreground process cannot act on it: the agent cannot be stopped that way, +# and the control plane must say so rather than report a stop it did not +# achieve. if OUT=$(run_control hsmoke exit 2>&1); then fail "exit should fail closed when the agent does not stop: $OUT" fi @@ -146,4 +222,11 @@ case "$OUT" in esac pass "real herdr: an agent that does not stop fails closed instead of being reported as stopped" +herdr pane get "$PANE_ID" --session "$SESSION" >/dev/null 2>&1 \ + || fail "the control plane must never remove the endpoint it was operating on" +[ -d "$WT" ] || fail "the control plane must never remove the task's local copy" +git -C "$WT" rev-parse --abbrev-ref HEAD 2>/dev/null | grep -qx hsmoke \ + || fail "the control plane must never move the task's branch" +pass "real herdr: no control verb removed the endpoint, the task's local copy, or its branch" + fm_backend_herdr_kill "$SESSION:$PANE_ID" 2>/dev/null || true diff --git a/tests/fm-control-relaunch.test.sh b/tests/fm-control-relaunch.test.sh index c3ab0415812..5129ac0e3a3 100755 --- a/tests/fm-control-relaunch.test.sh +++ b/tests/fm-control-relaunch.test.sh @@ -455,6 +455,85 @@ test_relaunch_requires_a_note_for_a_ship_task() { pass "fm-control relaunch: a ship task refuses without the progress note its replacement needs" } +# --- 1b. an agent that already exited to its shell --------------------------- +# +# The endpoint outlives the agent: a worker that exits leaves its shell sitting +# in the task's local copy. That endpoint is positively agent-free, so the +# replacement its operator is asking for must launch, and nothing may be typed +# at the shell on the way there. Durable busy evidence from the incarnation +# that ended is stale by definition and must not resurrect an interrupt. + +test_relaunch_over_an_exited_shell_stops_nothing_and_launches() { + local dir out rc gen head_before + dir=$(new_case exited rl40) + add_ship_task "$dir" rl40 claude + # The worker exited: the endpoint is a plain shell, not the harness. + printf 'zsh' > "$dir/fake/command" + # Its last busy report never got a settling counterpart, so the durable + # record still says busy long after the process producing it went away. + gen=$("$ROOT/bin/fm-busy-event.sh" arm "$dir/home/state" rl40) + printf 'busy_gen=%s\n' "$gen" >> "$dir/home/state/rl40.meta" + "$ROOT/bin/fm-busy-event.sh" apply "$dir/home/state" rl40 busy \ + --gen "$gen" --source pi-ext --event agent-start >/dev/null + # Real uncommitted work the replacement must inherit untouched. + printf 'half-finished\n' > "$dir/wt/scratch.txt" + head_before=$(git -C "$dir/wt" rev-parse HEAD) + + out=$(run_control "$dir" rl40 relaunch --note "prior worker exited after the PR went green"); rc=$? + expect_code 0 "$rc" "a relaunch over an exited shell should launch the replacement"$'\n'"$out" + assert_contains "$out" "relaunched rl40" "the outcome should name the completed transition" + [ "$(journal_field "$dir" rl40 exit_result)" = already-stopped ] \ + || fail "an already-exited agent should be recorded as already-stopped, not stopped" + # The keys log also carries the replacement launch's own input, so pin the + # interrupt key itself rather than emptiness. + case "$(cat "$dir/fake/keys")" in + *Escape*) fail "stale busy evidence must not deliver an interrupt key into a pane with no agent" ;; + esac + # Whole-line matches: the launch literal legitimately carries paths. + if grep -qxE '/(exit|quit)' "$dir/fake/literal"; then + fail "an already-exited agent must not be sent the harness's exit command" + fi + [ "$(git -C "$dir/wt" rev-parse HEAD)" = "$head_before" ] \ + || fail "relaunch must not move the task's branch" + [ "$(git -C "$dir/wt" rev-parse --abbrev-ref HEAD)" = "task-rl40" ] \ + || fail "relaunch must not change the task's branch identity" + [ "$(cat "$dir/wt/scratch.txt")" = half-finished ] \ + || fail "relaunch must leave every uncommitted change exactly as the previous worker left it" + [ "$(journal_field "$dir" rl40 worktree_dirty)" = yes ] \ + || fail "the checkpoint should record that there was uncommitted work to preserve" + [ "$(journal_field "$dir" rl40 worktree_head)" = "$head_before" ] \ + || fail "the checkpoint should record the exact head it preserved" + assert_grep "prior worker exited after the PR went green" "$dir/home/data/rl40/brief.md" \ + "the replacement must be told what happened" + pass "fm-control relaunch: an endpoint whose agent already exited is replaced without an interrupt, without the exit command, and without touching the branch or its uncommitted work" +} + +test_relaunch_over_an_exited_shell_preserves_work_when_the_launch_fails() { + local dir out rc head_before + dir=$(new_case exitedfail rl41) + add_ship_task "$dir" rl41 claude + printf 'zsh' > "$dir/fake/command" + # The replacement never comes up: `becomes` leaves the endpoint a shell. + printf 'zsh' > "$dir/fake/becomes" + printf 'half-finished\n' > "$dir/wt/scratch.txt" + head_before=$(git -C "$dir/wt" rev-parse HEAD) + + out=$(run_control "$dir" rl41 relaunch --note "prior worker exited"); rc=$? + expect_code 1 "$rc" "a replacement that never comes up must fail loudly" + assert_contains "$out" "did not come up" "the failure should name the missing replacement" + [ "$(meta_field "$dir" rl41 harness)" = claude ] \ + || fail "a failed launch must leave the durable record naming a harness, not a half-transition" + [ "$(git -C "$dir/wt" rev-parse HEAD)" = "$head_before" ] \ + || fail "a failed relaunch must not move the task's branch" + [ "$(git -C "$dir/wt" rev-parse --abbrev-ref HEAD)" = "task-rl41" ] \ + || fail "a failed relaunch must not change the task's branch identity" + [ "$(cat "$dir/wt/scratch.txt")" = half-finished ] \ + || fail "a failed relaunch must still preserve every uncommitted change" + assert_grep "prior worker exited" "$dir/home/state/rl41.control-relaunch.note" \ + "the progress note must survive a failed relaunch for the next recovery" + pass "fm-control relaunch: a replacement that never comes up still preserves the branch, the uncommitted work, and the progress note" +} + # --- 2. harness switch ------------------------------------------------------- test_harness_switch_moves_the_record_and_clears_prior_wiring() { @@ -1483,6 +1562,8 @@ test_relaunch_serializes_concurrent_durable_metadata_publication test_disabled_relaunch_clears_prior_trace_context test_relaunch_appends_the_progress_note_to_the_instructions test_relaunch_requires_a_note_for_a_ship_task +test_relaunch_over_an_exited_shell_stops_nothing_and_launches +test_relaunch_over_an_exited_shell_preserves_work_when_the_launch_fails test_harness_switch_moves_the_record_and_clears_prior_wiring test_harness_switch_does_not_carry_the_old_profile_axes test_harness_switch_resolves_a_prefixed_recorded_harness diff --git a/tests/fm-crewmate-autonomy-live-e2e.test.sh b/tests/fm-crewmate-autonomy-live-e2e.test.sh new file mode 100755 index 00000000000..a4856a9de1d --- /dev/null +++ b/tests/fm-crewmate-autonomy-live-e2e.test.sh @@ -0,0 +1,538 @@ +#!/usr/bin/env bash +# tests/fm-crewmate-autonomy-live-e2e.test.sh - opt-in live guard proving that a +# crewmate launched under the autonomy flags bin/fm-spawn.sh actually passes +# today can (a) reach its composer unattended and (b) append to its own status +# file, which lives OUTSIDE its worktree. +# +# Why this file exists: the status append is the load-bearing crewmate contract. +# bin/fm-brief.sh hands every worker `echo "{state}: ..." >> $FM_HOME/state/.status` +# as the ONLY way to report done, blocked, or needs-decision, and that path is +# outside the worktree the harness sandboxes to. Whether a sandboxed, non- +# bypassing launch posture still permits it is a property of the harness vendor's +# sandbox and approval classifier, not of firstmate's code, so no stub and no +# transcribed flag table can answer it. Only a real agent, driven for real, can. +# +# The readings are taken and reported separately, because they fail for different +# reasons and not all of them are a firstmate defect: +# 1. the unattended-launch reading is taken with NO key sent at all, so a +# harness that parks behind a trust, approval, or update modal is recorded +# as exactly that rather than being typed into (see reach_composer); +# 2. the write reading then costs ONE short turn, in which the agent is asked +# to run the very appends bin/fm-brief.sh hands it and to answer with a +# separate token, so reply-without-write is distinguishable from no-reply; +# 3. that same turn also asks for an append to a path the launch grants nothing +# for, so a pass proves the grant is NARROW rather than merely sufficient. +# A sandboxed harness that lands that write has had its sandbox widened past +# the brief and FAILS here; a harness launched with no sandbox flag at all +# has nothing to have widened, so its result there is only reported. +# +# The granted set is exactly what bin/fm-brief.sh permits a worker outside its +# worktree - state/.status for every kind, and data//report.md for a +# scout - so the launch posture quoted below grants those two directories and +# nothing else, and the guard exercises both of them. +# +# The lab home is created under $HOME and NEVER under TMPDIR or /tmp. That is not +# cosmetic: codex's `workspace-write` sandbox grants /tmp and $TMPDIR as writable +# roots by default, so a status file placed there would be writable for reasons +# that have nothing to do with the crewmate contract and this guard would pass +# while proving nothing. A real FM_HOME lives beside the firstmate checkout under +# $HOME, and the lab mirrors that shape. +# +# Unlike tests/fm-harness-liveness-drift-live-e2e.test.sh and +# tests/fm-herdr-agent-free-proof-live-e2e.test.sh, this guard DOES submit a +# prompt and therefore does consume model tokens - one short turn per installed +# harness. That cost is unavoidable: a shell command run outside the agent proves +# nothing about the agent's own sandbox. +# +# Standard CI has neither harness binaries nor credentials, so this is opt-in and +# on-demand. Run it after any harness upgrade, after any change to a launch +# template in bin/fm-spawn.sh, and before trusting the refreshed evidence in +# docs/verification/runtime-backends.md "Crewmate autonomy and the status-file +# write contract". +# +# Always runs on a private, named, throwaway Herdr lab session, never the default +# one (tests/herdr-test-safety.sh). +set -u + +if [ "${FM_CREWMATE_AUTONOMY_LIVE:-0}" != 1 ]; then + echo "skip: set FM_CREWMATE_AUTONOMY_LIVE=1 to run the live crewmate autonomy and status-write guard" + exit 0 +fi + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +fail() { printf 'not ok - %s\n' "$1" >&2; cleanup_all; exit 1; } +pass() { printf 'ok - %s\n' "$1"; } +note() { printf '# %s\n' "$1"; } + +command -v herdr >/dev/null 2>&1 || fail "herdr not found" +command -v jq >/dev/null 2>&1 || fail "jq not found (required by the herdr adapter)" +command -v git >/dev/null 2>&1 || fail "git not found" + +# shellcheck source=tests/herdr-test-safety.sh +. "$ROOT/tests/herdr-test-safety.sh" +herdr_forget_inherited_pane + +SESSION="fm-lab-autonomy-$$" +export HERDR_SESSION="$SESSION" +LAB= +CLEANED=0 +cleanup_all() { + [ "$CLEANED" = 0 ] || return 0 + CLEANED=1 + [ -n "$LAB" ] && rm -rf "$LAB" + herdr_safe_stop_and_delete "$SESSION" +} +trap cleanup_all EXIT +fm_herdr_lab_prepare "$SESSION" || fail "could not prepare isolated Herdr lab session" + +# See the header: $HOME, never TMPDIR, or codex's default writable roots make the +# write measurement vacuous. +LAB="$HOME/.fm-crewmate-autonomy-lab-$$" +# denied/ is created up front on purpose: an append into a directory that does not +# exist fails for a reason that has nothing to do with the sandbox, which would +# turn the narrowness reading into a false pass. +mkdir -p "$LAB/state" "$LAB/wt" "$LAB/denied" || fail "could not create the lab home at $LAB" +LAB=$(cd "$LAB" && pwd -P) +# A real crewmate is launched in a git worktree, and cursor's --workspace wants a +# repository root; give the lab worktree the same shape. +git -C "$LAB/wt" init -q >/dev/null 2>&1 || fail "could not initialise the lab worktree" + +# Label the lab's Herdr workspace from the lab home, so it can never collide with +# the captain's live per-home workspace. +export FM_HOME="$LAB" + +# shellcheck source=/dev/null +. "$ROOT/bin/fm-backend.sh" +# shellcheck source=/dev/null +. "$ROOT/bin/fm-cursor-lib.sh" +# Sourced for fm_busy_cursor_project_dir: the added writable roots must not move +# the workspace identity cursor's busy fold is bound to (see the cursor branch of +# the write reading below). +# shellcheck source=/dev/null +. "$ROOT/bin/fm-busy-lib.sh" +fm_backend_source herdr || fail "fm_backend_source herdr failed" + +CONTAINER_RAW=$(fm_backend_herdr_container_ensure "$LAB/wt") || fail "container_ensure failed" +CONTAINER=${CONTAINER_RAW%%$'\t'*} +SEEDED_TAB_ID=${CONTAINER_RAW#*$'\t'} + +HERDR_VERSION=$(herdr --version 2>/dev/null | head -1 | tr -d '\r') +[ -n "$HERDR_VERSION" ] || HERDR_VERSION=unknown +note "herdr: $HERDR_VERSION" +note "lab home: <\$HOME>/${LAB##*/} (status files outside the lab worktree)" + +# Mirror bin/fm-spawn.sh's own binary resolution, so this guard launches the same +# binary firstmate would. +resolve_harness_binary() { # + local harness=$1 candidate + # cursor first, and never through a bare PATH lookup: it installs as + # `cursor-agent` plus the legacy alias `agent`, while a `cursor` on PATH is + # routinely the editor launcher rather than the agent, which answers a + # --trust launch with an Electron warning and exits. fm_cursor_resolve_binary + # is the verified owner fm-spawn itself uses. + if [ "$harness" = cursor ]; then + fm_cursor_resolve_binary 2>/dev/null && return 0 + return 1 + fi + candidate=$(command -v "$harness" 2>/dev/null || true) + if [ -n "$candidate" ] && [ -x "$candidate" ]; then + printf '%s\n' "$candidate" + return 0 + fi + return 1 +} + +# The autonomy posture under measurement, quoted from bin/fm-spawn.sh's launch +# templates with only the positional brief removed. Anything else would measure a +# posture firstmate does not ship. +# $4 and $5 are the two brief-permitted directories bin/fm-spawn.sh grants through +# its __ADDDIRS__ placeholder, passed here in the scout shape (both grants), which +# is the widest posture firstmate ships. +launch_command() { # + case "$1" in + claude) printf '%s' "CLAUDE_CODE_ENABLE_PROMPT_SUGGESTION=false $2 --permission-mode auto" ;; + codex) printf '%s' "$2 -s workspace-write -a never --add-dir $4 --add-dir $5" ;; + cursor) printf '%s' "env -u CLAUDECODE -u PI_CODING_AGENT -u GROK_AGENT -u FM_PI_HARNESS -u CURSOR_INVOKED_AS $2 --trust --auto-review --sandbox enabled --add-dir $4 --add-dir $5 --workspace $3" ;; + *) return 1 ;; + esac +} + +# harness_is_sandboxed: does this harness's launch posture carry a sandbox that the +# grant is supposed to stay narrow inside? Only such a harness is FAILED for +# landing a write outside the granted set. claude's --permission-mode auto is an +# approval classifier, not a filesystem sandbox, so a write it allows outside the +# granted set is a property of that posture rather than a widened grant. +harness_is_sandboxed() { # + case "$1" in + codex|cursor) return 0 ;; + *) return 1 ;; + esac +} + +foreground_names() { # + fm_backend_herdr_cli "$SESSION" pane process-info --pane "$1" 2>/dev/null \ + | jq -r '[.result.process_info.foreground_processes[]? | "\(.name)/\(.argv0 // .argv[0] // "")"] | join(" ")' 2>/dev/null +} + +foreground_pids() { # + fm_backend_herdr_cli "$SESSION" pane process-info --pane "$1" 2>/dev/null \ + | jq -r '.result.process_info.foreground_processes[]?.pid | select(type == "number") | floor' 2>/dev/null +} + +pane_tail() { # + fm_backend_herdr_capture "$SESSION:$1" "$2" 2>/dev/null | tr '\n' '|' +} + +# end_harness: stop only the processes this guard launched into the pane, leaving +# the pane and its tab in place, so the next harness still has its container. +end_harness() { # + local pid + for pid in $(foreground_pids "$1"); do + kill -TERM "$pid" 2>/dev/null || true + done + for _ in $(seq 1 100); do + case "$(foreground_names "$1")" in + ''|*sh/*sh*) return 0 ;; + esac + sleep 0.1 + done + for pid in $(foreground_pids "$1"); do + kill -KILL "$pid" 2>/dev/null || true + done +} + +# numbered_menu_index / numbered_menu_selected: read a vendor TUI menu of the +# shape "> 1. Yes, continue" / " 2. No, quit" by its own printed labels rather +# than by a hardcoded keystroke count, so a vendor that reorders or inserts an +# option moves the guard's cursor with it instead of silently changing what the +# guard answers. +numbered_menu_index() { # -> index + printf '%s\n' "$1" | sed -n 's/^[^0-9]*\([0-9]\)\. *\(.*\)$/\1 \2/p' \ + | grep -i -- "$2" | head -1 | cut -d' ' -f1 +} + +numbered_menu_selected() { # -> index of the currently highlighted row + # The highlighted row is the one carrying a selection marker before its + # number; every other row is indented with whitespace only. Matching "some + # non-space, non-digit character before the digit" covers the markers the + # vendors actually print (>, U+203A, U+276F) without pinning any of them. + printf '%s\n' "$1" | grep -E '^[[:space:]]*[^[:space:][:digit:]][[:space:]]*[0-9]+\.' \ + | head -1 | sed -E 's/^[^0-9]*([0-9]+)\..*/\1/' +} + +select_numbered_option() { # + local target=$1 screen=$2 pattern=$3 want cur step + want=$(numbered_menu_index "$screen" "$pattern") + cur=$(numbered_menu_selected "$screen") + case "$want$cur" in ''|*[!0-9]*) return 1 ;; esac + step=$cur + while [ "$step" -lt "$want" ]; do + fm_backend_herdr_send_key "$target" Down || return 1 + sleep 0.3 + step=$((step + 1)) + done + while [ "$step" -gt "$want" ]; do + fm_backend_herdr_send_key "$target" Up || return 1 + sleep 0.3 + step=$((step - 1)) + done + fm_backend_herdr_send_key "$target" Enter +} + +# composer_settled: an empty composer that STAYS empty for a whole settle window. +# +# A single empty verdict is NOT safe to act on. Observed live: codex 0.150.1 +# rendered a classifiable empty composer and raised its update-available modal a +# moment later, and a run of this guard that trusted the earlier read submitted +# into that modal - whose preselected row is "Update now" - and upgraded the +# codex CLI on the machine. Every key this guard sends into a live harness is +# gated on a composer that is still a composer for the whole window, which is +# what makes a late modal impossible to race. The classifier alone is the test: +# a modal screen classifies pending or unknown, never empty, and unlike a text +# signature it cannot be fooled by a dismissed modal still sitting in scrollback. +COMPOSER_SETTLE_SECONDS=${FM_CREWMATE_AUTONOMY_SETTLE_SECONDS:-10} +composer_settled() { # + local target=$1 deadline=$((SECONDS + COMPOSER_SETTLE_SECONDS)) + while [ "$SECONDS" -lt "$deadline" ]; do + [ "$(fm_backend_composer_state herdr "$target" 2>/dev/null)" = empty ] || return 1 + sleep 1 + done + return 0 +} + +# reach_composer: drive to a proven-empty composer WITHOUT ever answering a +# question the guard cannot read, and set READY_VIA to how it got there: +# +# unattended - empty with no key sent at all; this is the guarantee under test +# - empty only after the listed startup gates were cleared, each one +# named in READY_VIA in the order it appeared +# blocked - never empty; the caller reports the pane instead of guessing +# +# Two rules keep this honest. First, phase one sends NOTHING, because typing into +# a modal answers it: an earlier run of this guard submitted its prompt into +# Claude's trust dialog, selected "No, exit", and destroyed the reading. Second, +# the only gate this guard ACCEPTS is the first-launch directory-trust prompt, +# which .agents/skills/harness-adapters/SKILL.md already tells the captain to +# accept by hand after a spawn; every other gate is DECLINED by its own printed +# decline option, so the guard never widens the trust the launch flags grant and +# never measures a posture more permissive than the one firstmate ships. An +# unrecognised trust-shaped prompt is refused outright rather than guessed at. +# +# tests/fm-composer-matrix-live-e2e.test.sh takes the narrower path of declining +# every modal, because an empty composer is all it needs; this guard has to get +# past the prompt to measure the write contract behind it. +READY_VIA=blocked +# Wall-clock, not a poll count: one composer read is a full pane fetch, so a +# fixed iteration budget silently becomes minutes. +REACH_ROUND_SECONDS=${FM_CREWMATE_AUTONOMY_REACH_SECONDS:-40} +REACH_ROUNDS=6 +reach_composer() { # + local harness=$1 pane=$2 target="$SESSION:$2" round deadline verdict screen step + READY_VIA= + for round in $(seq 1 "$REACH_ROUNDS"); do + deadline=$((SECONDS + REACH_ROUND_SECONDS)) + while [ "$SECONDS" -lt "$deadline" ]; do + verdict=$(fm_backend_composer_state herdr "$target" 2>/dev/null) + if [ "$verdict" = empty ] && composer_settled "$target"; then + [ -n "$READY_VIA" ] || READY_VIA=unattended + return 0 + fi + sleep 0.2 + done + [ "$round" -lt "$REACH_ROUNDS" ] || break + screen=$(fm_backend_herdr_capture "$target" 60 2>/dev/null) + step= + if printf '%s\n' "$screen" | grep -qF 'Yes, I trust this folder'; then + # Claude Code's workspace-trust prompt: an UNNUMBERED menu whose accept row + # sits directly below the preselected "No, exit". + fm_backend_herdr_send_key "$target" Down || return 1 + sleep 0.3 + fm_backend_herdr_send_key "$target" Enter || return 1 + step=accepted-workspace-trust + elif printf '%s\n' "$screen" | grep -qiF 'trust the contents of this directory'; then + select_numbered_option "$target" "$screen" 'yes, continue' || return 1 + step=accepted-directory-trust + elif printf '%s\n' "$screen" | grep -qiF 'Hooks need review'; then + # Declined, never "Trust all": letting hooks run would grant the measured + # agent capabilities the launch flags never asked for. + select_numbered_option "$target" "$screen" 'continue without trusting' || return 1 + step=declined-hook-trust + elif printf '%s\n' "$screen" | grep -qi 'update available'; then + # Escape, never Enter: Enter on codex's update dialog RUNS the upgrade. + fm_backend_herdr_send_key "$target" Escape || return 1 + step=dismissed-update-offer + elif printf '%s\n' "$screen" | grep -qi 'trust'; then + # An unrecognised trust-shaped prompt. Never guess at it. + return 1 + else + fm_backend_herdr_send_key "$target" Escape || return 1 + step=dismissed-startup-modal + fi + note "$harness: $step" + READY_VIA="${READY_VIA:+$READY_VIA,}$step" + sleep 1 + done + READY_VIA=blocked + return 1 +} + +CHECKED=0 +SKIPPED= +DENIED= +WIDENED= +BINDING= +BLOCKED= + +# The three adapters whose launch templates carry an explicit autonomy or sandbox +# posture. An adapter that gains one belongs here too. +for harness in claude codex cursor; do + if ! bin_path=$(resolve_harness_binary "$harness"); then + SKIPPED="$SKIPPED $harness" + note "skip: $harness is not installed on this machine, so its autonomy posture is unverified here" + continue + fi + + version=$("$bin_path" --version 2>/dev/null | head -1 | tr -d '\r') || version= + [ -n "$version" ] || version="unknown" + + TASK_IDS=$(fm_backend_herdr_create_task "$CONTAINER" "fm-auto-$harness" "$LAB/wt" "$SEEDED_TAB_ID") \ + || fail "$harness ($version): could not create a lab tab" + SEEDED_TAB_ID= + read -r _TAB_ID PANE_ID <&2 + end_harness "$PANE_ID" + continue + fi + case "$READY_VIA" in + unattended) + pass "unattended launch: $harness $version reaches an empty composer with no key sent" + ;; + *) + note "$harness $version: NOT unattended - reached its composer only via $READY_VIA (see the record in docs/verification/runtime-backends.md)" + ;; + esac + + TOKEN="FMWROTE${harness}$$$RANDOM" + # A DISTINCT reply token: the write token also appears in whatever tool-call + # transcript the harness renders, so counting it could confirm a reply the + # harness never gave. + REPLY_TOKEN="FMREPLY${harness}$$$RANDOM" + # The two writes the brief permits, at exactly the places it permits them: + # outside the worktree, under the crewmate's FM_HOME. + STATUS_FILE="$STATE_DIR/$harness-$$.status" + REPORT_FILE="$REPORT_DIR/report.md" + # The third write is permitted nowhere. It sits under the SAME lab home as the + # two granted ones, so a harness that lands it has been given the home rather + # than the two directories, which is precisely the over-grant this guard exists + # to catch. + DENIED_FILE="$LAB/denied/$harness-$$.status" + # The prompt asks for each result to be reported back, not just attempted, + # because an agent that simply answers without touching its shell is + # indistinguishable from an agent the sandbox refused, and both are wrong + # answers here. Demanding OK or the exact error text per step forces the tool + # calls and makes "refused" and "never tried" tell themselves apart in the pane. + # It also states that no directory may be created, so a refusal can never be an + # artefact of a missing parent. + PROMPT="Use your shell tool for each of these three steps in order, and do not create any directory. Step 1: append the line done: $TOKEN to the file $STATUS_FILE Step 2: append the line report: $TOKEN to the file $REPORT_FILE Step 3: append the line denied: $TOKEN to the file $DENIED_FILE Then reply with only $REPLY_TOKEN followed by three lines, one per step in order, each being either OK or the exact error text. Do not stop early: attempt all three steps even if an earlier one fails." + + # Last check before the only Enter this guard ever sends into a live harness. + composer_settled "$TARGET" || fail \ + "$harness $version: the composer stopped being a composer between the readiness gate and the submit, so nothing was sent. Pane tail: $(pane_tail "$PANE_ID" 60)" + + verdict=$(fm_backend_herdr_send_text_submit "$TARGET" "$PROMPT" 3 0.4 0.4) \ + || fail "$harness $version: send_text_submit failed to run" + + replied=0 + wrote=0 + wrote_report=0 + for _ in $(seq 1 180); do + [ "$wrote" = 1 ] || { grep -qF "done: $TOKEN" "$STATUS_FILE" 2>/dev/null && wrote=1; } + [ "$wrote_report" = 1 ] || { grep -qF "report: $TOKEN" "$REPORT_FILE" 2>/dev/null && wrote_report=1; } + if [ "$replied" = 0 ]; then + # The reply token appears once in the submitted prompt; a second + # occurrence is the harness's own reply. + occurrences=$(fm_backend_herdr_capture "$TARGET" 200 2>/dev/null | grep -F -c "$REPLY_TOKEN" || true) + [ "${occurrences:-0}" -ge 2 ] && replied=1 + fi + [ "$wrote" = 1 ] && [ "$wrote_report" = 1 ] && [ "$replied" = 1 ] && break + sleep 1 + done + # Read the denied path only after the turn has settled, and never as a loop exit + # condition: the absence of a write is not something to wait for. + leaked=0 + grep -qF "denied: $TOKEN" "$DENIED_FILE" 2>/dev/null && leaked=1 + + tail_evidence=$(pane_tail "$PANE_ID" 60) + + [ "$replied" = 1 ] || fail \ + "$harness $version: reached its composer ($READY_VIA) but never answered a one-line prompt within 180s, so the write contract behind it could not be measured. Submit verdict '$verdict'. Pane tail: $tail_evidence" + + note "$harness $version: ready=$READY_VIA submit=$verdict reply=landed" + + if [ "$wrote" != 1 ]; then + DENIED="$DENIED $harness" + printf 'not ok - %s\n' \ + "CREWMATE WRITE CONTRACT: $harness $version answered a prompt but could NOT append to its own status file outside its worktree ($STATUS_FILE). bin/fm-brief.sh makes that append the only way a worker reports done, blocked, or needs-decision, so under this launch posture such a worker is mute. Pane tail: $tail_evidence" >&2 + elif [ "$wrote_report" != 1 ]; then + DENIED="$DENIED $harness" + printf 'not ok - %s\n' \ + "CREWMATE WRITE CONTRACT: $harness $version appends its status file but could NOT write a scout report at $REPORT_FILE. bin/fm-brief.sh makes that file a scout's entire deliverable, so a scout on this posture cannot deliver. Pane tail: $tail_evidence" >&2 + else + pass "crewmate write contract: $harness $version appends its status file and writes its scout report, both outside its worktree" + fi + + # cursor only, and a direct consequence of granting it extra roots: + # state/.cursor-session binds the busy fold to the ONE cursor project whose + # recorded workspacePath is EXACTLY the task worktree (bin/fm-busy-lib.sh, exact + # match by design). If an added root moved that recorded identity, every cursor + # worker's turn state would go unreadable while every other reading here still + # passed, so the grant is only safe if this holds. + if [ "$harness" = cursor ]; then + if fm_busy_cursor_project_dir "${CURSOR_PROJECTS_ROOT_OVERRIDE:-$HOME/.cursor/projects}" "$LAB/wt" >/dev/null 2>&1; then + pass "workspace binding: cursor $version still records the task worktree as its exact workspacePath under the added writable roots" + else + BINDING="$BINDING $harness" + printf 'not ok - %s\n' \ + "WORKSPACE BINDING LOST: cursor $version no longer records $LAB/wt as the exact workspacePath of any project under ${CURSOR_PROJECTS_ROOT_OVERRIDE:-\$HOME/.cursor/projects}, so bin/fm-busy-lib.sh cannot bind a cursor worker's transcript and its turn state reads unknown forever. The added writable roots moved the workspace identity." >&2 + fi + fi + + if harness_is_sandboxed "$harness"; then + if [ "$leaked" = 1 ]; then + WIDENED="$WIDENED $harness" + printf 'not ok - %s\n' \ + "SANDBOX GRANT TOO WIDE: $harness $version landed a write at $DENIED_FILE, which the launch grants nothing for. The posture therefore hands a crewmate its whole firstmate home rather than the two directories bin/fm-brief.sh permits. Pane tail: $tail_evidence" >&2 + else + pass "narrow grant: $harness $version is still refused a write outside the two brief-permitted directories" + fi + elif [ "$leaked" = 1 ]; then + note "$harness $version: carries no sandbox flag, and did write outside the brief-permitted directories ($DENIED_FILE); its posture is an approval classifier, not a filesystem boundary" + else + note "$harness $version: carries no sandbox flag, and did not write outside the brief-permitted directories" + fi + + CHECKED=$((CHECKED + 1)) + end_harness "$PANE_ID" +done + +[ "$CHECKED" -gt 0 ] || fail \ + "no measured harness is installed here, so this run proved nothing; install at least one of claude, codex, or cursor before trusting a pass" + +if [ -n "$SKIPPED" ]; then + note "unverified on this machine (not installed):$SKIPPED" +fi +note "checked $CHECKED installed harness(es) on herdr $HERDR_VERSION" + +if [ -n "$BLOCKED" ]; then + fail "the launch posture bin/fm-spawn.sh ships parks behind an unrecognised blocker for:$BLOCKED" +fi + +if [ -n "$DENIED" ]; then + fail "the crewmate write contract is BROKEN under today's launch posture for:$DENIED" +fi + +if [ -n "$WIDENED" ]; then + fail "today's launch posture grants more than bin/fm-brief.sh permits for:$WIDENED" +fi + +if [ -n "$BINDING" ]; then + fail "the launch posture's writable roots broke the workspace identity the busy fold binds to for:$BINDING" +fi + +cleanup_all +trap - EXIT diff --git a/tests/fm-herdr-agent-free-proof-live-e2e.test.sh b/tests/fm-herdr-agent-free-proof-live-e2e.test.sh new file mode 100755 index 00000000000..68db30b40d9 --- /dev/null +++ b/tests/fm-herdr-agent-free-proof-live-e2e.test.sh @@ -0,0 +1,220 @@ +#!/usr/bin/env bash +# tests/fm-herdr-agent-free-proof-live-e2e.test.sh - opt-in drift guard proving +# no INSTALLED harness, while genuinely running in a Herdr pane, can be +# mistaken for an agent-free bare idle shell. +# +# Why this file exists: the Herdr liveness classifier settles its negative +# verdict from `pane process-info` - the pane must provably hold one lone bare +# idle shell before a reported registration is treated as stale +# (bin/backends/herdr.sh, fm_backend_herdr_pane_agent_state). That proof reads +# a process name and an argv0 that each harness vendor controls and can change +# without notice, so a stub can only confirm the assumption already written +# into the stub. A false agent-free verdict is the one outcome that could +# launch a replacement onto a live worker's local copy, which is exactly the +# direction this guard watches. +# +# The portable counterpart in tests/fm-backend-herdr.test.sh pins the +# classifier logic in CI with real processes and no harness, and +# tests/fm-control-herdr-smoke.test.sh pins the control plane's use of it on +# the required real-Herdr lane. Run this guard after any harness upgrade and +# before trusting refreshed per-harness evidence. +# +# Each harness is launched bare, with no prompt, so this consumes no model +# tokens. Standard CI has neither harness binaries nor credentials, so it is +# opt-in and on-demand. +# +# Always runs on a private, named, throwaway lab session, never the default one +# (tests/herdr-test-safety.sh). +set -u + +if [ "${FM_HERDR_AGENT_FREE_PROOF:-0}" != 1 ]; then + echo "skip: set FM_HERDR_AGENT_FREE_PROOF=1 to run the installed-harness Herdr agent-free proof guard" + exit 0 +fi + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +fail() { printf 'not ok - %s\n' "$1" >&2; cleanup_all; exit 1; } +pass() { printf 'ok - %s\n' "$1"; } +note() { printf '# %s\n' "$1"; } + +command -v herdr >/dev/null 2>&1 || fail "herdr not found" +command -v jq >/dev/null 2>&1 || fail "jq not found (required by the herdr adapter)" + +# shellcheck source=tests/herdr-test-safety.sh +. "$ROOT/tests/herdr-test-safety.sh" +herdr_forget_inherited_pane + +SESSION="fm-lab-agent-free-$$" +export HERDR_SESSION="$SESSION" +LAB= +CLEANED=0 +cleanup_all() { + [ "$CLEANED" = 0 ] || return 0 + CLEANED=1 + [ -n "$LAB" ] && rm -rf "$LAB" + herdr_safe_stop_and_delete "$SESSION" +} +trap cleanup_all EXIT +fm_herdr_lab_prepare "$SESSION" || fail "could not prepare isolated Herdr lab session" + +LAB=$(mktemp -d "${TMPDIR:-/tmp}/fm-herdr-agent-free.XXXXXX") +LAB=$(cd "$LAB" && pwd) +mkdir -p "$LAB/wt" + +# shellcheck source=/dev/null +. "$ROOT/bin/fm-backend.sh" +# shellcheck source=/dev/null +. "$ROOT/bin/fm-cursor-lib.sh" +fm_backend_source herdr || fail "fm_backend_source herdr failed" + +CONTAINER_RAW=$(fm_backend_herdr_container_ensure "$LAB/wt") || fail "container_ensure failed" +CONTAINER=${CONTAINER_RAW%%$'\t'*} +SEEDED_TAB_ID=${CONTAINER_RAW#*$'\t'} +WORKSPACE_ID=${CONTAINER#*:} + +HERDR_VERSION=$(herdr --version 2>/dev/null | head -1 | tr -d '\r') +[ -n "$HERDR_VERSION" ] || HERDR_VERSION=unknown +note "herdr: $HERDR_VERSION" + +# Mirror bin/fm-spawn.sh's own binary resolution, so this guard covers the same +# binary firstmate would actually launch. +resolve_harness_binary() { # + local harness=$1 candidate + # cursor first, and never through a bare PATH lookup: it installs as + # `cursor-agent` plus the legacy alias `agent`, while an unrelated `cursor` + # on PATH is routinely the editor launcher rather than the agent (observed on + # a developer machine, where it answered a `--trust` launch with an Electron + # warning and exited). fm_cursor_resolve_binary is the verified owner + # fm-spawn itself uses, so this guard launches exactly what firstmate would. + if [ "$harness" = cursor ]; then + fm_cursor_resolve_binary 2>/dev/null && return 0 + return 1 + fi + candidate=$(command -v "$harness" 2>/dev/null || true) + if [ -n "$candidate" ] && [ -x "$candidate" ]; then + printf '%s\n' "$candidate" + return 0 + fi + if [ "$harness" = kimi ] && [ -n "${HOME:-}" ] && [ -x "$HOME/.kimi-code/bin/kimi" ]; then + printf '%s\n' "$HOME/.kimi-code/bin/kimi" + return 0 + fi + return 1 +} + +foreground_names() { # + fm_backend_herdr_cli "$SESSION" pane process-info --pane "$1" 2>/dev/null \ + | jq -r '[.result.process_info.foreground_processes[]? | "\(.name)/\(.argv0 // .argv[0] // "")"] | join(" ")' 2>/dev/null +} + +foreground_pids() { # + fm_backend_herdr_cli "$SESSION" pane process-info --pane "$1" 2>/dev/null \ + | jq -r '.result.process_info.foreground_processes[]?.pid | select(type == "number") | floor' 2>/dev/null +} + +# end_harness: stop only the exact processes this guard launched into the pane, +# leaving the pane and its tab in place. Closing the pane instead would remove +# the workspace's last tab and destroy the container the next harness needs. +end_harness() { # + local pid + for pid in $(foreground_pids "$1"); do + kill -TERM "$pid" 2>/dev/null || true + done + for _ in $(seq 1 100); do + case "$(foreground_names "$1")" in + ''|*sh/*sh*) return 0 ;; + esac + sleep 0.1 + done + for pid in $(foreground_pids "$1"); do + kill -KILL "$pid" 2>/dev/null || true + done +} + +CHECKED=0 +SKIPPED= + +# The verified adapters, in the order .agents/skills/harness-adapters/SKILL.md +# records them. An adapter that gains a verified launch path belongs here too. +for harness in claude codex opencode pi pi-signed grok kimi cursor muse; do + if ! bin_path=$(resolve_harness_binary "$harness"); then + SKIPPED="$SKIPPED $harness" + note "skip: $harness is not installed on this machine, so its Herdr classification is unverified here" + continue + fi + + version=$("$bin_path" --version 2>/dev/null | head -1 | tr -d '\r') || version= + [ -n "$version" ] || version="unknown" + + TASK_IDS=$(fm_backend_herdr_create_task "$CONTAINER" "fm-afp-$harness" "$LAB/wt" "$SEEDED_TAB_ID") \ + || fail "$harness ($version): could not create a lab tab" + SEEDED_TAB_ID= + read -r _TAB_ID PANE_ID </dev/null | tail -5 | tr '\n' '|')" + + if fm_backend_herdr_pane_idle_shell_sample "$SESSION" "$PANE_ID" >/dev/null 2>&1; then + fail "AGENT-FREE DRIFT: $harness $version is running in a Herdr pane, but the pane's process inventory proves a lone bare idle shell, so bin/backends/herdr.sh would classify a live worker as agent-free and let a replacement launch onto its local copy. Observed foreground name/argv0 pairs [$names] on herdr $HERDR_VERSION." + fi + + # Herdr registers an agent only for the harnesses it ships an integration + # for, and that integration reports on its own schedule, so give it a bounded + # window before reading the recovery-grade verdict. + state= + for _ in $(seq 1 100); do + state=$(fm_backend_agent_state herdr "$SESSION:$PANE_ID") + [ "$state" = alive ] && break + sleep 0.2 + done + case "$state" in + alive) ;; + dead|missing) + # No integration ever registered this pane. That is a pre-existing + # property of the registry, not of the process inventory this guard + # watches, so it is reported rather than failed. + note "$harness $version: herdr registered no agent for this harness within the wait (state '$state'); the agent-free proof still correctly refuses" + ;; + *) fail "$harness $version: Herdr classified a running harness '$state'; only alive or an unregistered dead/missing is expected" ;; + esac + + note "$harness $version: foreground=[$names] state=$state" + pass "herdr agent-free proof: $harness $version running in a Herdr pane never proves a bare idle shell" + CHECKED=$((CHECKED + 1)) + + end_harness "$PANE_ID" +done + +[ "$CHECKED" -gt 0 ] || fail \ + "no verified harness is installed here, so this run proved nothing; install at least one harness before trusting a pass" + +if [ -n "$SKIPPED" ]; then + note "unverified on this machine (not installed):$SKIPPED" +fi +note "checked $CHECKED installed harness(es) on herdr $HERDR_VERSION in workspace $WORKSPACE_ID" + +cleanup_all +trap - EXIT diff --git a/tests/fm-secondmate-harness.test.sh b/tests/fm-secondmate-harness.test.sh index ee8039a9c88..c2a53be2131 100755 --- a/tests/fm-secondmate-harness.test.sh +++ b/tests/fm-secondmate-harness.test.sh @@ -767,7 +767,7 @@ test_spawn_secondmate_harness_model_token() { [ "$(meta_field "$meta" model)" = opus ] || fail "model-token: meta model not opus (got '$(meta_field "$meta" model)')" [ "$(meta_field "$meta" effort)" = default ] || fail "model-token: meta effort not default (got '$(meta_field "$meta" effort)')" launch=$(cat "$launchlog") - assert_contains "$launch" "claude --dangerously-skip-permissions --model 'opus'" \ + assert_contains "$launch" "claude --permission-mode auto --model 'opus'" \ "model-token: launch did not carry --model opus" assert_not_contains "$launch" "--effort" "model-token: launch must not carry an --effort flag" pass "C3 spawn: config/secondmate-harness's model token threads --model into the launch and meta" @@ -789,7 +789,7 @@ test_spawn_secondmate_harness_model_and_effort_tokens() { [ "$(meta_field "$meta" model)" = opus ] || fail "model-effort-tokens: meta model not opus" [ "$(meta_field "$meta" effort)" = high ] || fail "model-effort-tokens: meta effort not high (got '$(meta_field "$meta" effort)')" launch=$(cat "$launchlog") - assert_contains "$launch" "claude --dangerously-skip-permissions --model 'opus' --effort 'high'" \ + assert_contains "$launch" "claude --permission-mode auto --model 'opus' --effort 'high'" \ "model-effort-tokens: launch did not carry both --model opus and --effort high" pass "C4 spawn: config/secondmate-harness's model+effort tokens thread into the launch and meta" } @@ -854,7 +854,7 @@ test_spawn_explicit_harness_does_not_inherit_secondmate_harness_tokens() { [ "$(meta_field "$meta" model)" = default ] || fail "explicit-harness-no-tokens: meta model should stay default" [ "$(meta_field "$meta" effort)" = default ] || fail "explicit-harness-no-tokens: meta effort should stay default" launch=$(cat "$launchlog") - assert_contains "$launch" "codex --dangerously-bypass-approvals-and-sandbox" \ + assert_contains "$launch" "codex -s workspace-write -a never" \ "explicit-harness-no-tokens: launch did not use codex" assert_not_contains "$launch" "--model" "explicit-harness-no-tokens: launch must not carry a --model flag" assert_not_contains "$launch" "model_reasoning_effort" \ diff --git a/tests/fm-spawn-dispatch-profile.test.sh b/tests/fm-spawn-dispatch-profile.test.sh index bf9c047d77f..bbad9fa212e 100755 --- a/tests/fm-spawn-dispatch-profile.test.sh +++ b/tests/fm-spawn-dispatch-profile.test.sh @@ -131,7 +131,7 @@ test_no_profile_keeps_claude_profile_defaults() { assert_meta_profile "$HOME_DIR/state/$id.meta" claude default default launch=$(cat "$LAUNCH_LOG") - expected="env -u CURSOR_AGENT -u CURSOR_INVOKED_AS CLAUDE_CODE_ENABLE_PROMPT_SUGGESTION=false claude --dangerously-skip-permissions \"\$('${ROOT}/bin/fm-operational-input.sh' encode launch-brief < '$HOME_DIR/data/$id/brief.md')\"" + expected="env -u CURSOR_AGENT -u CURSOR_INVOKED_AS CLAUDE_CODE_ENABLE_PROMPT_SUGGESTION=false claude --permission-mode auto \"\$('${ROOT}/bin/fm-operational-input.sh' encode launch-brief < '$HOME_DIR/data/$id/brief.md')\"" [ "$launch" = "$expected" ] || fail "no-profile claude launch did not use the canonical launch kind"$'\n'"expected: $expected"$'\n'"actual: $launch" pass "no --model/--effort records defaults and types the claude launch instructions" } @@ -345,7 +345,7 @@ test_active_dispatch_profile_allows_explicit_harness() { assert_contains "$out" "spawned $id harness=codex" "spawn did not report explicit codex harness" assert_meta_profile "$HOME_DIR/state/$id.meta" codex gpt-5 high launch=$(cat "$LAUNCH_LOG") - assert_contains "$launch" "codex --model 'gpt-5' -c 'model_reasoning_effort=\"high\"' --dangerously-bypass-approvals-and-sandbox" \ + assert_contains "$launch" "codex --model 'gpt-5' -c 'model_reasoning_effort=\"high\"' -s workspace-write -a never" \ "explicit harness launch did not thread model and effort" pass "active crew-dispatch profile allows an explicit resolved harness" } @@ -395,7 +395,7 @@ test_claude_threads_model_and_effort() { expect_code 0 "$status" "claude spawn with profile flags should succeed" assert_meta_profile "$HOME_DIR/state/$id.meta" claude sonnet high launch=$(cat "$LAUNCH_LOG") - assert_contains "$launch" "claude --dangerously-skip-permissions --model 'sonnet' --effort 'high'" \ + assert_contains "$launch" "claude --permission-mode auto --model 'sonnet' --effort 'high'" \ "claude launch did not thread model and effort flags" assert_not_contains "$launch" "--tui-mode" "non-Pi launches must not receive Pi's TUI mode override" pass "claude receives --model and --effort profile flags" @@ -412,7 +412,7 @@ test_codex_threads_model_and_effort() { expect_code 0 "$status" "codex spawn with profile flags should succeed" assert_meta_profile "$HOME_DIR/state/$id.meta" codex gpt-5 high launch=$(cat "$LAUNCH_LOG") - assert_contains "$launch" "codex --model 'gpt-5' -c 'model_reasoning_effort=\"high\"' --dangerously-bypass-approvals-and-sandbox" \ + assert_contains "$launch" "codex --model 'gpt-5' -c 'model_reasoning_effort=\"high\"' -s workspace-write -a never" \ "codex launch did not thread model and reasoning effort config" pass "codex receives --model and model_reasoning_effort profile flags" } @@ -428,12 +428,57 @@ test_codex_omits_invalid_max_effort() { expect_code 0 "$status" "codex spawn with unsupported max effort should omit the effort flag" assert_meta_profile "$HOME_DIR/state/$id.meta" codex gpt-5 max launch=$(cat "$LAUNCH_LOG") - assert_contains "$launch" "codex --model 'gpt-5' --dangerously-bypass-approvals-and-sandbox" \ + assert_contains "$launch" "codex --model 'gpt-5' -s workspace-write -a never" \ "codex launch did not preserve the model flag when max effort was omitted" assert_not_contains "$launch" "model_reasoning_effort" "codex launch must omit unsupported max reasoning effort" pass "codex omits unsupported max effort instead of passing a bad config value" } +# codex runs sandboxed with no escalation path, so anything the brief tells a +# worker to write outside its worktree has to be granted at launch or the model +# just sees "operation not permitted". These two cases pin the grant to exactly +# what bin/fm-brief.sh permits, in both directions: the paths must be there, and +# the firstmate home itself must NOT be. +test_codex_ship_grants_only_the_brief_permitted_state_dir() { + local rec id out status launch + id=profile-codex-grant-z30 + rec=$(make_spawn_case profile-codex-grant codex "$id") + read_case_record "$rec" + + out=$(run_ship_spawn "$HOME_DIR" "$WT_DIR" "$FAKEBIN_DIR" "$LAUNCH_LOG" "$id" "$PROJ_DIR") + status=$? + expect_code 0 "$status" "codex ship spawn should succeed" + launch=$(cat "$LAUNCH_LOG") + assert_contains "$launch" "--add-dir '$HOME_DIR/state'" \ + "codex ship launch did not grant the state dir the status file lives in" + assert_not_contains "$launch" "--add-dir '$HOME_DIR/data" \ + "codex ship launch must not grant a scout report dir a crewmate never writes" + assert_not_contains "$launch" "--add-dir '$HOME_DIR'" \ + "codex launch must never grant the whole firstmate home" + pass "codex crewmate is granted exactly the state dir its status file needs" +} + +test_codex_scout_also_grants_its_own_report_dir() { + local rec id out status launch + id=profile-codex-scout-grant-z31 + rec=$(make_spawn_case profile-codex-scout-grant codex "$id") + read_case_record "$rec" + + out=$(run_spawn "$HOME_DIR" "$WT_DIR" "$FAKEBIN_DIR" "$LAUNCH_LOG" "$id" "$PROJ_DIR" --scout) + status=$? + expect_code 0 "$status" "codex scout spawn should succeed" + launch=$(cat "$LAUNCH_LOG") + assert_contains "$launch" "--add-dir '$HOME_DIR/state'" \ + "codex scout launch did not grant the state dir the status file lives in" + assert_contains "$launch" "--add-dir '$HOME_DIR/data/$id'" \ + "codex scout launch did not grant the one data dir its report lives in" + assert_not_contains "$launch" "--add-dir '$HOME_DIR/data'" \ + "codex scout launch must grant its own data dir, not every task's" + assert_not_contains "$launch" "--add-dir '$HOME_DIR'" \ + "codex launch must never grant the whole firstmate home" + pass "codex scout is granted its own report dir alongside the state dir" +} + test_grok_threads_model_and_reasoning_effort() { local rec id out status launch id=profile-grok-z5 @@ -500,7 +545,7 @@ test_cursor_threads_model_workspace_and_omits_effort_axis() { expect_code 0 "$status" "cursor spawn with a model-qualified reasoning class should succeed" assert_meta_profile "$HOME_DIR/state/$id.meta" cursor cursor-grok-4.5-high high launch=$(cat "$LAUNCH_LOG") - assert_contains "$launch" "--trust --yolo --model 'cursor-grok-4.5-high' --workspace '$WT_DIR'" \ + assert_contains "$launch" "--trust --auto-review --sandbox enabled --model 'cursor-grok-4.5-high' --workspace '$WT_DIR'" \ "cursor launch did not carry trust, autonomy, model, and exact workspace flags" # The executable is RESOLVED, never named: `cursor` is not the CLI, so a # literal `cursor agent` command cannot run on a machine that has only the @@ -806,6 +851,8 @@ test_active_dispatch_profile_allows_raw_launch_command test_claude_threads_model_and_effort test_codex_threads_model_and_effort test_codex_omits_invalid_max_effort +test_codex_ship_grants_only_the_brief_permitted_state_dir +test_codex_scout_also_grants_its_own_report_dir test_grok_threads_model_and_reasoning_effort test_grok_omits_invalid_max_reasoning_effort test_grok_omits_invalid_xhigh_reasoning_effort diff --git a/tests/fm-usage-harvest.test.sh b/tests/fm-usage-harvest.test.sh new file mode 100755 index 00000000000..f95151e7c84 --- /dev/null +++ b/tests/fm-usage-harvest.test.sh @@ -0,0 +1,455 @@ +#!/usr/bin/env bash +# Behavior tests for the fleet usage harvester and its report reader. +# Covers claude-log request dedupe and cache folding, codex per-request delta +# sums with cwd/window matching, cursor/unavailable null rows, the +# no-double-append idempotency guard, the staging file the append stages +# through leaving no debris, the ledger report rendering, and the teardown +# integration property that a harvest failure never blocks teardown. +set -u + +# shellcheck source=tests/lib.sh +# shellcheck disable=SC1091 +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +HARVEST="$ROOT/bin/fm-usage-harvest.sh" +REPORT="$ROOT/bin/fm-usage-report.sh" +TEARDOWN="$ROOT/bin/fm-teardown.sh" +TMP_ROOT=$(fm_test_tmproot fm-usage-harvest) + +command -v jq >/dev/null 2>&1 || { echo "skip: jq not found"; exit 0; } + +file_mtime_epoch() { # + local t + t=$(stat -f %m -- "$1" 2>/dev/null) || t=$(stat -c %Y -- "$1" 2>/dev/null) || return 1 + case "$t" in ''|*[!0-9]*) return 1 ;; esac + printf '%s' "$t" +} +file_birth_epoch() { # + local t + t=$(stat -f %B -- "$1" 2>/dev/null) || t=$(stat -c %W -- "$1" 2>/dev/null) || return 1 + case "$t" in ''|*[!0-9]*) return 1 ;; esac + [ "$t" -ge 1000000000 ] 2>/dev/null || return 1 + printf '%s' "$t" +} + +# harvest_case [model] [effort] : create a home with one task +# whose worktree is $TMP_ROOT/wt-, status and meta included, and echo the +# data dir. Exports the FM_USAGE_* fixture dirs per case. +harvest_case() { # [model] [effort] + local id=$1 harness=$2 wt=$3 + local home="$TMP_ROOT/home-$id" + mkdir -p "$home/state" "$home/data" + fm_write_meta "$home/state/$id.meta" \ + "window=firstmate:fm-$id" "endpoint_task_id=$id" "worktree=$wt" \ + "project=$TMP_ROOT/proj-$id" "harness=$harness" "kind=ship" "mode=no-mistakes" \ + "model=${4:-default}" "effort=${5:-default}" + { + printf 'working: started\n' + printf 'working: halfway\n' + printf 'done: finished the task\n' + } > "$home/state/$id.status" + printf '%s\n' "$home/data" +} +export_harvest_env() { # + FM_STATE_OVERRIDE="$1/state" + FM_DATA_OVERRIDE="$1/data" + FM_USAGE_CLAUDE_DIR="$TMP_ROOT/fake-claude/projects" + FM_USAGE_CODEX_DIR="$TMP_ROOT/fake-codex/sessions" + export FM_STATE_OVERRIDE FM_DATA_OVERRIDE FM_USAGE_CLAUDE_DIR FM_USAGE_CODEX_DIR +} + +# --- claude: request dedupe, cache folding, encoding resolution, window ----- + +claude_case() { + # The worktree path carries a dot segment (as every firstmate home under + # .no-mistakes does) so a slash-only encoding would resolve to the wrong + # on-disk project directory. + local id=usageclaude1 wt="$TMP_ROOT/.no-mistakes/wt-usageclaude1" + local data home state ledger row + data=$(harvest_case "$id" claude "$wt" default default) + home=$(dirname "$data") + state="$home/state" + export_harvest_env "$home" + + # Claude Code encodes the project directory by mapping BOTH '/' and '.' to + # '-', so the fixture log dir mirrors that full sanitization. + local encoded=${wt//\//-} + encoded=${encoded//./-} + local logdir="$FM_USAGE_CLAUDE_DIR/$encoded" + mkdir -p "$logdir" + cat > "$logdir/session-a.jsonl" <<'JSON' +{"type":"assistant","message":{"id":"msgA","model":"claude-test","usage":{"input_tokens":10,"cache_read_input_tokens":100,"cache_creation_input_tokens":20,"output_tokens":30,"output_tokens_details":{"thinking_tokens":5}}}} +{"type":"assistant","message":{"id":"msgA","model":"claude-test","usage":{"input_tokens":10,"cache_read_input_tokens":100,"cache_creation_input_tokens":20,"output_tokens":30,"output_tokens_details":{"thinking_tokens":5}}}} +{"type":"assistant","message":{"id":"msgB","model":"claude-test","usage":{"input_tokens":7,"cache_read_input_tokens":0,"cache_creation_input_tokens":0,"output_tokens":9}}} +{"type":"user","message":{"role":"user"}} +{"type":"assistant","message":{"id":"msgC"}} +JSON + # A request logged outside the task window (future mtime) must be excluded, + # as must a session in a differently encoded sibling directory. + cat > "$logdir/session-future.jsonl" <<'JSON' +{"type":"assistant","message":{"id":"msgX","model":"claude-test","usage":{"input_tokens":999,"cache_read_input_tokens":0,"cache_creation_input_tokens":0,"output_tokens":999}}} +JSON + touch -t "$(date -r $(( $(file_mtime_epoch "$state/$id.status") + 7200 )) +%Y%m%d%H%M.%S)" \ + "$logdir/session-future.jsonl" + mkdir -p "$FM_USAGE_CLAUDE_DIR/wrong-encoded-dir" + printf '%s\n' '{"type":"assistant","message":{"id":"msgY","model":"claude-test","usage":{"input_tokens":777,"output_tokens":777}}}' \ + > "$FM_USAGE_CLAUDE_DIR/wrong-encoded-dir/other.jsonl" + # Seal the window: the status end mtime must not predate the in-window + # session log, or the log correctly falls outside birth -> end. + touch -m -r "$logdir/session-a.jsonl" "$state/$id.status" + + out=$("$HARVEST" "$id" 2>&1) + expect_code 0 "$?" "claude harvest should succeed"$'\n'"$out" + ledger="$data/usage-ledger.jsonl" + [ -f "$ledger" ] || fail "claude harvest wrote no ledger" + [ "$(wc -l < "$ledger" | tr -d ' ')" = 1 ] || fail "claude harvest wrote more than one ledger line" + row=$(cat "$ledger") + assert_contains "$row" '"task":"usageclaude1"' "claude row names the task" + assert_contains "$row" '"harness":"claude"' "claude row names the harness" + assert_contains "$row" '"model":"claude-test"' "claude row captures the log model" + assert_contains "$row" '"source":"claude-projects"' "claude row names its source" + assert_contains "$row" '"input_tokens":17' "claude input tokens dedupe per request (10+7)" + assert_contains "$row" '"cached_input_tokens":120' "claude cached folds read+creation (100+20+0+0)" + assert_contains "$row" '"output_tokens":39' "claude output tokens (30+9)" + assert_contains "$row" '"reasoning_tokens":5' "claude reasoning captures thinking tokens" + assert_contains "$row" '"turns":2' "claude turn estimate counts working: lines" + assert_contains "$row" '"effort":null' "default effort renders null" + # Wall seconds is birth -> status mtime on this platform; assert the + # computed value equals an independent read of the same file facts. + local end birth wall_expected + end=$(file_mtime_epoch "$state/$id.status") + birth=$(file_birth_epoch "$state/$id.status" || file_mtime_epoch "$state/$id.status") + wall_expected=$((end - birth)) + [ "$wall_expected" -lt 0 ] && wall_expected=0 + assert_contains "$row" "\"wall_secs\":$wall_expected" \ + "claude wall seconds equals status birth -> last-mtime window" + + # Idempotency: a second harvest must not append a duplicate row. + out=$("$HARVEST" "$id" 2>&1) + expect_code 0 "$?" "second claude harvest should exit 0"$'\n'"$out" + [ "$(wc -l < "$ledger" | tr -d ' ')" = 1 ] \ + || fail "second claude harvest double-appended" + pass "claude harvest: per-request dedupe, cache folding, encoding, window, idempotency" +} + +# --- codex: cwd match, window match, delta sums, model capture -------------- + +codex_case() { + local id=usagecodex1 wt="$TMP_ROOT/wt-usagecodex1" + local data home ledger row out + data=$(harvest_case "$id" codex "$wt" default high) + home=$(dirname "$data") + export_harvest_env "$home" + + local d1="$FM_USAGE_CODEX_DIR/2026/08/28" + mkdir -p "$d1" + cat > "$d1/rollout-match.jsonl" < "$d1/rollout-othercwd.jsonl" < "$d1/rollout-future.jsonl" <&1) + expect_code 0 "$?" "codex harvest should succeed"$'\n'"$out" + ledger="$home/data/usage-ledger.jsonl" + row=$(cat "$ledger") + assert_contains "$row" '"harness":"codex"' "codex row names the harness" + assert_contains "$row" '"model":"glm-5.3"' "codex row captures the turn_context model" + assert_contains "$row" '"effort":"high"' "codex row carries meta effort" + assert_contains "$row" '"source":"codex-sessions"' "codex row names its source" + assert_contains "$row" '"input_tokens":150' "codex input sums per-request deltas (100+50)" + assert_contains "$row" '"cached_input_tokens":15' "codex cached folds cached+cache_write (10+5+0)" + assert_contains "$row" '"output_tokens":31' "codex output sums deltas (20+11)" + assert_contains "$row" '"reasoning_tokens":11' "codex reasoning sums deltas (8+3)" + pass "codex harvest: cwd/window matching, delta sums, model capture" +} + +# --- cursor: unavailable logs render nulls ---------------------------------- + +cursor_case() { + local id=usagecursor1 wt="$TMP_ROOT/wt-usagecursor1" + local data home ledger row out + data=$(harvest_case "$id" cursor "$wt" cursor-grok-4.5-high "") + home=$(dirname "$data") + export_harvest_env "$home" + out=$("$HARVEST" "$id" 2>&1) + expect_code 0 "$?" "cursor harvest should succeed"$'\n'"$out" + ledger="$home/data/usage-ledger.jsonl" + row=$(cat "$ledger") + assert_contains "$row" '"model":"cursor-grok-4.5-high"' "cursor row falls back to meta model" + assert_contains "$row" '"input_tokens":null' "cursor input tokens are null" + assert_contains "$row" '"cached_input_tokens":null' "cursor cached tokens are null" + assert_contains "$row" '"output_tokens":null' "cursor output tokens are null" + assert_contains "$row" '"reasoning_tokens":null' "cursor reasoning tokens are null" + assert_contains "$row" '"source":"unavailable"' "cursor row marks source unavailable" + pass "cursor harvest: unavailable source renders null token fields" +} + +# --- claude: window survives a host without usable birth time --------------- + +# A stat shim that reports no birth time (GNU statx unsupported returns 0 for +# %W) while still answering mtime, so the harvest must fall back to the meta +# mtime for the window start instead of collapsing to the status mtime. +nobirth_stat_bin() { # + mkdir -p "$1" + cat > "$1/stat" <<'SH' +#!/usr/bin/env bash +fmt=""; file="" +while [ "$#" -gt 0 ]; do + case "$1" in + -f) exit 1 ;; + -c) fmt="$2"; shift 2 ;; + --) shift; file="$1"; shift ;; + *) file="$1"; shift ;; + esac +done +case "$fmt" in + %W) printf '0\n' ;; + %Y) /usr/bin/stat -f %m -- "$file" 2>/dev/null || /usr/bin/stat -c %Y -- "$file" ;; + *) exit 1 ;; +esac +SH + chmod +x "$1/stat" +} + +claude_nobirth_case() { + local id=usagenobirth1 wt="$TMP_ROOT/.no-mistakes/wt-usagenobirth1" + local data home state ledger row out fb base + data=$(harvest_case "$id" claude "$wt" default default) + home=$(dirname "$data") + state="$home/state" + export_harvest_env "$home" + + local encoded=${wt//\//-} + encoded=${encoded//./-} + local logdir="$FM_USAGE_CLAUDE_DIR/$encoded" + mkdir -p "$logdir" + cat > "$logdir/session.jsonl" <<'JSON' +{"type":"assistant","message":{"id":"msgN","model":"claude-test","usage":{"input_tokens":12,"output_tokens":8}}} +JSON + + # Pin an explicit window: status finishes at T, meta was spawned 100s earlier, + # and the session log lands mid-window. Without the meta-mtime start fallback + # the birthless window collapses to [T, T] and drops the earlier log. + base=$(file_mtime_epoch "$state/$id.status") + touch -t "$(date -r "$base" +%Y%m%d%H%M.%S)" "$state/$id.status" + touch -t "$(date -r $((base - 100)) +%Y%m%d%H%M.%S)" "$state/$id.meta" + touch -t "$(date -r $((base - 50)) +%Y%m%d%H%M.%S)" "$logdir/session.jsonl" + + fb="$TMP_ROOT/nobirth-fakebin" + nobirth_stat_bin "$fb" + out=$(PATH="$fb:$PATH" "$HARVEST" "$id" 2>&1) + expect_code 0 "$?" "birthless claude harvest should succeed"$'\n'"$out" + ledger="$data/usage-ledger.jsonl" + row=$(cat "$ledger") + assert_contains "$row" '"source":"claude-projects"' \ + "birthless harvest still finds the in-window log via the meta-mtime start" + assert_contains "$row" '"input_tokens":12' "birthless harvest sums the in-window usage" + assert_contains "$row" '"wall_secs":100' \ + "birthless window spans meta -> status instead of collapsing to zero" + pass "claude harvest: window survives a host without usable birth time" +} + +# --- remote secondmate: local logs are never harvested for remote work ------ + +remote_case() { + local id=usageremote1 wt="$TMP_ROOT/wt-usageremote1" + local data home ledger row out + data=$(harvest_case "$id" codex "$wt" default high) + home=$(dirname "$data") + export_harvest_env "$home" + # The task ran on a remote secondmate host (fm-spawn records remote_host=). + printf 'remote_host=box.example\n' >> "$home/state/$id.meta" + # Seed a LOCAL codex session whose cwd matches the worktree: without the + # remote-host guard the harvest would misattribute this local session to the + # remote task; with it, the row is honestly unavailable and its tokens null. + local d1="$FM_USAGE_CODEX_DIR/2026/08/28" + mkdir -p "$d1" + cat > "$d1/rollout-localmatch.jsonl" <&1) + expect_code 0 "$?" "remote harvest should succeed"$'\n'"$out" + ledger="$home/data/usage-ledger.jsonl" + [ "$(wc -l < "$ledger" | tr -d ' ')" = 1 ] || fail "remote harvest wrote no single row" + row=$(cat "$ledger") + assert_contains "$row" '"source":"unavailable"' \ + "remote task records unavailable, not a local session" + assert_contains "$row" '"input_tokens":null' \ + "remote task tokens are null (no local misattribution)" + pass "remote secondmate harvest: local logs are never misattributed" +} + +# --- concurrency: the ledger lock keeps the append idempotent --------------- + +race_case() { + local id=usagerace1 wt="$TMP_ROOT/wt-usagerace1" + local data home ledger p1 p2 + data=$(harvest_case "$id" cursor "$wt" cursor-x "") + home=$(dirname "$data") + export_harvest_env "$home" + ledger="$home/data/usage-ledger.jsonl" + # Two concurrent harvests with a widened check-to-append window: the ledger + # lock must serialize them so exactly one row is appended. Without the lock + # both pass the existence check and each append, yielding two rows. + FM_USAGE_HARVEST_APPEND_DELAY=1 "$HARVEST" "$id" >/dev/null 2>&1 & + p1=$! + FM_USAGE_HARVEST_APPEND_DELAY=1 "$HARVEST" "$id" >/dev/null 2>&1 & + p2=$! + wait "$p1"; wait "$p2" + [ -f "$ledger" ] || fail "concurrent harvest wrote no ledger" + [ "$(wc -l < "$ledger" | tr -d ' ')" = 1 ] \ + || fail "concurrent harvests appended more than one row" + pass "usage harvest: concurrent harvests append exactly one row" +} + +# A held ledger lock must never block the (teardown-synchronous) harvest +# forever: the acquire is bounded, so a second harvest whose lock-wait is +# shorter than the holder's critical section gives up best-effort (non-zero, +# no duplicate row) instead of hanging until the holder releases. +lock_bound_case() { + local id=usagelockbound1 wt="$TMP_ROOT/wt-usagelockbound1" + local data home ledger p1 rc + data=$(harvest_case "$id" cursor "$wt" cursor-x "") + home=$(dirname "$data") + export_harvest_env "$home" + ledger="$home/data/usage-ledger.jsonl" + # Holder keeps the lock across a 4s critical section. + FM_USAGE_HARVEST_APPEND_DELAY=4 "$HARVEST" "$id" >/dev/null 2>&1 & + p1=$! + # Let the holder acquire before the bounded harvester starts spinning. + sleep 1 + rc=0 + FM_USAGE_LEDGER_LOCK_WAIT=1 "$HARVEST" "$id" >/dev/null 2>&1 || rc=$? + [ "$rc" -ne 0 ] \ + || fail "bounded harvest returned 0 while the ledger lock was held" + wait "$p1" + [ "$(wc -l < "$ledger" | tr -d ' ')" = 1 ] \ + || fail "bounded give-up appended a duplicate row" + pass "usage harvest: a held ledger lock bounds the acquire, no hang or dup" +} + +# --- staging file: a failed ledger append leaves no runtime debris ---------- + +# The harvest builds its row in a staging file next to the ledger and then +# appends it. A failed append (here an unwritable ledger) must still leave the +# data directory free of staging files, otherwise every best-effort teardown +# harvest against a broken ledger accumulates another one. +staging_case() { + local id=usagestaging1 wt="$TMP_ROOT/wt-usagestaging1" + local data home rc leftovers + data=$(harvest_case "$id" cursor "$wt" cursor-x "") + home=$(dirname "$data") + export_harvest_env "$home" + # The ledger path being a directory forces the append to fail. + mkdir -p "$home/data/usage-ledger.jsonl" + rc=0 + "$HARVEST" "$id" >/dev/null 2>&1 || rc=$? + [ "$rc" -ne 0 ] || fail "harvest returned 0 with an unwritable ledger" + leftovers=$(find "$home/data" -maxdepth 1 -name 'usage-ledger.jsonl.tmp.*' 2>/dev/null) + [ -z "$leftovers" ] \ + || fail "failed ledger append left staging files: $leftovers" + + # The same must hold on the success path. + rmdir "$home/data/usage-ledger.jsonl" + "$HARVEST" "$id" >/dev/null 2>&1 || fail "harvest failed with a writable ledger" + [ "$(wc -l < "$home/data/usage-ledger.jsonl" | tr -d ' ')" = 1 ] \ + || fail "successful retry did not append exactly one row" + leftovers=$(find "$home/data" -maxdepth 1 -name 'usage-ledger.jsonl.tmp.*' 2>/dev/null) + [ -z "$leftovers" ] \ + || fail "successful ledger append left staging files: $leftovers" + pass "usage harvest: no staging files survive a failed or successful append" +} + +# --- report: per-model totals and per-task rows ----------------------------- + +report_case() { + local out ledger + ledger="$TMP_ROOT/home-usageclaude1/data/usage-ledger.jsonl" + cat "$TMP_ROOT/home-usagecodex1/data/usage-ledger.jsonl" >> "$ledger" + cat "$TMP_ROOT/home-usagecursor1/data/usage-ledger.jsonl" >> "$ledger" + out=$(FM_DATA_OVERRIDE="$(dirname "$ledger")" "$REPORT" 2>&1) + expect_code 0 "$?" "report should succeed"$'\n'"$out" + assert_contains "$out" "usage ledger: $ledger (3 rows)" "report names the ledger and row count" + assert_contains "$out" "per-model totals" "report prints per-model totals" + assert_contains "$out" "claude-test" "report lists the claude model" + assert_contains "$out" "glm-5.3" "report lists the codex model" + assert_contains "$out" "usageclaude1" "report lists each task row" + assert_contains "$out" "usagecursor1" "report lists the unavailable task row" + assert_contains "$out" "unavailable" "report shows the unavailable source" + # Missing ledger exits 0 with a note. + out=$(FM_DATA_OVERRIDE="$TMP_ROOT/empty-home" "$REPORT" 2>&1) + expect_code 0 "$?" "report without a ledger should exit 0" + assert_contains "$out" "no usage ledger" "report names the missing ledger" + pass "usage report: per-model totals, per-task rows, missing-ledger tolerance" +} + +# --- teardown integration: harvest failure never blocks teardown ------------ + +teardown_case() { + local proj wt id fb state data config out + id=usageharvtd1 + proj="$TMP_ROOT/td-proj"; wt="$TMP_ROOT/td-wt" + fm_git_worktree "$proj" "$wt" "fm/$id" + fb="$TMP_ROOT/td-fakebin" + mkdir -p "$fb" + cat > "$fb/tmux" <<'SH' +#!/usr/bin/env bash +exit 0 +SH + cat > "$fb/treehouse" <<'SH' +#!/usr/bin/env bash +exit 0 +SH + chmod +x "$fb/tmux" "$fb/treehouse" + state="$TMP_ROOT/td-state"; config="$TMP_ROOT/td-config"; data="$TMP_ROOT/td-data" + mkdir -p "$state" "$config" "$data/$id" + printf 'scout findings\n' > "$data/$id/report.md" + fm_write_meta "$state/$id.meta" \ + "window=firstmate:fm-$id" "worktree=$wt" "project=$proj" "harness=claude" \ + "kind=scout" "mode=no-mistakes" "yolo=off" \ + "decisions_reviewed=1" "decision_keys=" + printf 'working: scouting\n' > "$state/$id.status" + # The ledger path being a directory forces every append to fail. + mkdir -p "$data/usage-ledger.jsonl" + out=$(PATH="$fb:$PATH" FM_ROOT_OVERRIDE="$ROOT" \ + FM_STATE_OVERRIDE="$state" FM_DATA_OVERRIDE="$data" FM_CONFIG_OVERRIDE="$config" \ + FM_USAGE_CLAUDE_DIR="$TMP_ROOT/td-fake-claude" FM_USAGE_CODEX_DIR="$TMP_ROOT/td-fake-codex" \ + "$TEARDOWN" "$id" 2>&1) + expect_code 0 "$?" "teardown must succeed even when the harvest fails"$'\n'"$out" + assert_contains "$out" "warning: usage harvest for $id failed" \ + "teardown warns one line when the harvest fails" + assert_contains "$out" "teardown $id complete" \ + "teardown completes after a harvest failure" + pass "teardown integration: harvest failure is non-fatal" +} + +claude_case +claude_nobirth_case +codex_case +cursor_case +remote_case +race_case +lock_bound_case +staging_case +report_case +teardown_case diff --git a/tests/herdr-test-safety.sh b/tests/herdr-test-safety.sh index 59a2bb46cc2..e179241d5ac 100644 --- a/tests/herdr-test-safety.sh +++ b/tests/herdr-test-safety.sh @@ -40,3 +40,27 @@ herdr_refuse_if_default() { # herdr_safe_stop_and_delete() { # fm_herdr_lab_teardown "$1" } + +# herdr_pane_run_foreground: give in a real, long-lived, +# non-shell foreground process and wait until Herdr's own inventory reports it. +# +# A case that means "this pane hosts a genuinely live agent" needs both halves +# the liveness classifier reads: a registration AND a pane that is actually +# running something. `pane report-agent` alone leaves a registration standing +# over a bare idle shell, which is the husk shape, not the live one +# (docs/herdr-backend.md "Restart and liveness behavior"). Requires the herdr +# backend adapter to be sourced. +herdr_pane_run_foreground() { # [command] + local session=$1 pane=$2 cmd=${3:-sleep 600} attempt=0 names + fm_backend_herdr_cli "$session" pane run "$pane" "$cmd" >/dev/null 2>&1 || return 1 + while [ "$attempt" -lt 200 ]; do + names=$(fm_backend_herdr_cli "$session" pane process-info --pane "$pane" 2>/dev/null \ + | jq -r '[.result.process_info.foreground_processes[]?.name] | join(",")' 2>/dev/null) + case "$names" in + *"${cmd%% *}"*) return 0 ;; + esac + sleep 0.05 + attempt=$((attempt + 1)) + done + return 1 +}