diff --git a/AGENTS.md b/AGENTS.md index d2ad7a7438c..d4d43ffcc5c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -7,6 +7,7 @@ This file is your entire job description. Address the user as "captain" at least once in every response. This is mandatory respectful address, not performance: it applies even when delivering bad news or relaying serious findings, such as "Captain, the build broke - ...". Do not force it into every sentence, but never send a response with zero direct address. +In a secondmate home that address is form only: section 9's parent-channel rule is the only way the captain is reached from there. Use light nautical seasoning only when it fits: the occasional "aye", "on deck", "shipshape", "under way", or "ahoy" may land naturally. Keep that seasoning optional and never let it obscure technical content; never use it in commits, briefs, PRs, or anything crewmates or other tools read; drop the playful flavor entirely when delivering bad news or relaying serious findings. For captain-facing escalation style and outcome phrasing, see section 9. @@ -115,6 +116,7 @@ state/ runtime records and signals; gitignored x-watch.check.sh generated Relay poll shim; present only when opted in (section 14) tool-updates.check.sh generated watched-tool update poll shim and its .check-trust binding; present only after bin/fm-tool-update-check.sh arm; its report record .tool-updates is what keeps one pending update from being reported on every poll pending-replies/ parent-owned secondmate pending-reply records (correlation id, delivery vs reply, recovery, escalation); fm-pending-reply-lib.sh + parent-mirror/ durable per-child state for a secondmate home's parent-channel mirror; written only by bin/fm-parent-mirror-lib.sh, never touch (docs/secondmate-parent-channel.md) procevent/ registered process-to-event sources, one private record per canonical source id; written only by bin/fm-procevent.sh, and their presence alone keeps supervision required (section 13) procevent-inbox/ private captured results and their durable handled-acknowledgement markers; source output lives here and never in an event line decision-bindings/ private records marking a captured-answer source as feeding the keyed-answer intake, with a legacy origin on pre-collapse records; written only by bin/fm-captain-hold.sh bind, dropped by unbind and by source retirement (section 13; docs/captain-hold-lifecycle.md) @@ -485,6 +487,7 @@ Reach the captain immediately for: - Anything destructive, irreversible, or security-sensitive. - A needed credential or login. +In a secondmate home, reaching the captain means appending the outcome to the parent channel named in `data/charter.md`; a captain-facing sentence in that home's chat has not been sent, and [`docs/secondmate-parent-channel.md`](docs/secondmate-parent-channel.md) owns which child outcomes the mirror delivers without you. Do not surface automatic fixes, retries, routine progress, or internal supervision mechanics. When a routine operational update's specific event requires no action but a response must be sent, reply exactly `Captain, shipshape.` without characterizing the visible session's unrelated decisions. Batch non-urgent updates into the next natural reply. diff --git a/bin/fm-brief.sh b/bin/fm-brief.sh index b5fba5b5a72..c25b1dc8a5e 100755 --- a/bin/fm-brief.sh +++ b/bin/fm-brief.sh @@ -239,6 +239,12 @@ You do not generate your own work. Act only on tasks the main firstmate routes to you. Never start a survey, audit, or "find improvements" sweep on your own initiative; that is not your job and it is unwanted. +# The captain and the parent channel +Nobody reads this chat: the captain and the main firstmate see only what is appended to $STATUS_FILE, and a captain-facing sentence that is not appended there has not been sent. +That file is your parent channel, and in this home it IS the captain: every sentence you would say to the captain, and every outcome the local AGENTS.md tells a firstmate to bring to the captain, is one appended line there, never chat. +Your own machinery delivers the durable facts about your crew's work for you (\`docs/secondmate-parent-channel.md\`): a child's PR-ready or terminal line, a scout report, a failed line, and a decision or blocker left open past its threshold are mirrored to the parent channel by \`bin/fm-parent-mirror.sh\` on every supervision poll, \`bin/fm-pr-check.sh\` and \`bin/fm-captain-hold.sh\` publish the PR and captain-hold facts they record, and a merge is reported by the merge outcome path. +What only you can append is judgement: the answer to a marked request below, a recommendation or caveat on a mirrored outcome, a blocker or failure of your own, and anything else you would otherwise say to the captain. + # Requests from the main firstmate You are a firstmate in your own home, so an incoming message reaches you in your own chat. You must distinguish who it is from, because the answer goes to a different place. diff --git a/bin/fm-captain-hold.sh b/bin/fm-captain-hold.sh index cb429d95238..0bd1e75e9fe 100755 --- a/bin/fm-captain-hold.sh +++ b/bin/fm-captain-hold.sh @@ -118,8 +118,9 @@ # `diverged` is the read-only guard over the seam between the two records of # one captain call. See "record divergence" beside command_diverged below. # -# Resolution records: the block written into the body names this script, the -# decision digest, and a `Resolution mode:` of answered, released, or repaired. +# Resolution records: each hold stores its durable generation in the body, and +# the length-framed answer block names this script, the decision digest, that +# generation, and a `Resolution mode:` of answered, released, or repaired. # Records written by the retired fm-decision-hold.sh (routed, declined, # answered, repaired) are recognized everywhere a record is read, so nothing # already closed needs rewriting. @@ -140,6 +141,45 @@ DATA="${FM_DATA_OVERRIDE:-$FM_HOME/data}" # shellcheck source=bin/fm-wake-lib.sh # shellcheck disable=SC1091 . "$SCRIPT_DIR/fm-wake-lib.sh" +# shellcheck source=bin/fm-parent-channel-lib.sh +# shellcheck disable=SC1091 +. "$SCRIPT_DIR/fm-parent-channel-lib.sh" + +# A task held for the captain inside a secondmate home is a parent-facing fact +# the moment it is recorded, and so is its answer, so both are published on the +# parent channel here rather than left to the mate remembering to append them +# (docs/secondmate-parent-channel.md). A main home has no channel and the +# publication is a silent no-op there. The hold or answer itself is already +# durable in this home's backlog, so a delivery problem is reported as +# actionable rather than undoing the record. +publish_parent_hold() { # + local id=$1 occurrence=$2 verb=$3 note=$4 rc=0 + fm_parent_channel_report "$FM_HOME" "$STATE" \ + "$verb [key=captain-hold-$id-$occurrence]: captain hold $id occurrence $occurrence: $(fm_parent_channel_clean_note "$note")" || rc=$? + case "$rc" in + 0|1) return 0 ;; + *) + printf 'actionable: task %s is held for the captain in this home but that did not reach the parent channel (rc=%s)\n' "$id" "$rc" >&2 + return "$rc" + ;; + esac +} + +# A resolution remains in the task body after a release, so a later re-hold can +# retry its parent close before advancing the occurrence. Refusing the advance +# while that retry cannot reach the parent prevents the old keyed decision from +# becoming an uncloseable orphan behind a newer hold. +publish_recorded_parent_resolution() { # + local id=$1 body=$2 generation mode note + generation=$(recorded_resolution_generation "$body") || return 0 + mode=$(recorded_resolution_mode "$body") || return 0 + case "$mode" in + released) note=released ;; + repaired) note='answered (repaired)' ;; + *) note=answered ;; + esac + publish_parent_hold "$id" "$generation" resolved "$note" +} CAPTAIN_META_LOCK= CAPTAIN_META_LOCK_HELD=0 @@ -299,45 +339,149 @@ origin_open_decisions() { # printf '%s' "$open" } -# A resolution record written by this script or by the retired -# fm-decision-hold.sh. Both carry the same leader-then-captain-decision shape. +RESOLUTION_RECORD_DIGEST= +RESOLUTION_RECORD_MODE= +RESOLUTION_RECORD_GENERATION= +RESOLUTION_RECORD_WRITER= + +resolution_record_parse() { # + local body parsed + RESOLUTION_RECORD_DIGEST= + RESOLUTION_RECORD_MODE= + RESOLUTION_RECORD_GENERATION= + RESOLUTION_RECORD_WRITER= + body=$(decode_shown_value "$1") || return 1 + parsed=$(printf '%s\n' "$body" | awk ' + { line[NR] = $0 } + END { + start = 1 + if (line[1] == "Captain hold state recorded by fm-captain-hold." \ + && line[2] ~ /^Generation: [0-9]+$/ \ + && line[3] == "End captain hold state." && line[4] == "") start = 5 + if (line[start] == "Captain resolution record begins (fm-captain-hold)." \ + && line[start + 1] == "Resolution recorded by fm-captain-hold." \ + && line[start + 2] ~ /^Decision digest: / \ + && line[start + 3] ~ /^Resolution mode: / \ + && line[start + 4] ~ /^Hold generation: [0-9]+$/ \ + && line[start + 5] ~ /^Decision lines: [0-9]+$/ \ + && line[start + 6] == "" && line[start + 7] == "Captain decision:") { + count = line[start + 5]; sub(/^Decision lines: /, "", count) + if (line[start + 8 + count] != "Captain resolution record ends (fm-captain-hold).") exit 1 + digest = line[start + 2]; sub(/^Decision digest: /, "", digest) + mode = line[start + 3]; sub(/^Resolution mode: /, "", mode) + generation = line[start + 4]; sub(/^Hold generation: /, "", generation) + print digest "\t" mode "\t" generation "\tfm-captain-hold" + exit + } + if (start != 1) exit 1 + if (line[1] != "Resolution recorded by fm-captain-hold." \ + && line[1] != "Resolution recorded by fm-decision-hold.") exit 1 + if (line[2] !~ /^Decision digest: /) exit 1 + digest = line[2]; sub(/^Decision digest: /, "", digest) + position = 3 + if (line[position] ~ /^Routed identities: /) position++ + if (line[position] !~ /^Resolution mode: /) exit 1 + mode = line[position]; sub(/^Resolution mode: /, "", mode); position++ + generation = 1 + if (line[position] ~ /^Hold generation: [0-9]+$/) { + generation = line[position]; sub(/^Hold generation: /, "", generation); position++ + } + if (line[position] != "" || line[position + 1] != "Captain decision:") exit 1 + writer = line[1]; sub(/^Resolution recorded by /, "", writer); sub(/\.$/, "", writer) + print digest "\t" mode "\t" generation "\t" writer + } + ') || return 1 + [ -n "$parsed" ] || return 1 + IFS=$'\t' read -r RESOLUTION_RECORD_DIGEST RESOLUTION_RECORD_MODE RESOLUTION_RECORD_GENERATION RESOLUTION_RECORD_WRITER < - case "$1" in - *"Resolution recorded by fm-captain-hold."*"Captain decision:"*) return 0 ;; - *"Resolution recorded by fm-decision-hold."*"Captain decision:"*) return 0 ;; - esac - return 1 + resolution_record_parse "$1" } -# The recorded decision digest of either record format, from the show-escaped -# body (multi-line bodies print as one quoted line with \n escapes). Records -# are prepended, so the first match is the newest record. recorded_decision_digest() { # - local rest=$1 - case "$rest" in - *"Decision digest: "*) rest=${rest#*"Decision digest: "} ;; - *) return 1 ;; - esac - rest=${rest%%\\n*} - rest=${rest%%$'\n'*} - printf '%s' "$rest" + resolution_record_parse "$1" || return 1 + printf '%s' "$RESOLUTION_RECORD_DIGEST" } -# The newest record's `Resolution mode:` value; empty for a record predating it. recorded_resolution_mode() { # - local rest=$1 - case "$rest" in - *"Resolution mode: "*) rest=${rest#*"Resolution mode: "} ;; - *) return 1 ;; - esac - rest=${rest%%\\n*} - rest=${rest%%$'\n'*} - printf '%s' "$rest" + resolution_record_parse "$1" || return 1 + printf '%s' "$RESOLUTION_RECORD_MODE" } -resolution_block() { # - printf 'Resolution recorded by fm-captain-hold.\nDecision digest: %s\nResolution mode: %s\n\nCaptain decision:\n%s\n' \ - "$DECISION_DIGEST" "$1" "$DECISION_TEXT" +body_has_hold_state_block() { # + local body + body=$(decode_shown_value "$1") || return 1 + printf '%s\n' "$body" | awk ' + NR == 1 && $0 != "Captain hold state recorded by fm-captain-hold." { exit 1 } + NR == 2 && $0 !~ /^Generation: [0-9]+$/ { exit 1 } + NR == 3 { exit !($0 == "End captain hold state.") } + END { if (NR < 3) exit 1 } + ' +} + +recorded_hold_generation() { # + local body + body=$(decode_shown_value "$1") || return 1 + printf '%s\n' "$body" | awk ' + NR == 1 && $0 == "Captain hold state recorded by fm-captain-hold." { state = 1; next } + NR == 1 && $0 ~ /^Captain hold generation: [0-9]+$/ { + sub(/^Captain hold generation: /, ""); print; exit + } + NR == 2 && state && $0 ~ /^Generation: [0-9]+$/ { + sub(/^Generation: /, ""); generation = $0; next + } + NR == 3 && state && $0 == "End captain hold state." { print generation; exit } + { exit 1 } + ' +} + +recorded_resolution_generation() { # + resolution_record_parse "$1" || return 1 + printf '%s' "$RESOLUTION_RECORD_GENERATION" +} + +resolution_record_count() { # + if resolution_record_parse "$1"; then + printf '%s\n' "$RESOLUTION_RECORD_GENERATION" + else + printf '0\n' + fi +} + +resolution_block() { # + local decision_lines + decision_lines=$(printf '%s\n' "$DECISION_TEXT" | awk 'END { print NR }') + printf 'Captain resolution record begins (fm-captain-hold).\nResolution recorded by fm-captain-hold.\nDecision digest: %s\nResolution mode: %s\nHold generation: %s\nDecision lines: %s\n\nCaptain decision:\n%s\nCaptain resolution record ends (fm-captain-hold).\n' \ + "$DECISION_DIGEST" "$1" "$2" "$decision_lines" "$DECISION_TEXT" +} + +write_hold_generation() { # + local id=$1 generation=$2 body=$3 old_generation new_body tmp prefix + body=$(decode_shown_value "$body") || fail "could not decode the existing body for $id" + old_generation=$(recorded_hold_generation "$body" || true) + if [ -n "$old_generation" ]; then + prefix=$(printf 'Captain hold state recorded by fm-captain-hold.\nGeneration: %s\nEnd captain hold state.' "$old_generation") + case "$body" in + "$prefix"$'\n\n'*) body=${body#"$prefix"$'\n\n'} ;; + "$prefix") body='' ;; + "Captain hold generation: $old_generation"$'\n\n'*) body=${body#"Captain hold generation: $old_generation"$'\n\n'} ;; + "Captain hold generation: $old_generation") body='' ;; + esac + fi + new_body=$(printf 'Captain hold state recorded by fm-captain-hold.\nGeneration: %s\nEnd captain hold state.' "$generation") + [ -z "$body" ] || new_body=$(printf '%s\n\n%s' "$new_body" "$body") + tmp=$(umask 077; mktemp "${TMPDIR:-/tmp}/fm-captain-hold-generation.XXXXXX") \ + || fail "cannot stage the hold generation" + printf '%s\n' "$new_body" > "$tmp" \ + || { rm -f -- "$tmp"; fail "cannot stage the hold generation for $id"; } + tasks_axi update "$id" --body-file "$tmp" --archive-body >/dev/null \ + || { rm -f -- "$tmp"; fail "could not record the hold generation on $id"; } + rm -f -- "$tmp" } # Durable state of one captain call: an active captain hold (annotations @@ -377,7 +521,7 @@ resolve_entry() { # ; prints the resolved id or fails } command_hold() { - local id=${1:-} title='' reason='' repo='' origin='' until='' show state existing_title body='' hold_kind + local id=${1:-} title='' reason='' repo='' origin='' until='' show state existing_title body='' hold_kind occurrence current_generation resolution_generation [ "$#" -ge 1 ] || { usage >&2; exit 2; } shift while [ "$#" -gt 0 ]; do @@ -431,6 +575,28 @@ command_hold() { || fail "could not create task $id" fi fi + show=$(task_show "$id") || fail "task $id disappeared before holding it" + body=$(show_field "$show" body) + hold_kind=$(show_field_value "$show" hold_kind) + current_generation=$(recorded_hold_generation "$body" || true) + if [ "$hold_kind" != captain ] && body_has_resolution_record "$body"; then + publish_recorded_parent_resolution "$id" "$body" \ + || fail "cannot re-hold task $id until its recorded answer reaches the parent channel" + fi + if [ "$hold_kind" = captain ] && [ -n "$current_generation" ]; then + occurrence=$current_generation + body_has_hold_state_block "$body" || write_hold_generation "$id" "$occurrence" "$body" + else + resolution_generation=$(recorded_resolution_generation "$body" || true) + if [ -n "$current_generation" ] && [ "$resolution_generation" != "$current_generation" ]; then + occurrence=$current_generation + elif [ -n "$current_generation" ]; then + occurrence=$((current_generation + 1)) + else + occurrence=$(( $(resolution_record_count "$body") + 1 )) + fi + write_hold_generation "$id" "$occurrence" "$body" + fi if [ -n "$until" ]; then tasks_axi hold "$id" --reason "$reason" --kind captain --until "$until" >/dev/null \ || fail "could not hold task $id for the captain" @@ -441,19 +607,27 @@ command_hold() { show=$(task_show "$id") || fail "task $id disappeared while holding it" hold_kind=$(show_field_value "$show" hold_kind) [ "$hold_kind" = captain ] || fail "task $id did not retain its captain hold" + if [ -n "$origin" ]; then + publish_parent_hold "$id" "$occurrence" needs-decision "$reason (origin $origin)" || true + else + publish_parent_hold "$id" "$occurrence" needs-decision "$reason" || true + fi printf '%s\n' "$id" } # Record a resolution block at the top of the task body, preserving the # previous body below it and archiving the pristine original. -write_resolution_record() { # - local id=$1 mode=$2 body=$3 new_body tmp - new_body=$(resolution_block "$mode") +write_resolution_record() { # + local id=$1 mode=$2 generation=$3 body=$4 new_body tmp prefix body=$(decode_shown_value "$body") \ || fail "could not decode the existing body for $id" - if [ -n "$body" ]; then - new_body=$(printf '%s\n\n%s' "$new_body" "$body") - fi + prefix=$(printf 'Captain hold state recorded by fm-captain-hold.\nGeneration: %s\nEnd captain hold state.' "$generation") + case "$body" in + "$prefix"$'\n\n'*) body=${body#"$prefix"$'\n\n'} ;; + "$prefix") body='' ;; + esac + new_body=$(printf '%s\n\n%s' "$prefix" "$(resolution_block "$mode" "$generation")") + [ -z "$body" ] || new_body=$(printf '%s\n\n%s' "$new_body" "$body") tmp=$(umask 077; mktemp "${TMPDIR:-/tmp}/fm-captain-hold-body.XXXXXX") \ || fail "cannot stage the resolution record" if ! printf '%s\n' "$new_body" > "$tmp"; then @@ -476,7 +650,8 @@ close_answered() { # } command_answer() { - local id=${1:-} decision_file='' release=0 show state hold_kind body outcome recorded_mode + local id=${1:-} decision_file='' release=0 show state hold_kind body outcome recorded_mode occurrence + local hold_generation resolution_generation [ "$#" -ge 1 ] || { usage >&2; exit 2; } shift while [ "$#" -gt 0 ]; do @@ -494,11 +669,14 @@ command_answer() { state=$(show_field "$show" state) hold_kind=$(show_field_value "$show" hold_kind) body=$(show_field "$show" body) + hold_generation=$(recorded_hold_generation "$body" || true) + resolution_generation=$(recorded_resolution_generation "$body" || true) + [ -n "$resolution_generation" ] || resolution_generation=$(resolution_record_count "$body") if [ "$release" = 1 ]; then outcome=released; else outcome=answered; fi if [ "$state" = "done" ]; then if body_has_resolution_record "$body"; then - # An exact compatible retry is an idempotent no-op; drift is rejected. + # An exact compatible retry is idempotent; drift is rejected. [ "$(recorded_decision_digest "$body" || true)" = "$DECISION_DIGEST" ] \ || fail "captain-held task $id records a different captain decision" recorded_mode=$(recorded_resolution_mode "$body" || true) @@ -506,6 +684,12 @@ command_answer() { || fail "task $id records this answer with mode released; a closed task cannot replay that release" [ "$release" = 0 ] \ || fail "task $id records this answer with mode ${recorded_mode:-unknown}; --release cannot reopen a closed task" + occurrence=$resolution_generation + if [ "$recorded_mode" = repaired ]; then + publish_parent_hold "$id" "$occurrence" resolved "answered (repaired)" || return "$?" + else + publish_parent_hold "$id" "$occurrence" resolved answered || return "$?" + fi printf 'answered: %s\n' "$id" return 0 fi @@ -515,11 +699,14 @@ command_answer() { # this really was the captain's item rather than ordinary finished work. [ "$hold_kind" = captain ] \ || fail "task $id was never held for the captain; nothing to record an answer on" - write_resolution_record "$id" repaired "$body" + occurrence=${hold_generation:-$(( $(resolution_record_count "$body") + 1 ))} + write_resolution_record "$id" repaired "$occurrence" "$body" show=$(task_show "$id") || fail "task $id disappeared while recording the answer" [ "$(show_field "$show" state)" = "done" ] || fail "recording the answer reopened closed task $id" - body_has_resolution_record "$(show_field "$show" body)" \ + body=$(show_field "$show" body) + body_has_resolution_record "$body" \ || fail "captain-held task $id did not retain its durable resolution record" + publish_parent_hold "$id" "$occurrence" resolved "answered (repaired)" || true printf 'repaired: %s\n' "$id" return 0 fi @@ -531,22 +718,31 @@ command_answer() { # its own record on top. Either way the close mode is the caller's flag, # checked against an interrupted close's recorded mode so a retry cannot # silently flip a release into a close. + if [ -z "$hold_generation" ]; then + hold_generation=$resolution_generation + [ "$hold_generation" -gt 0 ] || hold_generation=$(( $(resolution_record_count "$body") + 1 )) + fi if body_has_resolution_record "$body" \ - && [ "$(recorded_decision_digest "$body" || true)" = "$DECISION_DIGEST" ]; then + && [ "$(recorded_decision_digest "$body" || true)" = "$DECISION_DIGEST" ] \ + && [ "$resolution_generation" = "$hold_generation" ]; then recorded_mode=$(recorded_resolution_mode "$body" || true) case "$recorded_mode" in released) [ "$release" = 1 ] || fail "task $id records this answer as a release; retry with --release" ;; answered) [ "$release" = 0 ] || fail "task $id records this answer as a close; retry without --release" ;; esac + occurrence=$hold_generation close_answered "$id" "$release" + publish_parent_hold "$id" "$occurrence" resolved "$outcome" || return "$?" printf '%s: %s\n' "$outcome" "$id" return 0 fi - write_resolution_record "$id" "$outcome" "$body" + occurrence=$hold_generation + write_resolution_record "$id" "$outcome" "$occurrence" "$body" close_answered "$id" "$release" show=$(task_show "$id") || fail "task $id disappeared after closing" body_has_resolution_record "$(show_field "$show" body)" \ || fail "captain-held task $id did not retain its durable resolution record" + publish_parent_hold "$id" "$occurrence" resolved "$outcome" || true printf '%s: %s\n' "$outcome" "$id" return 0 fi @@ -558,6 +754,8 @@ command_answer() { || fail "task $id records a different captain decision with mode ${recorded_mode:-unknown}" [ "$recorded_mode" = released ] && [ "$release" = 1 ] \ || fail "task $id records this answer with mode ${recorded_mode:-unknown}; replay requires matching --release" + occurrence=$resolution_generation + publish_parent_hold "$id" "$occurrence" resolved released || return "$?" printf 'released: %s\n' "$id" return 0 fi @@ -657,7 +855,7 @@ sanitize_field() { # command_answers() { local origin='' source='' row rest key answer label mode id show state hold_kind body digest legacy_digest legacy_key - local recorded_digest recorded_mode tmp err closed=0 skipped=0 reason release_flag tab=$'\t' + local recorded_digest recorded_mode resolution_generation parent_note tmp err closed=0 skipped=0 reason release_flag tab=$'\t' while [ "$#" -gt 0 ]; do case "$1" in --source) shift; source=${1:-} ;; @@ -731,11 +929,22 @@ command_answers() { recorded_mode=$(recorded_resolution_mode "$body" || true) if body_has_resolution_record "$body" \ && { [ "$recorded_digest" = "$digest" ] \ - || { case "$body" in *"Resolution recorded by fm-decision-hold."*) true ;; *) false ;; esac \ + || { [ "$RESOLUTION_RECORD_WRITER" = fm-decision-hold ] \ && [ -n "$legacy_digest" ] && [ "$recorded_digest" = "$legacy_digest" ]; }; }; then if { [ -z "$release_flag" ] && [ "$state" = "done" ] && [ "$recorded_mode" != released ]; } \ || { [ "$release_flag" = --release ] && [ "$state" != "done" ] \ && [ "$hold_kind" != captain ] && [ "$recorded_mode" = released ]; }; then + resolution_generation=$(recorded_resolution_generation "$body" || resolution_record_count "$body") + case "$recorded_mode" in + released) parent_note=released ;; + repaired) parent_note='answered (repaired)' ;; + *) parent_note=answered ;; + esac + if ! publish_parent_hold "$id" "$resolution_generation" resolved "$parent_note"; then + printf 'skipped: %s (parent channel delivery failed)\n' "$id" + skipped=$((skipped + 1)) + continue + fi printf 'closed: %s\n' "$id" closed=$((closed + 1)) continue @@ -753,6 +962,7 @@ command_answers() { fi # shellcheck disable=SC2086 # release_flag is empty or a single literal flag. if "$0" answer "$id" --decision-file "$tmp" $release_flag /dev/null 2>"$err"; then + [ ! -s "$err" ] || cat "$err" >&2 printf 'closed: %s\n' "$id" closed=$((closed + 1)) else diff --git a/bin/fm-classify-lib.sh b/bin/fm-classify-lib.sh index 506f398cce9..187532c59d9 100755 --- a/bin/fm-classify-lib.sh +++ b/bin/fm-classify-lib.sh @@ -1138,8 +1138,8 @@ status_presentation_marker_commit() { printf 'v2\t%s\t%s' "$reported" "$classified" > "$marker" } -status_retire_presentation_task() { # - local state=$1 task=$2 lock manifest tmp data row_task ident offset backstop extra rc=0 found=0 +status_retire_presentation_task() { # [] + local state=$1 task=$2 seconds=${3:-} lock manifest tmp data row_task ident offset backstop extra rc=0 found=0 local signal_marker heartbeat_marker daemon_marker lock="$state/.status-presentation-lock" manifest="$state/.status-presentation-cursor" @@ -1178,7 +1178,11 @@ EOF fi fi - fm_lock_acquire_wait "$lock" || return 1 + if [ -n "$seconds" ]; then + fm_lock_acquire_wait_bounded "$lock" "$seconds" || return 1 + else + fm_lock_acquire_wait "$lock" || return 1 + fi if [ -e "$manifest" ] || [ -L "$manifest" ]; then if [ ! -f "$manifest" ] || [ ! -r "$manifest" ] || [ -L "$manifest" ]; then rc=1 diff --git a/bin/fm-inactive-reconcile.sh b/bin/fm-inactive-reconcile.sh index a7fc30c5244..9a596801ad3 100755 --- a/bin/fm-inactive-reconcile.sh +++ b/bin/fm-inactive-reconcile.sh @@ -28,11 +28,15 @@ # outcome record or wake the supervisor. # Working, paused, parked, blocked, unknown, persistent secondmates, and # captain-held work retain their existing supervision semantics. +# In a secondmate home this scan owns only the SILENT ledger: a child whose +# ledger already ends in a terminal captain verb is the parent mirror's +# evidence (bin/fm-parent-mirror-lib.sh delivers it on its own clock), so it +# is skipped here rather than reported twice. # # A terminal-outcomes/.pending record remains until its upstream # receipt is durable. -# In a secondmate home, that receipt is an idempotent parent-channel status -# append. +# In a secondmate home, that receipt is an idempotent parent-channel append +# through bin/fm-parent-channel-lib.sh. # In a main home, a presentation-stage record is acknowledged by fm-wake-drain # only after its corresponding inactive-outcome wake is handled. # A receipt is intentionally independent of .hb-surfaced-* bookkeeping. @@ -62,8 +66,8 @@ CREW_STATE_BIN="${FM_INACTIVE_CREW_STATE_BIN:-$SCRIPT_DIR/fm-crew-state.sh}" . "$SCRIPT_DIR/fm-wake-lib.sh" # shellcheck source=bin/fm-classify-lib.sh . "$SCRIPT_DIR/fm-classify-lib.sh" -# shellcheck source=bin/fm-secondmate-parent-lib.sh -. "$SCRIPT_DIR/fm-secondmate-parent-lib.sh" +# shellcheck source=bin/fm-parent-mirror-lib.sh +. "$SCRIPT_DIR/fm-parent-mirror-lib.sh" # shellcheck source=bin/fm-timeout-lib.sh . "$SCRIPT_DIR/fm-timeout-lib.sh" @@ -290,44 +294,15 @@ pr_for_task() { # } home_secondmate_id() { - local marker="$FM_HOME/.fm-secondmate-home" id - if [ ! -e "$marker" ] && [ ! -L "$marker" ]; then - return 1 - fi - [ -f "$marker" ] && [ ! -L "$marker" ] || return 2 - [ "$(wc -c < "$marker")" -eq "$(LC_ALL=C tr -d '\0' < "$marker" | wc -c)" ] || return 2 - id=$(cat "$marker" 2>/dev/null) || return 2 - valid_id "$id" || return 2 - printf '%s\n' "$id" -} - -append_once() { # - local path=$1 line=$2 - [ ! -L "$path" ] || return 1 - mkdir -p "$(dirname "$path")" || return 1 - if grep -Fqx -- "$line" "$path" 2>/dev/null; then - return 0 - fi - printf '%s\n' "$line" >> "$path" + fm_parent_channel_home_id "$FM_HOME" } report_to_parent() { # - local self=$1 task=$2 state=$3 outcome_key=$4 fingerprint=$5 pr=$6 parent_record destination line - parent_record="$FM_HOME/.fm-secondmate-parent" - fm_secondmate_parent_record_parse "$parent_record" || return 1 - case "$FM_SECONDMATE_PARENT_ROUTE" in - local) - [ -n "$FM_SECONDMATE_PARENT_HOME" ] || return 1 - destination="$FM_SECONDMATE_PARENT_HOME/state/$self.status" - ;; - remote) - destination="$STATE/parent-replies.status" - ;; - *) return 1 ;; - esac + local self=$1 task=$2 state=$3 outcome_key=$4 fingerprint=$5 pr=$6 line + [ -n "$self" ] || return 1 line="$state [key=$outcome_key]: inactive terminal child=$task fingerprint=$fingerprint" [ -z "$pr" ] || line="$line pr=$pr" - append_once "$destination" "$line" + fm_parent_channel_report "$FM_HOME" "$STATE" "$line" } reconcile_direct_child_locked() { # @@ -339,6 +314,10 @@ reconcile_direct_child_locked() { # /dev/null 2>&1 && return 0 + # A ledger that states its own outcome is the parent mirror's to deliver. + if [ -n "$self" ] && fm_parent_mirror_owns_ledger "$STATE" "$id"; then + return 0 + fi age=$(last_activity_age "$meta" "$status" "$turn") [ "$age" -ge "$FM_INACTIVE_RECONCILE_SECS" ] || return 0 state_line=$(fm_run_timed "$timeout" env FM_HOME="$FM_HOME" FM_STATE_OVERRIDE="$STATE" \ diff --git a/bin/fm-merge-outcome-lib.sh b/bin/fm-merge-outcome-lib.sh index 849a0d54a25..54b27fdd8cc 100755 --- a/bin/fm-merge-outcome-lib.sh +++ b/bin/fm-merge-outcome-lib.sh @@ -8,8 +8,8 @@ # path. # # The destination is the home's role, never the caller's choice: -# - a secondmate home reports upward to its parent on the same reply channel -# bin/fm-inactive-reconcile.sh's report_to_parent already uses, in the same +# - a secondmate home reports upward on its parent channel, resolved and +# appended through bin/fm-parent-channel-lib.sh in the same # " [key=]: " shape the charter contract defines; # - a main home reports to the captain through the durable wake queue. # A poll observed in a secondmate home also receives a local durable wake after @@ -29,37 +29,8 @@ _FM_MERGE_OUTCOME_LIB_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck source=bin/fm-pr-lib.sh . "$_FM_MERGE_OUTCOME_LIB_DIR/fm-pr-lib.sh" -# shellcheck source=bin/fm-secondmate-parent-lib.sh -. "$_FM_MERGE_OUTCOME_LIB_DIR/fm-secondmate-parent-lib.sh" - -# The secondmate identity of the home reporting, or non-zero when this home is -# a main home (1) or carries an unusable identity marker (2). Mirrors -# bin/fm-inactive-reconcile.sh's home_secondmate_id, which owns the same -# marker's contract. -fm_merge_outcome_home_id() { # - local home=$1 marker id - marker="$home/.fm-secondmate-home" - if [ ! -e "$marker" ] && [ ! -L "$marker" ]; then - return 1 - fi - [ -f "$marker" ] && [ ! -L "$marker" ] || return 2 - [ "$(wc -c < "$marker")" -eq "$(LC_ALL=C tr -d '\0' < "$marker" | wc -c)" ] || return 2 - id=$(cat "$marker" 2>/dev/null) || return 2 - fm_pr_task_id_valid "$id" || return 2 - printf '%s\n' "$id" -} - -# Append to unless that exact line is already there, so a repeat -# report of the same merge cannot duplicate it. -fm_merge_outcome_append_once() { # - local path=$1 line=$2 - [ ! -L "$path" ] || return 1 - mkdir -p "$(dirname "$path")" || return 1 - if grep -Fqx -- "$line" "$path" 2>/dev/null; then - return 0 - fi - printf '%s\n' "$line" >> "$path" -} +# shellcheck source=bin/fm-parent-channel-lib.sh +. "$_FM_MERGE_OUTCOME_LIB_DIR/fm-parent-channel-lib.sh" # shellcheck disable=SC2034 # Public result consumed by sourcing callers. FM_MERGE_OUTCOME_ALREADY_RECORDED=false @@ -79,7 +50,7 @@ FM_MERGE_OUTCOME_ALREADY_RECORDED=false # than treat it as success: the merge landed and the record did not. fm_merge_outcome_report() { # local home=$1 state=$2 id=$3 url=$4 origin=$5 - local self='' self_rc=0 destination='' line lock status=0 + local self_rc=0 destination='' line lock status=0 local provider host path number # shellcheck disable=SC2034 # Sourced wake helpers consume these scoped globals. local STATE FM_WAKE_QUEUE FM_WAKE_QUEUE_LOCK @@ -93,20 +64,12 @@ fm_merge_outcome_report() { # number=$FM_PR_NUMBER [ -d "$state" ] && [ ! -L "$state" ] || return 1 - if self=$(fm_merge_outcome_home_id "$home"); then - fm_secondmate_parent_record_parse "$home/.fm-secondmate-parent" || return 3 - case "$FM_SECONDMATE_PARENT_ROUTE" in - local) - [ -n "$FM_SECONDMATE_PARENT_HOME" ] || return 3 - destination="$FM_SECONDMATE_PARENT_HOME/state/$self.status" - ;; - remote) destination="$state/parent-replies.status" ;; - *) return 3 ;; - esac + if destination=$(fm_parent_channel_destination "$home" "$state"); then line="done [key=merged-$id]: merged $id $FM_PR_URL" else self_rc=$? [ "$self_rc" -eq 1 ] || return 3 + destination='' fi STATE=$state @@ -123,7 +86,7 @@ fm_merge_outcome_report() { # fi if [ -n "$destination" ]; then - fm_merge_outcome_append_once "$destination" "$line" || status=1 + fm_parent_channel_append_once "$state" "$destination" "$line" || status=1 fi if [ "$status" -eq 0 ] && { [ "$origin" = poll ] || [ -z "$destination" ]; }; then fm_wake_append check "merged-$id-$FM_PR_URL" \ diff --git a/bin/fm-parent-channel-lib.sh b/bin/fm-parent-channel-lib.sh new file mode 100644 index 00000000000..39c9b549152 --- /dev/null +++ b/bin/fm-parent-channel-lib.sh @@ -0,0 +1,164 @@ +#!/usr/bin/env bash +# fm-parent-channel-lib.sh - the one owner of a secondmate home's parent channel. +# +# A secondmate home reports upward through exactly one channel, resolved from +# the home's own durable identity and parent binding rather than from any +# caller's choice (docs/secondmate-parent-channel.md): +# - the .fm-secondmate-home marker names the mate's id in its parent home; +# - the .fm-secondmate-parent record (bin/fm-secondmate-parent-lib.sh) names +# the route: a local route reports into the parent home's +# state/.status, a remote route into this home's own +# state/parent-replies.status, which the parent's remote reply adapter +# mirrors line for line into that same parent file. +# Every writer that publishes a parent-facing fact from inside a mate home - +# the merge outcome path, the inactive-outcome scan, the ledger mirror, the PR +# registration, the captain-hold record, and teardown's final sweep - resolves +# the destination here and appends through fm_parent_channel_report, so no +# writer can pick a different file, format a route by hand, or duplicate a +# line it already delivered. +# +# Lines follow the charter's " [key=]: " shape and are +# appended at most once by exact newline-terminated content; an incomplete +# destination tail is delimited before deduplication or append, so a retried +# publication after a crash never concatenates or duplicates channel events. +# +# Return codes (shared by every entry point that resolves the channel): +# 0 resolved, or appended / already present +# 1 this is a main home (no .fm-secondmate-home marker): nothing to report +# 2 the identity marker exists but is unusable (symlink, NUL, bad id) +# 3 the parent binding is missing or unreadable +# 4 the append itself failed +# A caller that has already recorded the outcome locally must surface a +# non-zero return rather than treat it as delivered. +# +# Sourced by bin/fm-merge-outcome-lib.sh, bin/fm-inactive-reconcile.sh, +# bin/fm-parent-mirror-lib.sh, bin/fm-pr-check.sh, bin/fm-captain-hold.sh, +# bin/fm-teardown.sh, and tests. No side effects on source. + +_FM_PARENT_CHANNEL_LIB_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=bin/fm-secondmate-parent-lib.sh +. "$_FM_PARENT_CHANNEL_LIB_DIR/fm-secondmate-parent-lib.sh" + +FM_PARENT_CHANNEL_LOCK_WAIT_SECS=${FM_PARENT_CHANNEL_LOCK_WAIT_SECS:-3} + +_fm_parent_channel_require_lock() { + if ! command -v fm_lock_acquire_wait_bounded >/dev/null 2>&1; then + # shellcheck source=bin/fm-wake-lib.sh + . "$_FM_PARENT_CHANNEL_LIB_DIR/fm-wake-lib.sh" + fi + _fm_wake_require_timeout +} + +# shellcheck disable=SC2034 # Output globals read by sourcing callers. +FM_PARENT_CHANNEL_ID= +# shellcheck disable=SC2034 # Output globals read by sourcing callers. +FM_PARENT_CHANNEL_ROUTE= + +# Same path-safety rule as bin/fm-pr-lib.sh's fm_task_id_path_safe, restated +# here so this library stays free of the PR library's larger surface. +_fm_parent_channel_id_valid() { # + local id=${1-} + local LC_ALL=C + case "$id" in + ''|.*|*[!A-Za-z0-9._-]*) return 1 ;; + esac +} + +# The secondmate identity of , printed, or non-zero for a main home (1) +# or an unusable identity marker (2). +fm_parent_channel_home_id() { # + local home=$1 marker id + marker="$home/.fm-secondmate-home" + if [ ! -e "$marker" ] && [ ! -L "$marker" ]; then + return 1 + fi + [ -f "$marker" ] && [ ! -L "$marker" ] || return 2 + [ "$(wc -c < "$marker")" -eq "$(LC_ALL=C tr -d '\0' < "$marker" | wc -c)" ] || return 2 + id=$(cat "$marker" 2>/dev/null) || return 2 + _fm_parent_channel_id_valid "$id" || return 2 + printf '%s\n' "$id" +} + +# Resolve the channel destination for whose state dir is . +# Prints the destination path and sets FM_PARENT_CHANNEL_ID and +# FM_PARENT_CHANNEL_ROUTE. Returns 1 for a main home, 2 for an unusable +# marker, 3 for a missing or unreadable parent binding. +fm_parent_channel_destination() { # + local home=$1 state=$2 id rc=0 + FM_PARENT_CHANNEL_ID= + FM_PARENT_CHANNEL_ROUTE= + id=$(fm_parent_channel_home_id "$home") || rc=$? + [ "$rc" -eq 0 ] || return "$rc" + fm_secondmate_parent_record_parse "$home/.fm-secondmate-parent" || return 3 + case "$FM_SECONDMATE_PARENT_ROUTE" in + local) + [ -n "$FM_SECONDMATE_PARENT_HOME" ] || return 3 + # shellcheck disable=SC2034 # Output globals read by sourcing callers. + FM_PARENT_CHANNEL_ID=$id + # shellcheck disable=SC2034 # Output globals read by sourcing callers. + FM_PARENT_CHANNEL_ROUTE=local + printf '%s/state/%s.status\n' "$FM_SECONDMATE_PARENT_HOME" "$id" + ;; + remote) + # shellcheck disable=SC2034 # Output globals read by sourcing callers. + FM_PARENT_CHANNEL_ID=$id + # shellcheck disable=SC2034 # Output globals read by sourcing callers. + FM_PARENT_CHANNEL_ROUTE=remote + printf '%s/parent-replies.status\n' "$state" + ;; + *) return 3 ;; + esac +} + +# Fold onto one line and bound it, so a note copied from a child ledger +# or a hold reason cannot break the channel's line framing. +fm_parent_channel_clean_note() { # + printf '%s' "$1" | LC_ALL=C tr '\t\r\n' ' ' | cut -c1-1200 +} + +_fm_parent_channel_destination_hash() { + if command -v shasum >/dev/null 2>&1; then + printf '%s' "$1" | shasum -a 256 | awk '{print $1}' + elif command -v sha256sum >/dev/null 2>&1; then + printf '%s' "$1" | sha256sum | awk '{print $1}' + else + printf '%s' "$1" | perl -MDigest::SHA=sha256_hex -0777 -ne 'print sha256_hex($_)' + fi +} + +# Append to unless that exact line is already there. +fm_parent_channel_append_once() { # + local STATE=$1 path=$2 line=$3 hash lock last_byte status=0 + [ -d "$STATE" ] && [ ! -L "$STATE" ] || return 1 + _fm_parent_channel_require_lock || return 1 + hash=$(_fm_parent_channel_destination_hash "$path") || return 1 + case "$hash" in ''|*[!0-9a-f]*) return 1 ;; esac + lock="$STATE/.parent-channel-$hash.lock" + fm_lock_acquire_wait_bounded "$lock" "$FM_PARENT_CHANNEL_LOCK_WAIT_SECS" || return 1 + if [ -e "$path" ] || [ -L "$path" ]; then + if [ ! -f "$path" ] || [ -L "$path" ]; then status=1; fi + elif ! mkdir -p "$(dirname "$path")"; then + status=1 + fi + if [ "$status" -eq 0 ] && [ -s "$path" ]; then + last_byte=$(tail -c 1 "$path" 2>/dev/null | od -An -tu1 | tr -d '[:space:]') || status=1 + if [ "$status" -eq 0 ] && [ "$last_byte" != 10 ] \ + && ! printf '\n' >> "$path"; then + status=1 + fi + fi + if [ "$status" -eq 0 ] && ! grep -Fqx -- "$line" "$path" 2>/dev/null \ + && ! printf '%s\n' "$line" >> "$path"; then + status=1 + fi + fm_lock_release "$lock" + return "$status" +} + +# Publish one parent-facing line from . See the return codes above. +fm_parent_channel_report() { # + local home=$1 state=$2 line=$3 destination rc=0 + destination=$(fm_parent_channel_destination "$home" "$state") || rc=$? + [ "$rc" -eq 0 ] || return "$rc" + fm_parent_channel_append_once "$state" "$destination" "$line" || return 4 +} diff --git a/bin/fm-parent-mirror-lib.sh b/bin/fm-parent-mirror-lib.sh new file mode 100644 index 00000000000..a54e17c8bec --- /dev/null +++ b/bin/fm-parent-mirror-lib.sh @@ -0,0 +1,784 @@ +#!/usr/bin/env bash +# fm-parent-mirror-lib.sh - deterministic mirror of a secondmate's child ledgers +# onto its parent channel. +# +# docs/secondmate-parent-channel.md owns the design and the delivery rule this +# library implements: the machinery reports facts, the mate reports judgement. +# Inside a secondmate home every captain-relevant event a direct child records +# in its append-only state/.status ledger is copied onto the parent +# channel (bin/fm-parent-channel-lib.sh) by this sweep, so a child's PR-ready, +# terminal, failed, or long-open decision line reaches the parent whether or +# not the mate model ever appends a line of its own. A main home is a silent +# no-op: it has no parent channel. +# +# Per-child durable record: state/parent-mirror/.record, key=value: +# schema=fm-parent-mirror.v1 +# offset= +# ident= +# incarnation= +# terminal_line= +# terminal_offset= +# terminal_first_seen= +# terminal_reported=0|1 +# tail=0|1 (1: captured bytes remain past the last complete line) +# orphan=0|1 (1: the child's record is gone but delivery is still owed) +# context_pr= +# context_mode= +# context_yolo= +# context_report=0|1 +# line_count= +# open=||||| +# The record is rewritten atomically; a changed ledger identity or a shrunk +# ledger resets the offset to 0, carries mirrored open decisions forward as +# pending closes, and clears terminal state from the invalidated identity. +# An incarnation change marks the prior generation's standing failure handled +# while a later failed line starts a new clock, and exact-line deduplication on +# the channel keeps a re-examined line from being delivered twice. A record +# whose child record is gone is an orphan and is swept until delivery completes. +# +# What is mirrored, and when: +# - a done line, or a legacy free-text captain-relevant line, immediately: +# done [key=mirror---l]: mirror: child= [pr=] [report=data//report.md] [mode=] [yolo=] +# - a failed line, once it has stood as the ledger's last line for +# FM_PARENT_MIRROR_OPEN_SECS (default 600, valid 60..86400), or immediately +# when the child has retired and no first responder remains: +# failed [key=mirror---l]: mirror: child= (unhandled past s) +# - a needs-decision or blocked decision, once it has stayed open in the +# child's fold (bin/fm-classify-lib.sh's status_open_decisions) for that +# same threshold without a resolution or captain-held transfer: +# [key=mirror---]: mirror: child= decision (opened at line ) open past s without an answer or a captain hold: +# A retiring or orphaned child's still-open decision is raised immediately, +# because its first-responder window has ended. Its close, when the child's +# fold no longer holds the key, is: +# resolved [key=mirror---]: mirror: child= decision (opened at line ) closed +# An opening is identified by its key and the ledger line that opened it, +# so a key re-opened after a close is a new delivery even with the same +# note, while the exact-line deduplication still holds for one opening. +# - a decision line the fold cannot track (invalid or reserved key) is +# delivered immediately under the done verb with an explicit note, because +# an untrackable open must never open a parent decision nothing can close. +# The threshold is what keeps the mate the first responder: a decision it +# answers, a blocker it clears, or a failure it relaunches inside the window +# is never raised to the parent, and one it leaves standing is. Line text is +# deterministic for one event so a replayed sweep after a lost record cannot +# deliver it twice. +# +# Ordering and safety: each child is examined under its own meta lock, the +# same lock teardown and relaunch hold, so a record is never read while it is +# being replaced; the *_locked entry points are for callers that already hold +# that meta lock and take no other lock, so the lock order is always meta lock +# first and a teardown can never wait on a sweep. A sweep serializes with other +# sweeps through state/.parent-mirror.lock, and every wait is bounded: a child +# whose lock is busy, or a sweep whose lock is busy, is simply left for the +# next poll, so the watcher's beacon is never held hostage by a long teardown. +# Only whole lines are examined, so a line still being appended is left for +# the next sweep. Nothing here reads a pane, calls a harness, a forge, or +# bin/fm-crew-state.sh. Every examination runs in its own subshell, so a +# caller under set -e is never aborted by the sweep's own bookkeeping. +# +# Return codes for the sweep entry points: 0 delivered or nothing to deliver, +# 2 unusable identity marker, 3 unreadable parent binding, 4 an append or +# record write failed (the record keeps the undelivered position for the next +# sweep), and 5 a targeted sweep deferred by lock contention. Channel and +# delivery failures 2 through 4 are queued once per unhandled episode as a +# durable check wake in this home, and printed on stdout only when newly +# queued, so an unreportable home is loud rather than quietly silent and a +# standing problem does not wake the mate on every poll. +# +# Sourced by bin/fm-parent-mirror.sh, bin/fm-pr-check.sh, bin/fm-teardown.sh, +# bin/fm-inactive-reconcile.sh, and tests. No side effects on source. + +_FM_PARENT_MIRROR_LIB_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=bin/fm-parent-channel-lib.sh +. "$_FM_PARENT_MIRROR_LIB_DIR/fm-parent-channel-lib.sh" +# shellcheck source=bin/fm-wake-lib.sh +. "$_FM_PARENT_MIRROR_LIB_DIR/fm-wake-lib.sh" +_fm_wake_require_classify +_fm_wake_require_timeout + +FM_PARENT_MIRROR_SCHEMA='fm-parent-mirror.v1' +FM_PARENT_MIRROR_OPEN_SECS_DEFAULT=600 +# Bounded lock waits: a busy child or sweep is retried on the next poll. +FM_PARENT_MIRROR_LOCK_WAIT_SECS=${FM_PARENT_MIRROR_LOCK_WAIT_SECS:-3} + +fm_parent_mirror_dir() { # + printf '%s/parent-mirror\n' "$1" +} + +fm_parent_mirror_record_path() { # + printf '%s/parent-mirror/%s.record\n' "$1" "$2" +} + +fm_parent_mirror_lock_path() { # + printf '%s/.parent-mirror.lock\n' "$1" +} + +_fm_parent_mirror_key() { # + printf 'mirror-%s-%s-%s' "${#1}" "$1" "$2" +} + +_fm_parent_mirror_now() { + case "${FM_PARENT_MIRROR_NOW:-}" in + ''|*[!0-9]*) date +%s ;; + *) printf '%s\n' "$FM_PARENT_MIRROR_NOW" ;; + esac +} + +# The open threshold, validated on every call so a bad override is loud. +fm_parent_mirror_open_secs() { + local secs=${FM_PARENT_MIRROR_OPEN_SECS:-$FM_PARENT_MIRROR_OPEN_SECS_DEFAULT} + case "$secs" in + ''|*[!0-9]*) return 1 ;; + esac + [ "$secs" -ge 60 ] && [ "$secs" -le 86400 ] || return 1 + printf '%s\n' "$secs" +} + +_fm_parent_mirror_meta_field() { # + [ -n "$1" ] && [ -f "$1" ] && [ ! -L "$1" ] || return 0 + grep "^$2=" "$1" 2>/dev/null | tail -1 | cut -d= -f2- || true +} + +_fm_parent_mirror_record_field() { # + [ -f "$1" ] && [ ! -L "$1" ] || return 0 + grep "^$2=" "$1" 2>/dev/null | tail -1 | cut -d= -f2- || true +} + +_fm_parent_mirror_record_open_lines() { # + [ -f "$1" ] && [ ! -L "$1" ] || return 0 + grep '^open=' "$1" 2>/dev/null | cut -d= -f2- || true +} + +# 0 when the child's ledger already states its own outcome: it ends in a +# terminal captain verb. That evidence belongs to this mirror, so the +# current-state inactive-outcome scan yields it (bin/fm-inactive-reconcile.sh). +fm_parent_mirror_owns_ledger() { # + local state=$1 child=$2 status size captured prefix complete_size=0 line last rc=1 + local LC_ALL=C + status="$state/$child.status" + [ -f "$status" ] && [ ! -L "$status" ] || return 1 + size=$(_fm_status_file_size "$status") || return 1 + size=${size//[[:space:]]/} + case "$size" in ''|*[!0-9]*) return 1 ;; esac + captured=$(mktemp "$state/.parent-mirror-owns.XXXXXX") || return 1 + prefix=$(mktemp "$state/.parent-mirror-prefix.XXXXXX") || { rm -f "$captured"; return 1; } + if _fm_status_read_span "$status" 0 "$size" > "$captured" 2>/dev/null; then + while IFS= read -r line; do + complete_size=$((complete_size + ${#line} + 1)) + done < "$captured" + if _fm_status_read_span "$captured" 0 "$complete_size" > "$prefix" 2>/dev/null; then + last=$(last_status_line "$prefix") + status_is_terminal_verb "$last" && rc=0 + fi + fi + rm -f "$captured" "$prefix" + return "$rc" +} + +# Exact-key lookups over the TAB-separated fold and the |-separated open set. +_fm_parent_mirror_fold_has() { # + local line + while IFS= read -r line; do + case "$line" in "$2"$'\t'*) return 0 ;; esac + done < -> "\t" + local line + while IFS= read -r line; do + case "$line" in "$2"$'\t'*) printf '%s' "${line#*$'\t'}"; return 0 ;; esac + done < -> "|||||" + local line + while IFS= read -r line; do + case "$line" in "$2|"*) printf '%s' "$line"; return 0 ;; esac + done < -> opening line number + local line + while IFS= read -r line; do + case "$line" in "$2"$'\t'*) printf '%s' "${line#*$'\t'}"; return 0 ;; esac + done < + local rc=$1 key payload append_rc=0 + case "$rc" in + 2) key='parent-mirror-diagnostic:channel' + payload='parent channel unavailable: invalid .fm-secondmate-home marker; child outcomes are not reaching the parent' ;; + 3) key='parent-mirror-diagnostic:channel' + payload='parent channel unavailable: missing or unreadable parent binding .fm-secondmate-parent; child outcomes are not reaching the parent' ;; + 4) key='parent-mirror-diagnostic:delivery' + payload='parent channel delivery failed; a child outcome is retained for retry but has not reached the parent' ;; + *) return 0 ;; + esac + fm_wake_append_if_key_absent_bounded check "$key" "check: $payload" \ + "$FM_PARENT_MIRROR_LOCK_WAIT_SECS" >/dev/null 2>&1 || append_rc=$? + case "$append_rc" in + 0) printf 'actionable: %s\n' "$payload" ;; + 3) ;; + *) printf '%s\n' "$payload" >&2 ;; + esac +} + +_fm_parent_mirror_dir_ready() { # + local dir=$1 + if [ -e "$dir" ] || [ -L "$dir" ]; then + [ -d "$dir" ] && [ ! -L "$dir" ] + return + fi + mkdir -p "$dir" || return 1 + [ -d "$dir" ] && [ ! -L "$dir" ] +} + +# Rewrite atomically from the fields the caller assembled. +_fm_parent_mirror_record_write() { # [ ] + local record=$1 offset=$2 ident=$3 incarnation=$4 terminal_line=$5 terminal_offset=$6 + local terminal_first_seen=$7 terminal_reported=$8 tail=$9 orphan=${10} open_lines=${11} + local context_pr=${12:-} context_mode=${13:-} context_yolo=${14:-} context_report=${15:-0} line_count=${16:-0} dir tmp entry + dir=$(dirname "$record") + _fm_parent_mirror_dir_ready "$dir" || return 1 + tmp=$(mktemp "$dir/.record.XXXXXX") || return 1 + { + printf 'schema=%s\n' "$FM_PARENT_MIRROR_SCHEMA" + printf 'offset=%s\n' "$offset" + printf 'ident=%s\n' "$ident" + printf 'incarnation=%s\n' "$incarnation" + printf 'terminal_line=%s\n' "$terminal_line" + printf 'terminal_offset=%s\n' "$terminal_offset" + printf 'terminal_first_seen=%s\n' "$terminal_first_seen" + printf 'terminal_reported=%s\n' "$terminal_reported" + printf 'tail=%s\n' "$tail" + printf 'orphan=%s\n' "$orphan" + printf 'context_pr=%s\n' "$context_pr" + printf 'context_mode=%s\n' "$context_mode" + printf 'context_yolo=%s\n' "$context_yolo" + printf 'context_report=%s\n' "$context_report" + printf 'line_count=%s\n' "$line_count" + while IFS= read -r entry; do + [ -n "$entry" ] || continue + printf 'open=%s\n' "$entry" + done < "$tmp" || { rm -f "$tmp"; return 1; } + chmod 600 "$tmp" 2>/dev/null || true + mv -f "$tmp" "$record" || { rm -f "$tmp"; return 1; } +} + +# The context suffix every mirrored line carries: the child's recorded PR, +# its scout report when one exists, and its delivery mode and merge posture. +_fm_parent_mirror_context() { # + local child=$1 pr=$2 mode=$3 yolo=$4 report=$5 out='' + [ -z "$pr" ] || out="$out pr=$(fm_parent_channel_clean_note "$pr")" + [ "$report" != 1 ] || out="$out report=data/$child/report.md" + [ -z "$mode" ] || out="$out mode=$(fm_parent_channel_clean_note "$mode")" + [ -z "$yolo" ] || out="$out yolo=$(fm_parent_channel_clean_note "$yolo")" + printf '%s' "$out" +} + +_fm_parent_mirror_publish() { # + fm_parent_channel_report "$FM_HOME" "$STATE" "$1" +} + +# Examine one child ledger and deliver what it owes. is the +# child's record when it still exists. Returns the codes documented above. +# Runs in the caller's shell; the entry points below wrap it in a subshell. +_fm_parent_mirror_child() { # [] + local child=$1 meta=$2 orphan=$3 deliver_now=${4:-0} status record now open_secs + local size ident offset rec_ident incarnation rec_incarnation complete_size=0 scan_start=0 span_complete=0 + local terminal_line terminal_offset terminal_first_seen terminal_reported tail=0 captured='' prefix='' dir + local open_lines='' next_open='' close_failed='' reset_open='' fold='' rc=0 rec_line_count=0 line_count=0 processed_lines=0 + local chunk line verb note key mirror_key line_offset line_start committed context last last_verb last_offset + local entry entry_key entry_seen entry_mirrored entry_origin entry_verb entry_note origins origin fold_verb fold_note age timing + local line_number resolve held after was_open + local context_pr context_mode context_yolo context_report report data + local LC_ALL=C + status="$STATE/$child.status" + record=$(fm_parent_mirror_record_path "$STATE" "$child") + dir=$(dirname "$record") + _fm_parent_mirror_dir_ready "$dir" || return 4 + now=$(_fm_parent_mirror_now) + open_secs=$(fm_parent_mirror_open_secs) || { + printf 'FM_PARENT_MIRROR_OPEN_SECS must be a whole number from 60 to 86400\n' >&2 + return 4 + } + if [ ! -e "$status" ]; then + [ "$orphan" -ne 1 ] || return 4 + return 0 + fi + [ -f "$status" ] && [ ! -L "$status" ] || return 4 + ident=$(_fm_open_decisions_file_ident "$status") || return 4 + size=$(_fm_status_file_size "$status") || return 4 + size=${size//[[:space:]]/} + case "$size" in ''|*[!0-9]*) return 4 ;; esac + + offset=$(_fm_parent_mirror_record_field "$record" offset) + rec_ident=$(_fm_parent_mirror_record_field "$record" ident) + rec_incarnation=$(_fm_parent_mirror_record_field "$record" incarnation) + terminal_line=$(_fm_parent_mirror_record_field "$record" terminal_line) + terminal_offset=$(_fm_parent_mirror_record_field "$record" terminal_offset) + terminal_first_seen=$(_fm_parent_mirror_record_field "$record" terminal_first_seen) + terminal_reported=$(_fm_parent_mirror_record_field "$record" terminal_reported) + open_lines=$(_fm_parent_mirror_record_open_lines "$record") + rec_line_count=$(_fm_parent_mirror_record_field "$record" line_count) + case "$offset" in ''|*[!0-9]*) offset=0 ;; esac + case "$rec_line_count" in ''|*[!0-9]*) rec_line_count=0; offset=0; open_lines='' ;; esac + case "$terminal_reported" in 1) ;; *) terminal_reported=0 ;; esac + if [ "$rec_ident" != "$ident" ] || [ "$offset" -gt "$size" ]; then + reset_open='' + while IFS='|' read -r entry_key entry_seen entry_mirrored entry_origin entry_verb entry_note; do + [ -n "$entry_key" ] && [ "$entry_mirrored" = 1 ] || continue + reset_open="${reset_open}${entry_key}|${entry_seen}|${entry_mirrored}|${entry_origin}|closed|${entry_note}"$'\n' + done < "$captured" 2>/dev/null; then + rm -f "$captured" "$prefix" + return 4 + fi + while IFS= read -r line; do + span_complete=$((span_complete + ${#line} + 1)) + done < "$captured" + if ! _fm_status_read_span "$captured" 0 "$span_complete" > "$prefix" 2>/dev/null; then + rm -f "$captured" "$prefix" + return 4 + fi + rm -f "$captured" + complete_size=$((scan_start + span_complete)) + [ "$complete_size" -eq "$size" ] || tail=1 + incarnation=$(_fm_parent_mirror_meta_field "$meta" spawn_gen) + [ -n "$incarnation" ] || incarnation=$rec_incarnation + if [ -n "$rec_incarnation" ] && [ "$incarnation" != "$rec_incarnation" ] \ + && [ "$rec_ident" = "$ident" ]; then + terminal_reported=1 + fi + + context_pr=$(_fm_parent_mirror_record_field "$record" context_pr) + context_mode=$(_fm_parent_mirror_record_field "$record" context_mode) + context_yolo=$(_fm_parent_mirror_record_field "$record" context_yolo) + context_report=$(_fm_parent_mirror_record_field "$record" context_report) + case "$context_report" in 1) ;; *) context_report=0 ;; esac + if [ -n "$meta" ] && [ -f "$meta" ] && [ ! -L "$meta" ]; then + context_pr=$(_fm_parent_mirror_meta_field "$meta" pr) + context_mode=$(_fm_parent_mirror_meta_field "$meta" mode) + context_yolo=$(_fm_parent_mirror_meta_field "$meta" yolo) + data="${FM_DATA_OVERRIDE:-$FM_HOME/data}" + report="$data/$child/report.md" + if [ -f "$report" ] && [ ! -L "$report" ]; then context_report=1; else context_report=0; fi + fi + context=$(_fm_parent_mirror_context "$child" "$context_pr" "$context_mode" "$context_yolo" "$context_report") + committed=$offset + + # 1. New whole lines since the cursor: done-class lines deliver now. + if [ "$offset" -lt "$complete_size" ]; then + chunk=$(mktemp "$dir/.span.XXXXXX") || { rm -f "$prefix"; return 4; } + if ! _fm_status_read_span "$prefix" 0 "$span_complete" > "$chunk" 2>/dev/null; then + rm -f "$chunk" "$prefix" + return 4 + fi + line_offset=$offset + while IFS= read -r line; do + line_start=$line_offset + line_offset=$((line_offset + ${#line} + 1)) + case "$line" in *[![:space:]]*) ;; *) committed=$line_offset; processed_lines=$((processed_lines + 1)); continue ;; esac + if ! status_is_captain_relevant "$line"; then + committed=$line_offset + processed_lines=$((processed_lines + 1)) + continue + fi + verb=$(status_line_verb "$line") + note=$(fm_parent_channel_clean_note "$(status_line_note "$line")") + case "$verb" in + failed) + committed=$line_offset + processed_lines=$((processed_lines + 1)) + continue + ;; + needs-decision|blocked) + if key=$(_fm_decision_key "$line") \ + && _fm_decision_key_transition_allowed "$key" "$(status_line_note "$line")"; then + committed=$line_offset + processed_lines=$((processed_lines + 1)) + continue + fi + mirror_key=$(_fm_parent_mirror_key "$child" "l$line_start") + _fm_parent_mirror_publish \ + "done [key=$mirror_key]: mirror: child=$child untracked $verb line: $note$context" \ + || { rc=$?; break; } + ;; + done) + mirror_key=$(_fm_parent_mirror_key "$child" "l$line_start") + _fm_parent_mirror_publish \ + "done [key=$mirror_key]: mirror: child=$child $note$context" \ + || { rc=$?; break; } + ;; + *) + mirror_key=$(_fm_parent_mirror_key "$child" "l$line_start") + _fm_parent_mirror_publish \ + "done [key=$mirror_key]: mirror: child=$child $(fm_parent_channel_clean_note "$line")$context" \ + || { rc=$?; break; } + ;; + esac + committed=$line_offset + processed_lines=$((processed_lines + 1)) + done < "$chunk" + rm -f "$chunk" + fi + + line_count=$((rec_line_count + processed_lines)) + + # 2. A standing failed line: deliver once it has stood past the threshold. + if [ "$rc" -eq 0 ]; then + last=$terminal_line + last_offset=$terminal_offset + line_offset=$scan_start + line_number=0 + while IFS= read -r line && [ "$line_number" -lt "$processed_lines" ]; do + line_start=$line_offset + line_offset=$((line_offset + ${#line} + 1)) + line_number=$((line_number + 1)) + case "$line" in + *[![:space:]]*) last=$line; last_offset=$line_start ;; + esac + done < "$prefix" + last_verb=$(status_line_verb "$last") + if [ "$last_verb" = failed ]; then + if [ "$terminal_line" != "$last" ] || [ "$terminal_offset" != "$last_offset" ]; then + terminal_line=$last + terminal_offset=$last_offset + terminal_first_seen=$now + terminal_reported=0 + fi + case "$terminal_first_seen" in ''|*[!0-9]*) terminal_first_seen=$now ;; esac + age=$((now - terminal_first_seen)) + if [ "$terminal_reported" -eq 0 ] \ + && { [ "$deliver_now" -eq 1 ] || [ "$age" -ge "$open_secs" ]; }; then + note=$(fm_parent_channel_clean_note "$(status_line_note "$last")") + if [ "$deliver_now" -eq 1 ]; then timing='unhandled when child retired'; else timing="unhandled past ${open_secs}s"; fi + mirror_key=$(_fm_parent_mirror_key "$child" "l$terminal_offset") + if _fm_parent_mirror_publish \ + "failed [key=$mirror_key]: mirror: child=$child $note ($timing)$context"; then + terminal_reported=1 + else + rc=4 + fi + fi + else + terminal_line='' + terminal_offset='' + terminal_first_seen='' + terminal_reported=0 + fi + fi + + # 3. Fold only the new committed lines onto the compact durable open set. + origins='' + while IFS='|' read -r entry_key entry_seen entry_mirrored entry_origin entry_verb entry_note; do + [ -n "$entry_key" ] || continue + if [ "$entry_verb" != closed ]; then + [ -n "$fold" ] && fold="${fold}"$'\n' + fold="${fold}${entry_key}"$'\t'"${entry_verb}"$'\t'"${entry_note}" + [ -n "$origins" ] && origins="${origins}"$'\n' + origins="${origins}${entry_key}"$'\t'"${entry_origin}" + fi + done < [] + local child=$1 report_contention=${2:-0} meta lock rc=0 record + meta="$STATE/$child.meta" + record=$(fm_parent_mirror_record_path "$STATE" "$child") + if [ -f "$meta" ] && [ ! -L "$meta" ]; then + [ "$(_fm_parent_mirror_meta_field "$meta" kind)" != secondmate ] || return 0 + lock=$(fm_meta_lock_path "$meta") || return 4 + # A busy child (teardown or relaunch holds its record) is left for the + # next poll rather than waited on; nothing is lost, the ledger stays. + if ! fm_lock_acquire_wait_bounded "$lock" "$FM_PARENT_MIRROR_LOCK_WAIT_SECS"; then + [ "$report_contention" -eq 1 ] && return 5 + return 0 + fi + ( set +e; _fm_parent_mirror_child "$child" "$meta" 0 ) || rc=$? + fm_lock_release "$lock" + return "$rc" + fi + [ -f "$record" ] && [ ! -L "$record" ] || return 0 + ( set +e; _fm_parent_mirror_child "$child" '' 1 1 ) || rc=$? + if [ "$rc" -eq 0 ] && [ "$(_fm_parent_mirror_record_field "$record" tail)" != 1 ]; then + if status_retire_presentation_task "$STATE" "$child" "$FM_PARENT_MIRROR_LOCK_WAIT_SECS"; then + rm -f "$record" + else + rc=4 + fi + fi + return "$rc" +} + +# Same as above for a caller that already holds the child's meta lock. +fm_parent_mirror_sweep_child_locked() { # [] + local child=$1 deliver_now=${2:-0} meta rc=0 + meta="$STATE/$child.meta" + if [ -f "$meta" ] && [ ! -L "$meta" ]; then + [ "$(_fm_parent_mirror_meta_field "$meta" kind)" != secondmate ] || return 0 + ( set +e; _fm_parent_mirror_child "$child" "$meta" 0 "$deliver_now" ) || rc=$? + return "$rc" + fi + ( set +e; _fm_parent_mirror_child "$child" '' 1 1 ) || rc=$? + return "$rc" +} + +# Resolve the channel once for a sweep; a main home returns 1 silently and an +# unresolvable channel is reported and returned as 2 or 3. +_fm_parent_mirror_channel_ready() { + local rc=0 + fm_parent_channel_destination "$FM_HOME" "$STATE" >/dev/null || rc=$? + case "$rc" in + 0) return 0 ;; + 1) return 1 ;; + *) + _fm_parent_mirror_diagnostic "$rc" + return "$rc" + ;; + esac +} + +# Sweep every direct child and every orphan record under the sweep lock. +# With , sweep only that child. A main home returns 0 silently. +fm_parent_mirror_sweep() { # [] + local only=${1:-} lock rc=0 child_rc meta child record + _fm_parent_mirror_channel_ready || { rc=$?; [ "$rc" -eq 1 ] && return 0; return "$rc"; } + lock=$(fm_parent_mirror_lock_path "$STATE") + # Another sweep is already delivering; this poll has nothing to add. + if ! fm_lock_acquire_wait_bounded "$lock" "$FM_PARENT_MIRROR_LOCK_WAIT_SECS"; then + [ -n "$only" ] && return 5 + return 0 + fi + if [ -n "$only" ]; then + fm_parent_mirror_sweep_child "$only" 1 || rc=$? + else + for meta in "$STATE"/*.meta; do + [ -f "$meta" ] || continue + child=$(basename "$meta" .meta) + _fm_parent_channel_id_valid "$child" || continue + child_rc=0 + fm_parent_mirror_sweep_child "$child" || child_rc=$? + [ "$child_rc" -eq 0 ] || rc=$child_rc + done + for record in "$(fm_parent_mirror_dir "$STATE")"/*.record; do + [ -f "$record" ] || continue + child=$(basename "$record" .record) + _fm_parent_channel_id_valid "$child" || continue + [ -f "$STATE/$child.meta" ] && continue + child_rc=0 + fm_parent_mirror_sweep_child "$child" || child_rc=$? + [ "$child_rc" -eq 0 ] || rc=$child_rc + done + fi + fm_lock_release "$lock" + [ "$rc" -eq 0 ] || _fm_parent_mirror_diagnostic "$rc" + return "$rc" +} + +fm_parent_mirror_orphan_durable() { # + local child=$1 dir record + _fm_parent_channel_id_valid "$child" || return 1 + dir=$(fm_parent_mirror_dir "$STATE") + record=$(fm_parent_mirror_record_path "$STATE" "$child") + [ -d "$dir" ] && [ ! -L "$dir" ] || return 1 + [ -f "$record" ] && [ ! -L "$record" ] || return 1 + [ "$(_fm_parent_mirror_record_field "$record" schema)" = "$FM_PARENT_MIRROR_SCHEMA" ] || return 1 + [ "$(_fm_parent_mirror_record_field "$record" orphan)" = 1 ] +} + +# Final sweep for a child that is leaving the home, then retire its record. +# The caller holds the child's meta lock, which is the only lock this path +# takes. When the final sweep cannot deliver for any reason, a record is kept +# (or created) as an orphan so later sweeps keep trying, because the ledger +# outlives the child's record. +fm_parent_mirror_retire_locked() { # + local child=$1 record meta status rc=0 context_pr context_mode context_yolo context_report report data + record=$(fm_parent_mirror_record_path "$STATE" "$child") + meta="$STATE/$child.meta" + status="$STATE/$child.status" + # A pre-ledger task has no parent-facing evidence to preserve or deliver. + # Avoid resolving a channel merely because an unrelated teardown is running. + if { { [ ! -e "$status" ] && [ ! -L "$status" ]; } \ + || { [ -f "$status" ] && [ ! -L "$status" ] && [ ! -s "$status" ]; }; } \ + && [ ! -e "$record" ] && [ ! -L "$record" ]; then + return 0 + fi + _fm_parent_mirror_channel_ready || { rc=$?; [ "$rc" -eq 1 ] && return 0; } + if [ "$rc" -eq 0 ]; then + fm_parent_mirror_sweep_child_locked "$child" 1 || rc=$? + fi + if [ "$rc" -eq 0 ] && [ "$(_fm_parent_mirror_record_field "$record" tail)" != 1 ]; then + rm -f "$record" + return 0 + fi + if [ -e "$STATE/$child.status" ]; then + context_pr=$(_fm_parent_mirror_record_field "$record" context_pr) + context_mode=$(_fm_parent_mirror_record_field "$record" context_mode) + context_yolo=$(_fm_parent_mirror_record_field "$record" context_yolo) + context_report=$(_fm_parent_mirror_record_field "$record" context_report) + if [ -f "$meta" ] && [ ! -L "$meta" ]; then + context_pr=$(_fm_parent_mirror_meta_field "$meta" pr) + context_mode=$(_fm_parent_mirror_meta_field "$meta" mode) + context_yolo=$(_fm_parent_mirror_meta_field "$meta" yolo) + data="${FM_DATA_OVERRIDE:-$FM_HOME/data}" + report="$data/$child/report.md" + if [ -f "$report" ] && [ ! -L "$report" ]; then context_report=1; else context_report=0; fi + fi + if ! _fm_parent_mirror_record_write "$record" \ + "$(_fm_parent_mirror_record_field "$record" offset)" \ + "$(_fm_parent_mirror_record_field "$record" ident)" \ + "$(_fm_parent_mirror_record_field "$record" incarnation)" \ + "$(_fm_parent_mirror_record_field "$record" terminal_line)" \ + "$(_fm_parent_mirror_record_field "$record" terminal_offset)" \ + "$(_fm_parent_mirror_record_field "$record" terminal_first_seen)" \ + "$(_fm_parent_mirror_record_field "$record" terminal_reported)" \ + "$(_fm_parent_mirror_record_field "$record" tail)" \ + 1 \ + "$(_fm_parent_mirror_record_open_lines "$record")" \ + "$context_pr" "$context_mode" "$context_yolo" "$context_report" \ + "$(_fm_parent_mirror_record_field "$record" line_count)"; then + [ "$rc" -ne 0 ] || rc=4 + fi + fi + return "$rc" +} diff --git a/bin/fm-parent-mirror.sh b/bin/fm-parent-mirror.sh new file mode 100755 index 00000000000..f5c06f8887b --- /dev/null +++ b/bin/fm-parent-mirror.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# fm-parent-mirror.sh - mirror a secondmate's child ledgers onto its parent channel. +# +# Usage: +# fm-parent-mirror.sh sweep [--child ] +# fm-parent-mirror.sh owns-ledger +# +# `sweep` examines every direct child ledger in this home (or one child) and +# delivers every captain-relevant event that has not reached the parent +# channel yet; bin/fm-parent-mirror-lib.sh owns what is mirrored, when, and +# the durable per-child record under state/parent-mirror/. In a main home it +# is a silent no-op. The watcher (bin/fm-watch.sh) runs it on every poll, +# bin/fm-pr-check.sh runs it for a child whose PR it just registered, and +# bin/fm-teardown.sh runs the same library in-process before a child's record +# is removed. Output is empty on a quiet sweep and prints an `actionable:` line +# only when a channel or delivery diagnostic is newly queued; the exit status +# is the library's return code, including 5 for targeted lock contention. +# +# `owns-ledger` exits 0 when the child's ledger ends in a terminal captain +# verb, the evidence this mirror delivers on its own clock; the inactive +# outcome scan uses it to yield. +# +# Tunables (env): FM_PARENT_MIRROR_OPEN_SECS (default 600, valid 60..86400), +# the standing time after which a failed line, an open decision, or an open +# blocker in a child ledger is raised to the parent. +set -u +export LC_ALL=C + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +FM_HOME="${FM_HOME:-${FM_ROOT_OVERRIDE:-$(cd "$SCRIPT_DIR/.." && pwd)}}" +STATE="${FM_STATE_OVERRIDE:-$FM_HOME/state}" +export FM_HOME STATE + +# shellcheck source=bin/fm-parent-mirror-lib.sh +. "$SCRIPT_DIR/fm-parent-mirror-lib.sh" + +usage() { + sed -n '2,25{s/^# \{0,1\}//;p;}' "$0" +} + +mode=${1:-} +case "$mode" in + sweep) + shift + child='' + while [ "$#" -gt 0 ]; do + case "$1" in + --child) + [ "$#" -ge 2 ] || { usage >&2; exit 2; } + child=$2 + shift 2 + ;; + *) usage >&2; exit 2 ;; + esac + done + if [ -n "$child" ] && ! _fm_parent_channel_id_valid "$child"; then + printf 'error: invalid child id\n' >&2 + exit 2 + fi + fm_parent_mirror_sweep "$child" + ;; + owns-ledger) + [ "$#" -eq 2 ] || { usage >&2; exit 2; } + _fm_parent_channel_id_valid "$2" || { printf 'error: invalid child id\n' >&2; exit 2; } + fm_parent_mirror_owns_ledger "$STATE" "$2" + ;; + -h|--help) + usage + ;; + *) + usage >&2 + exit 2 + ;; +esac diff --git a/bin/fm-pr-check.sh b/bin/fm-pr-check.sh index 198755207f7..bcf0e3d25b6 100755 --- a/bin/fm-pr-check.sh +++ b/bin/fm-pr-check.sh @@ -17,6 +17,8 @@ STATE="${FM_STATE_OVERRIDE:-$FM_HOME/state}" . "$SCRIPT_DIR/fm-pr-lib.sh" # shellcheck source=bin/fm-wake-lib.sh . "$SCRIPT_DIR/fm-wake-lib.sh" +# shellcheck source=bin/fm-parent-mirror-lib.sh +. "$SCRIPT_DIR/fm-parent-mirror-lib.sh" if [ "$#" -ne 2 ]; then echo "error: invalid PR check request" >&2 @@ -132,4 +134,16 @@ fm_pr_poll_publish_prepared || { echo "error: could not publish PR poll" >&2 exit 1 } +# In a secondmate home the registration itself is a parent-facing fact: deliver +# the child's ready line now, carrying the canonical pr= just recorded, instead +# of on the next watcher poll (docs/secondmate-parent-channel.md). A main home +# has no parent channel and this is a silent no-op there. The poll is armed +# either way; a delivery problem is reported, queued once as a durable wake by +# the library, and retried by the next supervision poll. +MIRROR_RC=0 +fm_parent_mirror_sweep "$ID" || MIRROR_RC=$? +if [ "$MIRROR_RC" -ne 0 ]; then + printf 'actionable: PR %s is registered but its ready line did not reach the parent channel (rc=%s); the next supervision poll retries\n' \ + "$URL" "$MIRROR_RC" >&2 +fi printf 'armed: state/%s.check.sh\n' "$ID" diff --git a/bin/fm-teardown.sh b/bin/fm-teardown.sh index ad9e042ba11..ec33c885c26 100755 --- a/bin/fm-teardown.sh +++ b/bin/fm-teardown.sh @@ -185,6 +185,8 @@ SUB_HOME_PARENT_MARKER=".fm-secondmate-parent" . "$SCRIPT_DIR/fm-secondmate-registry-lib.sh" # shellcheck source=bin/fm-secondmate-parent-lib.sh . "$SCRIPT_DIR/fm-secondmate-parent-lib.sh" +# shellcheck source=bin/fm-parent-mirror-lib.sh +. "$SCRIPT_DIR/fm-parent-mirror-lib.sh" # shellcheck source=bin/fm-pending-reply-lib.sh . "$SCRIPT_DIR/fm-pending-reply-lib.sh" # shellcheck source=bin/fm-nm-run-lib.sh @@ -2871,7 +2873,28 @@ fm_backend_clear_transition "$BACKEND" "$STATE" "$T" || true [ -n "$TASK_TMP" ] && rm -rf "$TASK_TMP" remove_pr_poll_artifacts "$STATE" "$ID" || exit 1 retire_busy_state "$STATE" "$ID" "$BUSY_GEN" || exit 1 -status_retire_presentation_task "$STATE" "$ID" || exit 1 +# In a secondmate home, deliver whatever this child's ledger still owes the +# parent before the child's record disappears, then retire its mirror state +# (docs/secondmate-parent-channel.md). The ledger itself survives teardown, so +# an undelivered final sweep leaves an orphan record that later sweeps retry; +# a main home has no channel and this is a silent no-op there. +MIRROR_RETAINED=0 +if [ "$KIND" != secondmate ]; then + MIRROR_RC=0 + fm_parent_mirror_retire_locked "$ID" || MIRROR_RC=$? + if fm_parent_mirror_orphan_durable "$ID"; then + MIRROR_RETAINED=1 + elif [ "$MIRROR_RC" -ne 0 ]; then + echo "error: $ID's final outcome did not reach the parent channel (rc=$MIRROR_RC), and no durable orphan record exists; retaining the child record for retry" >&2 + exit 1 + fi + if [ "$MIRROR_RC" -ne 0 ]; then + echo "actionable: $ID's final outcome did not reach the parent channel (rc=$MIRROR_RC); its durable orphan record and ledger are retained for the next supervision poll" >&2 + fi +fi +if [ "$MIRROR_RETAINED" -ne 1 ]; then + status_retire_presentation_task "$STATE" "$ID" || exit 1 +fi rm -f "$STATE/$ID.turn-ended" \ "$STATE/$ID.pi-ext.ts" "$STATE/$ID.grok-turnend-token" \ "$STATE/$ID.kimi-turnend-token" "$STATE/$ID.muse-session" \ diff --git a/bin/fm-test-run.sh b/bin/fm-test-run.sh index a4ef94500fe..c39d767bced 100755 --- a/bin/fm-test-run.sh +++ b/bin/fm-test-run.sh @@ -251,7 +251,8 @@ family_for_basename() { fm-secondmate-liveness.test.sh|fm-secondmate-reconcile.test.sh|\ fm-secondmate-safety.test.sh|fm-secondmate-sync.test.sh|\ fm-startup-memory-budget.test.sh|fm-stow-cascade.test.sh|\ - fm-send-secondmate-marker.test.sh|fm-shared-captain-inheritance.test.sh) + fm-send-secondmate-marker.test.sh|fm-shared-captain-inheritance.test.sh|\ + fm-parent-mirror.test.sh) printf '%s\n' secondmate ;; fm-backlog-atomicity.test.sh|\ @@ -575,6 +576,7 @@ tests/fm-pi-branch-extension.test.sh 22239 tests/fm-pi-branch-live-e2e.test.sh 56 tests/fm-pi-primary-live-e2e.test.sh 20 tests/fm-pi-watch-extension.test.sh 42970 +tests/fm-parent-mirror.test.sh 43000 tests/fm-pr-check-security.test.sh 160475 tests/fm-procevent-quota.test.sh 1949 tests/fm-procevent-when.test.sh 17392 @@ -1191,6 +1193,12 @@ families_for_changed_path() { bin/fm-stow-cascade.sh) printf '%s\n' secondmate ;; + bin/fm-parent-channel-lib.sh|bin/fm-parent-mirror*) + printf '%s\n' secondmate + printf '%s\n' watcher-wake-lock + printf '%s\n' pr-forge + printf '%s\n' pure-contract-unit + ;; bin/fm-session-start.sh|bin/fm-bootstrap.sh|bin/fm-fleet-sync.sh|\ bin/fm-sessionstart-nudge.sh|bin/fm-startup-network.sh|bin/fm-tangle*|bin/fm-update.sh|\ bin/fm-gate-refuse*|bin/fm-lock*) diff --git a/bin/fm-wake-lib.sh b/bin/fm-wake-lib.sh index a9ccddcb02b..b544900e3a6 100755 --- a/bin/fm-wake-lib.sh +++ b/bin/fm-wake-lib.sh @@ -536,19 +536,12 @@ _fm_recovery_marker_write_locked() { # already-announced generation announced so it cannot be re-presented until a # new down stretch mints a new generation. # docs/watcher-continuity.md owns the recovery contract and sequence-safety rationale. -_fm_recovery_marker_publish() { - local marker=$1 kind=${2:-downtime} lock saved_token generation='' status=pending - case "$kind" in handling|downtime) ;; *) return 1 ;; esac - lock="${marker}.lock" - fm_lock_acquire_wait "$lock" || return 1 +_fm_recovery_marker_publish_locked() { + local marker=$1 kind=${2:-downtime} saved_token generation='' status=pending if [ -d "$marker" ] && [ ! -L "$marker" ]; then - fm_lock_release "$lock" return 1 fi if [ "$kind" = downtime ]; then - # Read inline rather than in a command substitution: this runs inside the - # marker-lock critical section, so it must not add a subshell fork there. - # The token is restored because publishing owns no snapshot of its own. saved_token=$FM_RECOVERY_MARKER_TOKEN if fm_recovery_marker_read "$marker"; then case "$FM_RECOVERY_MARKER_TOKEN" in @@ -564,11 +557,27 @@ _fm_recovery_marker_publish() { fi FM_RECOVERY_MARKER_TOKEN=$saved_token fi - if ! _fm_recovery_marker_write_locked "$marker" "$kind" "$generation" "$status"; then - fm_lock_release "$lock" - return 1 - fi + _fm_recovery_marker_write_locked "$marker" "$kind" "$generation" "$status" +} + +_fm_recovery_marker_publish() { + local marker=$1 kind=${2:-downtime} lock rc=0 + case "$kind" in handling|downtime) ;; *) return 1 ;; esac + lock="${marker}.lock" + fm_lock_acquire_wait "$lock" || return 1 + _fm_recovery_marker_publish_locked "$marker" "$kind" || rc=$? fm_lock_release "$lock" + return "$rc" +} + +_fm_recovery_marker_publish_bounded() { + local marker=$1 kind=${2:-downtime} seconds=$3 lock rc=0 + case "$kind" in handling|downtime) ;; *) return 1 ;; esac + lock="${marker}.lock" + fm_lock_acquire_wait_bounded "$lock" "$seconds" || return $? + _fm_recovery_marker_publish_locked "$marker" "$kind" || rc=$? + fm_lock_release "$lock" + return "$rc" } _fm_recovery_marker_begin_handling() { @@ -1462,23 +1471,27 @@ fm_wake_clean_field() { LC_ALL=C tr '\t\r\n' ' ' } -fm_wake_append() { - local kind=$1 key=$2 payload=$3 clean_key clean_payload epoch seq seq_file status - local recovery_marker - case "$kind" in - signal|stale|check|heartbeat) ;; - *) printf 'fm_wake_append: invalid wake kind: %s\n' "$kind" >&2; return 2 ;; +_fm_wake_kind_valid() { + case "$1" in + signal|stale|check|heartbeat) return 0 ;; + *) return 1 ;; esac +} +_fm_wake_append_locked() { + local kind=$1 key=$2 payload=$3 recovery_seconds=${4:-} clean_key clean_payload epoch seq seq_file status + local recovery_marker clean_key=$(printf '%s' "$key" | fm_wake_clean_field) clean_payload=$(printf '%s' "$payload" | fm_wake_clean_field) epoch=$(date +%s) seq_file="$STATE/.wake-queue.seq" recovery_marker="$STATE/.watcher-down" status=0 - - fm_lock_acquire_wait "$FM_WAKE_QUEUE_LOCK" - _fm_recovery_marker_publish "$recovery_marker" downtime || status=$? + if [ -n "$recovery_seconds" ]; then + _fm_recovery_marker_publish_bounded "$recovery_marker" downtime "$recovery_seconds" || status=$? + else + _fm_recovery_marker_publish "$recovery_marker" downtime || status=$? + fi if [ "$status" -eq 0 ]; then seq=$(cat "$seq_file" 2>/dev/null || echo 0) case "$seq" in @@ -1490,6 +1503,43 @@ fm_wake_append() { if [ "$status" -eq 0 ]; then printf '%s\t%s\t%s\t%s\t%s\n' "$epoch" "$seq" "$kind" "$clean_key" "$clean_payload" >> "$FM_WAKE_QUEUE" || status=$? fi + return "$status" +} + +fm_wake_append() { + local kind=$1 status + _fm_wake_kind_valid "$kind" \ + || { printf 'fm_wake_append: invalid wake kind: %s\n' "$kind" >&2; return 2; } + fm_lock_acquire_wait "$FM_WAKE_QUEUE_LOCK" || return 1 + _fm_wake_append_locked "$kind" "$2" "$3" + status=$? + fm_lock_release "$FM_WAKE_QUEUE_LOCK" + return "$status" +} + +fm_wake_append_bounded() { + local kind=$1 seconds=$4 status + _fm_wake_kind_valid "$kind" \ + || { printf 'fm_wake_append_bounded: invalid wake kind: %s\n' "$kind" >&2; return 2; } + fm_lock_acquire_wait_bounded "$FM_WAKE_QUEUE_LOCK" "$seconds" || return $? + _fm_wake_append_locked "$kind" "$2" "$3" "$seconds" + status=$? + fm_lock_release "$FM_WAKE_QUEUE_LOCK" + return "$status" +} + +fm_wake_append_if_key_absent_bounded() { + local kind=$1 key=$2 seconds=$4 clean_key status + _fm_wake_kind_valid "$kind" \ + || { printf 'fm_wake_append_if_key_absent_bounded: invalid wake kind: %s\n' "$kind" >&2; return 2; } + clean_key=$(printf '%s' "$key" | fm_wake_clean_field) + fm_lock_acquire_wait_bounded "$FM_WAKE_QUEUE_LOCK" "$seconds" || return $? + if fm_wake_queued_keys_locked "$kind" | grep -Fx -- "$clean_key" >/dev/null 2>&1; then + fm_lock_release "$FM_WAKE_QUEUE_LOCK" + return 3 + fi + _fm_wake_append_locked "$kind" "$clean_key" "$3" "$seconds" + status=$? fm_lock_release "$FM_WAKE_QUEUE_LOCK" return "$status" } @@ -1501,14 +1551,25 @@ fm_wake_append() { # for it is queued and unacknowledged, and disappears only after post-handling # acknowledgement consumes it. fm_wake_queued_keys() { - local kind=$1 - case "$kind" in - signal|stale|check|heartbeat) ;; - *) printf 'fm_wake_queued_keys: invalid wake kind: %s\n' "$kind" >&2; return 2 ;; - esac - fm_lock_acquire_wait "$FM_WAKE_QUEUE_LOCK" + local kind=$1 status + _fm_wake_kind_valid "$kind" \ + || { printf 'fm_wake_queued_keys: invalid wake kind: %s\n' "$kind" >&2; return 2; } + fm_lock_acquire_wait "$FM_WAKE_QUEUE_LOCK" || return 1 fm_wake_queued_keys_locked "$kind" + status=$? fm_lock_release "$FM_WAKE_QUEUE_LOCK" + return "$status" +} + +fm_wake_queued_keys_bounded() { + local kind=$1 seconds=$2 status + _fm_wake_kind_valid "$kind" \ + || { printf 'fm_wake_queued_keys_bounded: invalid wake kind: %s\n' "$kind" >&2; return 2; } + fm_lock_acquire_wait_bounded "$FM_WAKE_QUEUE_LOCK" "$seconds" || return $? + fm_wake_queued_keys_locked "$kind" + status=$? + fm_lock_release "$FM_WAKE_QUEUE_LOCK" + return "$status" } fm_wake_queued_keys_locked() { diff --git a/bin/fm-watch.sh b/bin/fm-watch.sh index fd4f11a4b1a..98e507fc91a 100755 --- a/bin/fm-watch.sh +++ b/bin/fm-watch.sh @@ -1644,6 +1644,37 @@ while :; do touch "$STATE/.last-check" fi + # Secondmate parent mirror: deliver every child ledger event the parent is + # still owed (docs/secondmate-parent-channel.md). Silent in a main home and + # on a quiet poll; the library queues a delivery problem once as a durable + # wake and prints it only when newly queued, so a standing problem wakes + # firstmate once per episode rather than every poll. It runs after the slow + # checks so a merge poll's own loud failure is never masked by a mirror + # diagnostic in the same cycle, and before the signal scan so a chatty + # sibling cannot starve delivery. + if [ -e "$FM_HOME/.fm-secondmate-home" ] || [ -L "$FM_HOME/.fm-secondmate-home" ]; then + mirror_out= + mirror_err= + if [ -d "$STATE" ] && [ ! -L "$STATE" ]; then + mirror_err=$(umask 077; mktemp "$STATE/.parent-mirror.stderr.XXXXXX") || mirror_err= + fi + if [ -n "$mirror_err" ]; then + mirror_out=$(FM_HOME="$FM_HOME" FM_STATE_OVERRIDE="$STATE" \ + "$SCRIPT_DIR/fm-parent-mirror.sh" sweep 2>"$mirror_err") || true + if [ -s "$mirror_err" ]; then + triage_log "parent mirror: $(head -c 400 "$mirror_err" | tr '\n' ' ')" + fi + rm -f "$mirror_err" + else + mirror_out=$(FM_HOME="$FM_HOME" FM_STATE_OVERRIDE="$STATE" \ + "$SCRIPT_DIR/fm-parent-mirror.sh" sweep 2>/dev/null) || true + triage_log "parent mirror: temporary diagnostic capture unavailable" + fi + if [ -n "$mirror_out" ]; then + wake "check: parent-mirror" + fi + fi + # On the first changed signal, linger one grace period and re-scan before # classifying: a crewmate's final status write and the same turn's turn-end # hook land seconds apart, and reporting them as separate actionable wakes diff --git a/docs/architecture.md b/docs/architecture.md index 0e3e0042635..c177f537d17 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -50,6 +50,7 @@ No-change heartbeats are also benign. Separately from heartbeat backoff and wedge handling, the watcher poll runs `bin/fm-inactive-reconcile.sh` on its own bounded cadence, while locked session start sends the same bounded local scan through `bin/fm-startup-network.sh`'s deferred worker so current-state reads never block the digest. In each home the scan considers only that home's long-inactive direct ordinary crewmates, excludes captain-held work, and accepts only `done` or `failed` from `bin/fm-crew-state.sh`. A secondmate retains a durable receipt for its idempotent report through the established parent route, and main-home captain presentation retains a separate receipt; neither path performs a forge or PR check. +In a secondmate home that scan yields every child whose ledger already ends in a terminal verb to the parent-channel mirror, which delivers ledger evidence on every poll under the contract in [secondmate-parent-channel.md](secondmate-parent-channel.md). Absorbed wakes advance their suppression markers, log to `state/.watch-triage.log`, and keep the watcher blocking without a queue record or LLM turn. Each `fm-wake-drain.sh` presentation runs the same liveness guard as the supervision scripts, so a lapsed watcher chain surfaces even on a turn that only handles queued wakes. Routine watcher polling, supervision no-ops, elapsed waiting time, and absorbed benign wakes stay silent. @@ -250,6 +251,7 @@ The same project may appear in multiple secondmate homes when their scopes diffe Secondmates are idle by default: after startup recovery reconciles only work already in their own home, an empty queue waits silently for routed tasks, and they never self-initiate surveys or audits. When called with `FM_HOME=` or when `FM_HOME` is already set to the active firstmate home, metadata-routed `fm-send.sh` requests to a live `kind=secondmate` use the live-charter-compatible `from-firstmate` carrier owned by `bin/fm-operational-input.sh`, so the secondmate returns terse answers through status lines and detailed answers through docs plus status pointers instead of replying only in its own chat. The parent guards every reply-bearing marked request against a missing correlated report without reading the secondmate conversation; `bin/fm-pending-reply-lib.sh` owns the correlation, recovery, escalation, and retention contract, while `bin/fm-send.sh` owns the explicit fire-and-forget exception. +Every other parent-facing fact from inside a mate home - a child's PR-ready or terminal line, a scout report, a failed line, an unanswered decision or blocker, a task held for the captain, and its answer - reaches the parent through the ledger mirror and at-source publication owned by [secondmate-parent-channel.md](secondmate-parent-channel.md), so delivery never depends on the mate model appending a line. Explicit backend-target sends and direct human typing stay unmarked, so captain intervention in a secondmate pane remains conversational. After seeding a secondmate, `fm-backlog-handoff.sh` validates the fleet-specific handoff, atomically delegates already-judged in-scope queued item moves to `tasks-axi mv`, and then sends a marked routed-work wake through the receiver's recorded endpoint. A durable move with a missing, failed, or unresolved wake is reported as failure rather than success; rerunning the same handoff recovers known-undelivered wake intent without moving the item again, while an unresolved delivery is never blindly resent. diff --git a/docs/configuration.md b/docs/configuration.md index ea301bf6db0..d66386d4260 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -11,7 +11,7 @@ The shared orchestrator behavior lives in [`AGENTS.md`](../AGENTS.md) - edit it This section is the single owner of the top-level operational-home layout; producer script headers and their help own exact child-file fields and mutation contracts. The tracked code root contains the shared instruction, skill, documentation, workflow, and `bin/` surfaces, while each effective `FM_HOME` contains private operational directories. `data/` holds durable private fleet records such as the project and secondmate registries, captain preferences, optional shared captain preferences, learnings, backlog, briefs, scout reports, and explicitly installed content-addressed extension packages under `data/extensions/packages/`. -`state/` holds runtime records such as task metadata, append-only status events, endpoint signals, watcher and wake-queue coordination, inactive terminal-outcome receipts under `state/terminal-outcomes/`, enabled extension working namespaces under `state/extensions/`, away-mode state, generated Relay artifacts, parent-side remote ledger copies under `state/secondmate-summary-cache/`, one-shot Bearings reconcile requests under `state/reconcile-notify/`, private secondmate config-reread generations with their retry and quarantine state, per-task steering-inbox records under `state/.inbox/` (`bin/fm-task-inbox-lib.sh`), and parent-owned secondmate pending-reply records under `state/pending-replies/` (`bin/fm-pending-reply-lib.sh`). +`state/` holds runtime records such as task metadata, append-only status events, endpoint signals, watcher and wake-queue coordination, inactive terminal-outcome receipts under `state/terminal-outcomes/`, enabled extension working namespaces under `state/extensions/`, away-mode state, generated Relay artifacts, parent-side remote ledger copies under `state/secondmate-summary-cache/`, one-shot Bearings reconcile requests under `state/reconcile-notify/`, private secondmate config-reread generations with their retry and quarantine state, per-task steering-inbox records under `state/.inbox/` (`bin/fm-task-inbox-lib.sh`), parent-owned secondmate pending-reply records under `state/pending-replies/` (`bin/fm-pending-reply-lib.sh`), and a secondmate home's parent-channel mirror records under `state/parent-mirror/` (`bin/fm-parent-mirror-lib.sh`). `config/` holds local gitignored operating choices, including explicit extension bindings under `config/extensions.d/`, and `projects/` holds the local project clones that Firstmate reads but changes only through the narrow guarded and concrete captain-approved exceptions in `AGENTS.md`. Untracked files and directories whose names begin with `scratchpad` are also gitignored, so temporary scratch does not make porcelain-based secondmate sync guards treat a home as dirty. diff --git a/docs/documentation-audiences.json b/docs/documentation-audiences.json index 8bb68bd4ba2..250ecf995aa 100644 --- a/docs/documentation-audiences.json +++ b/docs/documentation-audiences.json @@ -360,6 +360,10 @@ "path": "docs/remote-secondmates.md", "audience": "operator-current" }, + { + "path": "docs/secondmate-parent-channel.md", + "audience": "maintainer-architecture" + }, { "path": "docs/scripts.md", "audience": "operator-current" diff --git a/docs/remote-secondmates.md b/docs/remote-secondmates.md index 5a36fef02a3..b26397536f7 100644 --- a/docs/remote-secondmates.md +++ b/docs/remote-secondmates.md @@ -185,6 +185,7 @@ An unreachable or unreadable remote read is unknown, not evidence that the endpo Marked requests keep the existing correlation contract. The remote charter appends replies to `state/parent-replies.status` in the remote home. +The mate's own machinery appends to that same log: the ledger mirror and the at-source publishers in [`secondmate-parent-channel.md`](secondmate-parent-channel.md) resolve the remote route to it, so a child outcome crosses this relay whether or not the mate model wrote a line. A process-event source performs a non-destructive, cursor-anchored delta read, fetches only referenced `data/*.md` documents through the confined reader, mirrors every content-bearing line at most once into the primary status channel, and does not carry blank separators. The channel carries the mate's status and decision model: an uncorrelated progress line and a newly raised `needs-decision` travel the same path as a correlated answer, and reach the parent's open-decision fold identically. Correlation is a per-line property that settles a pending request; it is never a gate on the stream, so no single line can stop or wedge the relay or hold the cursor back. diff --git a/docs/scripts.md b/docs/scripts.md index d23aafca7cc..423b53294d5 100644 --- a/docs/scripts.md +++ b/docs/scripts.md @@ -71,6 +71,9 @@ The shared no-mistakes gate refusal for fleet lifecycle entrypoints is summarize | `fm-task-inbox-lib.sh` | Single owner of durable steering-inbox records, acknowledgement, doorbells, and the delivery-attempt ladder | | `fm-pending-reply-lib.sh` | Parent-owned secondmate pending-reply expectations, recovery, and keyed escalation lifecycle | | `fm-secondmate-report.sh` | Optional helper to append a correlated parent status or document-pointer report | +| `fm-parent-channel-lib.sh` | Single owner of a secondmate home's parent channel: route resolution and idempotent append | +| `fm-parent-mirror-lib.sh` | Deterministic mirror of a secondmate's child ledgers onto its parent channel, with per-child cursors and thresholds | +| `fm-parent-mirror.sh` | Sweep child ledgers onto the parent channel on every watcher poll, or on demand for one child | | `fm-extension.mjs` | Bind, inspect, verify, and strictly invoke trusted external process-event adapter packages | | `fm-extension-launch-barrier.mjs` | Publish one exact static core-owned invocation group before package code runs | | `fm-extension.sh` | Expose extension binding commands through the tracked shell and remote-home command boundary | diff --git a/docs/secondmate-parent-channel.md b/docs/secondmate-parent-channel.md new file mode 100644 index 00000000000..4be7d761f3c --- /dev/null +++ b/docs/secondmate-parent-channel.md @@ -0,0 +1,125 @@ +# Secondmate parent channel + +This is the design note and the authoritative contract for how a captain-facing outcome inside a secondmate home reaches the parent firstmate, and through it the captain. +`bin/fm-parent-channel-lib.sh` owns where the channel lives and how a line is appended to it. +`bin/fm-parent-mirror.sh` owns the deterministic mirror of child ledgers onto that channel. +[`remote-secondmates.md`](remote-secondmates.md) owns the transport that carries the remote form of the channel back to the parent. + +## The channel + +A secondmate home has exactly one parent channel, named by the durable `.fm-secondmate-parent` binding written at seeding. +For a local route it is the parent home's `state/.status`. +For a remote route it is the mate home's own `state/parent-replies.status`, which the parent's remote reply adapter mirrors line for line into that same parent file. +The parent watcher classifies new lines there exactly as it classifies any crewmate's status stream, so a captain-relevant line appended to the channel becomes a parent wake, and a parent wake becomes a captain-facing message. +Nothing else reaches the parent: a secondmate's chat is read by nobody, and the parent never scrapes it. + +## The defect this design removes + +Four instances on 2026-09-02 across two mate homes showed the same failure. +The watcher delivered the parent's request within a minute every time, the mate did the work, and then the mate addressed "captain" in its own chat instead of appending to the channel. +The cause is structural, not a one-off lapse: `AGENTS.md` tells every firstmate to reach the captain and to address the captain in every response, while the charter's return-channel rule is a smaller, later instruction. +A PR-ready report was the observed symptom, but a finding, a decision, a blocker, and a failure all fail the same way, because every one of them depended on the mate model remembering to write one line. + +The design goal is therefore: the parent channel must not depend on the model remembering to write to it. + +## Candidate mechanisms + +Four mechanisms were evaluated together against coverage across all outcome kinds, harness independence, false-positive noise, and failure direction. + +### A. Turn-end text mirror + +At the mate's turn end, if the assistant turn contained captain-facing text and no parent append happened, mirror a bounded note of that text to the channel. + +- Coverage looks total but the predicate is empty: every firstmate turn contains captain-facing text by mandate, so the hook would fire on every idle turn, and deciding which captain-facing sentence is an outcome is itself a model-behavior judgement. +- Harness dependence is maximal: Claude exposes a transcript path in its Stop payload, Codex and Grok differ, Cursor's stop hook carries no message text and cannot block, Pi needs an extension reading session messages, OpenCode a plugin, and Kimi has no project hooks at all; every one is a harness-dependent check that needs live proof per harness per upgrade. +- Noise is unbounded, and a mirrored sentence that happens to contain a decision verb would open a decision in the parent's fold. +- Failure direction is wrong: an absent or broken hook drops outcomes silently, which is exactly the current failure. + +Rejected as a delivery mechanism. + +### B. Charter and persona carve-out + +Redefine "the captain" inside a secondmate home as the parent channel: no captain in chat, every captain sentence is an append, with a one-line carve-out at each `AGENTS.md` risk point. + +- Coverage is total in principle and zero in guarantee; it is the instruction that already failed four times. +- It is still necessary, because it is the only mechanism that can carry an outcome that exists nowhere but in the model's own reasoning, and because it fixes the persona confusion at its source. + +Adopted as the belt. + +### C. Deterministic ledger mirror + +On every supervision poll in a secondmate home, sweep each child's append-only status ledger and mirror every captain-relevant event that has no parent line yet onto the channel. + +- Coverage is every outcome that leaves durable evidence in the mate home: a child's terminal done or failed line, a PR ready line, a scout report, and a decision or blocker that stays open. +- Harness independence is complete: the sweep reads files and calls no harness, no forge, and no current-state reader. +- Noise is controlled by a per-child cursor and per-event receipts, so each event is mirrored once, and by an age threshold on decisions, blockers, and failures, so the mate keeps first responder authority and only an event it neither answered nor escalated is raised. +- Failure direction is at-least-once: a missed sweep is retried on the next poll, and a duplicate line is harmless while a missed one is not. +- It runs on the mate side, because only the mate's watcher can read the mate's state, and the existing parent binding and remote transport already carry the result; the parent cannot read a remote mate's ledger at all. + +Adopted as the guarantee. + +### D. At-source typed lines + +Make the scripts that record an outcome in a secondmate home publish the typed parent line themselves at record time. + +- Coverage is exactly the recorded outcomes: `bin/fm-pr-check.sh` for a registered PR, `bin/fm-captain-hold.sh` for a task held for the captain and for its answer, `bin/fm-teardown.sh` for a child leaving the home, and the merge outcome path that already existed. +- It is immediate and precise where the mirror is bounded by a poll, and it carries the richest context, such as the canonical PR URL and the hold reason. +- Its gap is every outcome that never passes through a script, which is precisely what C covers. + +Adopted, layered under C. + +## The adopted design + +The delivery rule has one sentence: the machinery reports facts, the mate reports judgement. + +1. `bin/fm-parent-channel-lib.sh` is the single owner of channel resolution and idempotent append; it serializes exact-line checks under a bounded destination-specific lock in the writing home's state directory, requires an existing destination to be a regular non-symlinked file, repairs an unterminated destination tail before deduplication or append, and is used by the merge outcome path, the inactive-outcome scan, the mirror, and every at-source publisher. +2. `bin/fm-parent-mirror.sh sweep` runs on every mate watcher poll and is a silent no-op in a main home. + For each direct ordinary child it validates `state/parent-mirror/` as a real directory, creates unpredictable private scratch files there, performs one recovery scan, then reads only whole bytes after the durable per-child cursor and folds them onto compact stored terminal and open-decision state, mirrors a terminal done line immediately, and mirrors a failed line or a still-open decision or blocker once it has stayed unanswered for `FM_PARENT_MIRROR_OPEN_SECS`. + A child status ledger is append-only by contract, and spawn, relaunch, and teardown never truncate or rewrite it in place. + The mirror detects a replaced ledger only when its file identity changes or its size shrinks below the durable cursor. + An in-place same-inode rewrite is explicitly outside the deterministic delivery guarantee. + The standing terminal is the last nonblank whole line with its absolute byte offset, so trailing blank lines cannot hide a failure. + A changed child spawn generation marks the prior generation's standing failure handled by relaunch, while a later failed line at a new ledger offset starts its own threshold clock. + A replaced or truncated ledger closes previously mirrored decisions as durable pending closures, clears terminal state from the invalidated identity, and starts fresh terminal clocks from the replacement ledger. + Retired and orphaned records retain the ledger, its presentation row, and the rendered PR, mode, yolo, and scout-report context while delivery is owed or an unterminated tail exists, and the successful orphan sweep retires all three only after every complete byte is delivered. + Repeated decision or blocker lines for an already-open key update its current verb and note without changing its original age, receipt, or opening line. + A mirrored open decision is closed on the channel with a keyed `resolved` line only when the child's own decision folds closed, so the parent's open-decision view tracks the mate's. + The mirrored line names the child, carries the child's own note, and adds the recorded PR URL, the scout report pointer, the delivery mode, and the merge posture when those are recorded, so the parent has what the captain needs without reading the mate home. +3. `bin/fm-pr-check.sh` sweeps the registered child after arming its merge poll, so a PR-ready registration reaches the parent at registration time with the canonical URL, and lock contention is reported as an actionable deferral. +4. `bin/fm-captain-hold.sh` publishes `needs-decision [key=captain-hold--]` when it holds a task for the captain in a secondmate home and the matching occurrence-keyed `resolved` line when the answer is recorded, including batch answers and idempotent retries. + Each hold writes a durable generation into a script-owned block anchored at the start of the task body, reuses that generation if the hold transition is interrupted before any matching resolution, and reads resolution presence, digest, mode, and generation through one parser that accepts a length-framed block or an exact legacy record anchored at the start of the body, so unrestricted captain prose cannot impersonate lifecycle state, a matching completed resolution advances the next hold, a re-held task opens and closes a distinct parent decision even when the captain repeats the same words, and an exact same-generation retry remains one delivery. + Before advancing a resolved task to another hold occurrence, it retries the durable recorded resolution and refuses the new hold while that old parent close remains undeliverable, so a later opening can never orphan the earlier parent decision. +5. `bin/fm-teardown.sh` performs the child's final sweep before it removes the child's record, then retires the child's mirror state; an undelivered final sweep must leave a verified durable orphan record and preserve its ledger and presentation row before the discoverable child record is removed, otherwise teardown refuses so a later poll or retry can still find the child. +6. `bin/fm-inactive-reconcile.sh` keeps its current-state role and yields to the mirror: in a secondmate home it no longer reports a child whose ledger already ends in a terminal verb, because that evidence is the mirror's, and it still reports a child whose ledger is silent while `bin/fm-crew-state.sh` says done or failed. +7. The charter scaffold in `bin/fm-brief.sh` opens with the channel rule, states that the mirror carries child facts, and confines the mate's own appends to judgement, marked-request answers, and its own blockers; `AGENTS.md` carries a one-line carve-out at the persona address rule and at the escalation list. + +## Outcome coverage + +| Outcome | Durable evidence in the mate home | Delivered by | +|---|---|---| +| Ship child PR ready | child `done: PR ...` line, then `pr=` in the child's record | D at `fm-pr-check`, C on the next poll | +| PR merged | merge poll or self merge | the existing merge outcome path | +| Scout child findings | child `done:` line plus `data//report.md` | C on the next poll, D at teardown | +| Child decision escalated | task held for the captain in the mate backlog | D at `fm-captain-hold hold` | +| Child decision or blocker left open | open keyed line in the child ledger | C after the open threshold | +| Child failed | child `failed:` line | C after the open threshold | +| Child ended silently | terminal current state with a silent ledger | the inactive-outcome scan | +| Answer to a marked request | correlated line, guarded by the pending-reply record | the existing pending-reply recovery and escalation | +| An outcome that exists only in the mate's reasoning | none | B only | + +## Noise and failure direction + +A mate that also appends its own line about a mirrored child produces a second line, never a missed one; the parent reads the mirror line as the fact and the mate's line as commentary. +Offset-addressed events use `mirror---l`, while decisions use `mirror---` and carry their opening line in the rendered event text. +The child-length prefix prevents ambiguous child and suffix boundaries, exact-line append deduplicates a retry of one rendered event, and a decision explicitly closed and later reopened at a new line remains a distinct opening. +Standing failures retain their ledger offset as event identity, so an identical failure recurring after recovery is delivered as a new event. +An unreadable parent binding is reported once through an atomic bounded append-if-key-absent wake operation in the mate home, so concurrent diagnostics cannot duplicate the episode, an already-queued episode stays quiet, and contention on either the queue or recovery-marker lock degrades to stderr rather than wedging the watcher. +The mirror needs a live mate watcher, which is already required whenever the mate has work in flight; a mate with no work has no ledger to mirror. + +## Regression coverage + +`tests/fm-parent-mirror.test.sh` covers the sweep against real ledgers with no harness: immediate done delivery with recorded context and the scout report pointer, whole-line and incremental-span delivery, thresholded decision and failure delivery, decision updates and keyed closes and reopens, ledger replacement recovery, trailing blank lines, silence for anything handled inside the threshold, untrackable decision lines, the remote route, the PR registration hook, the retire path with orphan retry, bounded lock waits, main-home inertness, the once-per-episode unreadable-binding diagnostic, and the real watcher poll driving the sweep. +`tests/fm-captain-hold-lifecycle.test.sh` covers a mate home publishing a captain hold and its answer, refusing to advance a re-hold past an undelivered close, and the real teardown delivering a scout's final line before retiring its record. +`tests/fm-inactive-reconcile.test.sh` covers the inactive scan yielding terminal-verb ledgers to the mirror while keeping the silent-ledger cases that remain its own. +`tests/fm-pr-merge.test.sh` keeps the merge outcome path's upward reporting and its loud refusal without a binding. +`tests/fm-brief.test.sh` pins the charter's channel rule. diff --git a/tests/fm-backend-herdr-presentation-e2e.test.sh b/tests/fm-backend-herdr-presentation-e2e.test.sh index ac62abb4275..bb6f12668e7 100755 --- a/tests/fm-backend-herdr-presentation-e2e.test.sh +++ b/tests/fm-backend-herdr-presentation-e2e.test.sh @@ -960,6 +960,8 @@ mkdir -p "$SECOND_HOME_A/state" "$SECOND_HOME_A/config" "$SECOND_HOME_A/data" \ "$SECOND_HOME_B/state" "$SECOND_HOME_B/config" "$SECOND_HOME_B/data" printf 'alpha\n' > "$SECOND_HOME_A/.fm-secondmate-home" printf 'bravo\n' > "$SECOND_HOME_B/.fm-secondmate-home" +printf 'schema=fm-secondmate-parent.v1\nroute=remote\n' > "$SECOND_HOME_A/.fm-secondmate-parent" +printf 'schema=fm-secondmate-parent.v1\nroute=remote\n' > "$SECOND_HOME_B/.fm-secondmate-parent" touch "$SECOND_HOME_A/state/.last-watcher-beat" "$SECOND_HOME_B/state/.last-watcher-beat" # Ensure the secondmate homes look like gitignored firstmate homes so inheritance # may write config/herdr-presentation-spaces. diff --git a/tests/fm-backlog-atomicity.test.sh b/tests/fm-backlog-atomicity.test.sh index 3097413fcba..68697a589d3 100755 --- a/tests/fm-backlog-atomicity.test.sh +++ b/tests/fm-backlog-atomicity.test.sh @@ -2189,6 +2189,8 @@ test_a_secondmate_home_keeps_its_own_books() { # single-host, so its own dispatch and completion keep its own two records # paired with no parent involved. printf '%s\n' mate-h1 > "$(home_of "$case_dir")/.fm-secondmate-home" + printf 'schema=fm-secondmate-parent.v1\nroute=remote\n' \ + > "$(home_of "$case_dir")/.fm-secondmate-parent" add_item "$case_dir" "$id" out=$(run_ship_spawn "$case_dir" "$id") || fail "mate-home spawn failed: $out" diff --git a/tests/fm-brief.test.sh b/tests/fm-brief.test.sh index 3e5d3064fd5..007c1e2159b 100755 --- a/tests/fm-brief.test.sh +++ b/tests/fm-brief.test.sh @@ -511,6 +511,23 @@ test_secondmate_no_projects_charter() { "secondmate charter did not close a quietly ended routed-work phase" assert_grep 'use the same key on its later' "$brief" \ "secondmate charter did not supersede working phases with later states" + # The parent-channel rule is the belt under the deterministic mirror + # (docs/secondmate-parent-channel.md): the charter must open with it, name + # the machinery that delivers child facts, and leave only judgement to the + # mate's own appends. + assert_grep '# The captain and the parent channel' "$brief" \ + "secondmate charter lost its parent-channel section" + assert_grep 'a captain-facing sentence that is not appended there has not been sent' "$brief" \ + "secondmate charter does not state that chat never reaches the captain" + assert_grep 'in this home it IS the captain' "$brief" \ + "secondmate charter does not bind the captain to the parent channel" + assert_grep 'bin/fm-parent-mirror.sh' "$brief" \ + "secondmate charter does not name the mirror that delivers child facts" + assert_grep 'What only you can append is judgement' "$brief" \ + "secondmate charter does not confine the mate's own appends to judgement" + if ! awk '/^# The captain and the parent channel/{seen=1} /^# Requests from the main firstmate/{exit !seen}' "$brief"; then + fail "secondmate charter states the parent-channel rule after the marker rules instead of before" + fi if grep -nE '^-[[:space:]]*$' "$brief" >/dev/null; then fail "project-less charter left a stray empty project bullet" fi diff --git a/tests/fm-captain-hold-lifecycle.test.sh b/tests/fm-captain-hold-lifecycle.test.sh index 5136d2f9b3e..fcbf2e1903e 100755 --- a/tests/fm-captain-hold-lifecycle.test.sh +++ b/tests/fm-captain-hold-lifecycle.test.sh @@ -226,6 +226,264 @@ EOF pass "the completion gate attests captain-held inventory and transfers open status decisions" } +# In a secondmate home a captain hold and its answer are parent-facing facts +# published by the record itself on the parent channel, and a scout's final +# done line reaches the parent inside teardown before the child's record goes +# (docs/secondmate-parent-channel.md). The parent here is a registry-valid +# home so teardown's own parent resolution is the real one. +test_secondmate_home_publishes_holds_and_final_outcomes() { + local home parent channel id open legacy_text legacy_digest prose_body + home=$(make_home mate-home) + parent="$TMP_ROOT/mate-parent" + mkdir -p "$parent/state" "$parent/data" + fm_write_secondmate_meta "$parent/state/mate.meta" "$home" + printf -- '- mate - Sample domain mate (home: %s; scope: sample work; projects: sample; added 2026-07-14)\n' "$home" \ + > "$parent/data/secondmates.md" + printf 'mate\n' > "$home/.fm-secondmate-home" + printf 'schema=fm-secondmate-parent.v1\nroute=local\nparent_home=%s\n' "$parent" > "$home/.fm-secondmate-parent" + channel="$parent/state/mate.status" + + run_captain "$home" hold mate-route-call \ + --title "Choose route: north, south" --reason "captain route choice pending" --repo sample >/dev/null \ + || fail "could not hold a task for the captain in the mate home" + grep -Fx 'needs-decision [key=captain-hold-mate-route-call-1]: captain hold mate-route-call occurrence 1: captain route choice pending' "$channel" >/dev/null \ + || fail "the hold did not reach the parent channel: $(cat "$channel" 2>/dev/null)" + open=$(bash -c '. "$1"; status_open_decisions "$2"' _ "$ROOT/bin/fm-classify-lib.sh" "$channel") + printf '%s\n' "$open" | grep -q "^captain-hold-mate-route-call-1 needs-decision " \ + || fail "the parent's fold did not open the published hold: $open" + run_captain "$home" hold mate-route-call \ + --title "Choose route: north, south" --reason "captain route choice pending" --repo sample >/dev/null \ + || fail "idempotent hold retry failed in the mate home" + [ "$(grep -c 'captain-hold-mate-route-call' "$channel")" = 1 ] \ + || fail "an idempotent hold retry duplicated the parent line" + + printf 'north\n' > "$home/answer.txt" + mv "$home/.fm-secondmate-parent" "$home/.fm-secondmate-parent.saved" + run_captain "$home" answer mate-route-call --decision-file "$home/answer.txt" \ + > "$home/answer.out" 2> "$home/answer.err" \ + || fail "could not durably record the captain's answer while parent delivery was unavailable" + grep -F 'actionable:' "$home/answer.err" >/dev/null \ + || fail "the unavailable parent close was not reported as actionable" + [ "$(grep -c 'resolved \[key=captain-hold-mate-route-call-1\]' "$channel" || true)" = 0 ] \ + || fail "the unavailable answer unexpectedly reached the parent" + if run_captain "$home" answer mate-route-call --decision-file "$home/answer.txt" \ + > "$home/answer-retry.out" 2> "$home/answer-retry.err"; then + fail "an answer retry reported success while its durable parent close was still unavailable" + fi + [ ! -s "$home/answer-retry.out" ] \ + || fail "the failed answer retry reported the task answered: $(cat "$home/answer-retry.out")" + grep -F 'actionable:' "$home/answer-retry.err" >/dev/null \ + || fail "the failed answer retry did not preserve its actionable parent-channel diagnostic" + mv "$home/.fm-secondmate-parent.saved" "$home/.fm-secondmate-parent" + run_captain "$home" answer mate-route-call --decision-file "$home/answer.txt" >/dev/null \ + || fail "idempotent answer retry failed in the mate home" + grep -Fx 'resolved [key=captain-hold-mate-route-call-1]: captain hold mate-route-call occurrence 1: answered' "$channel" >/dev/null \ + || fail "the answer retry did not close the hold on the parent channel: $(cat "$channel")" + open=$(bash -c '. "$1"; status_open_decisions "$2"' _ "$ROOT/bin/fm-classify-lib.sh" "$channel") + [ -z "$open" ] || fail "the parent's fold still holds the answered hold: $open" + run_captain "$home" answer mate-route-call --decision-file "$home/answer.txt" >/dev/null \ + || fail "second idempotent answer retry failed in the mate home" + [ "$(grep -c 'resolved \[key=captain-hold-mate-route-call-1\]' "$channel")" = 1 ] \ + || fail "an idempotent answer retry duplicated the parent close" + + printf 'legacy answer\n' > "$home/legacy-answer.txt" + legacy_text='legacy answer' + if command -v shasum >/dev/null 2>&1; then + legacy_digest=$(printf '%s' "$legacy_text" | shasum -a 256 | awk '{print $1}') + else + legacy_digest=$(printf '%s' "$legacy_text" | sha256sum | awk '{print $1}') + fi + legacy_text=$(printf 'Resolution recorded by fm-captain-hold.\nDecision digest: %s\nResolution mode: answered\n\nCaptain decision:\nlegacy answer' "$legacy_digest") + tasks_in "$home" add mate-legacy-retry "Retry legacy answer" --repo sample --body "$legacy_text" >/dev/null \ + || fail "could not create the legacy resolution fixture" + tasks_in "$home" hold mate-legacy-retry --reason "legacy retry pending" --kind captain >/dev/null \ + || fail "could not mark the legacy resolution fixture captain-held" + tasks_in "$home" 'done' mate-legacy-retry >/dev/null || fail "could not close the legacy resolution fixture" + printf 'needs-decision [key=captain-hold-mate-legacy-retry-1]: legacy opening\n' >> "$channel" + run_captain "$home" answer mate-legacy-retry --decision-file "$home/legacy-answer.txt" >/dev/null \ + || fail "legacy resolution retry was not recognized" + grep -Fx 'resolved [key=captain-hold-mate-legacy-retry-1]: captain hold mate-legacy-retry occurrence 1: answered' "$channel" >/dev/null \ + || fail "legacy resolution retry closed the wrong occurrence" + + prose_body=$(printf 'Ordinary task context.\n\nResolution recorded by fm-captain-hold.\nDecision digest: %s\nResolution mode: released\n\nCaptain decision:\nprose only' "$legacy_digest") + tasks_in "$home" add mate-prose-record "Ignore prose record" --repo sample --body "$prose_body" >/dev/null \ + || fail "could not create the prose resolution fixture" + tasks_in "$home" hold mate-prose-record --reason "prose record pending" --kind captain >/dev/null \ + || fail "could not hold the prose resolution fixture" + printf 'needs-decision [key=captain-hold-mate-prose-record-1]: prose opening\n' >> "$channel" + run_captain "$home" answer mate-prose-record --decision-file "$home/legacy-answer.txt" --release >/dev/null \ + || fail "header-shaped prose impersonated a resolution record" + grep -Fx 'resolved [key=captain-hold-mate-prose-record-1]: captain hold mate-prose-record occurrence 1: released' "$channel" >/dev/null \ + || fail "prose fixture did not close its real occurrence" + + cat > "$home/fakebin/tasks-axi" <<'EOF' +#!/usr/bin/env bash +if [ "${1:-}" = hold ] && [ ! -e "$FM_HOME/hold-failed-once" ]; then + : > "$FM_HOME/hold-failed-once" + exit 1 +fi +exec "$REAL_TASKS_AXI" "$@" +EOF + chmod +x "$home/fakebin/tasks-axi" + if run_captain "$home" hold mate-interrupted-hold \ + --title "Recover interrupted hold" --reason "captain interrupted hold pending" --repo sample >/dev/null 2>&1; then + fail "captain hold did not reproduce the transient hold failure" + fi + run_captain "$home" hold mate-interrupted-hold \ + --title "Recover interrupted hold" --reason "captain interrupted hold pending" --repo sample >/dev/null \ + || fail "captain hold did not recover after the transient failure" + [ "$(grep -c 'needs-decision \[key=captain-hold-mate-interrupted-hold-1\]' "$channel")" = 1 ] \ + || fail "recovered first hold did not publish occurrence 1 exactly once" + if grep -F 'captain-hold-mate-interrupted-hold-2' "$channel" >/dev/null; then + fail "failed first hold consumed an occurrence" + fi + printf 'release interrupted hold\n' > "$home/interrupted-hold.txt" + run_captain "$home" answer mate-interrupted-hold --decision-file "$home/interrupted-hold.txt" --release >/dev/null \ + || fail "could not release the recovered first hold" + run_captain "$home" hold mate-interrupted-hold \ + --title "Recover interrupted hold" --reason "captain interrupted hold pending" --repo sample >/dev/null \ + || fail "could not re-hold after the recovered occurrence" + grep -Fx 'needs-decision [key=captain-hold-mate-interrupted-hold-2]: captain hold mate-interrupted-hold occurrence 2: captain interrupted hold pending' "$channel" >/dev/null \ + || fail "completed recovered hold did not advance to occurrence 2" + run_captain "$home" answer mate-interrupted-hold --decision-file "$home/interrupted-hold.txt" --release >/dev/null \ + || fail "could not release the second interrupted-hold occurrence" + rm -f "$home/fakebin/tasks-axi" + + run_captain "$home" hold mate-release-call \ + --title "Release held work" --reason "captain release pending" --repo sample >/dev/null \ + || fail "could not hold releasable work in the mate home" + printf 'release\n' > "$home/release.txt" + mv "$home/.fm-secondmate-parent" "$home/.fm-secondmate-parent.saved" + run_captain "$home" answer mate-release-call --decision-file "$home/release.txt" --release \ + > "$home/release.out" 2> "$home/release.err" \ + || fail "could not durably release work while parent delivery was unavailable" + mv "$home/.fm-secondmate-parent.saved" "$home/.fm-secondmate-parent" + run_captain "$home" answer mate-release-call --decision-file "$home/release.txt" --release >/dev/null \ + || fail "idempotent release retry failed in the mate home" + [ "$(grep -c 'resolved \[key=captain-hold-mate-release-call-1\].*: released' "$channel")" = 1 ] \ + || fail "the release retry did not close its parent hold exactly once" + run_captain "$home" hold mate-release-call \ + --title "Release held work" --reason "captain release pending" --repo sample >/dev/null \ + || fail "could not re-hold released work in the mate home" + grep -Fx 'needs-decision [key=captain-hold-mate-release-call-2]: captain hold mate-release-call occurrence 2: captain release pending' "$channel" >/dev/null \ + || fail "re-held work did not publish a distinct parent opening" + run_captain "$home" answer mate-release-call --decision-file "$home/release.txt" --release >/dev/null \ + || fail "could not answer the re-held occurrence with the same words" + run_captain "$home" answer mate-release-call --decision-file "$home/release.txt" --release >/dev/null \ + || fail "re-held same-generation retry failed" + [ "$(grep -c 'resolved \[key=captain-hold-mate-release-call-2\].*: released' "$channel")" = 1 ] \ + || fail "re-held occurrence did not publish exactly one matching close" + open=$(bash -c '. "$1"; status_open_decisions "$2"' _ "$ROOT/bin/fm-classify-lib.sh" "$channel") + [ -z "$open" ] || fail "same-answer re-hold left a parent occurrence open: $open" + + run_captain "$home" hold mate-undelivered-close \ + --title "Retry an undelivered close" --reason "captain close retry pending" --repo sample >/dev/null \ + || fail "could not create the undelivered-close fixture" + printf 'release before re-hold\n' > "$home/undelivered-close.txt" + mv "$home/.fm-secondmate-parent" "$home/.fm-secondmate-parent.saved" + run_captain "$home" answer mate-undelivered-close \ + --decision-file "$home/undelivered-close.txt" --release >/dev/null 2> "$home/undelivered-close.err" \ + || fail "could not record the answer while its parent close was unavailable" + if run_captain "$home" hold mate-undelivered-close \ + --title "Retry an undelivered close" --reason "captain second occurrence pending" --repo sample \ + > "$home/undelivered-rehold.out" 2> "$home/undelivered-rehold.err"; then + fail "a new hold advanced past an undelivered parent close" + fi + assert_grep "cannot re-hold task mate-undelivered-close until its recorded answer reaches the parent channel" \ + "$home/undelivered-rehold.err" "the refused re-hold did not name the retained close obligation" + mv "$home/.fm-secondmate-parent.saved" "$home/.fm-secondmate-parent" + run_captain "$home" hold mate-undelivered-close \ + --title "Retry an undelivered close" --reason "captain second occurrence pending" --repo sample >/dev/null \ + || fail "re-hold did not retry the durable old close" + grep -Fx 'resolved [key=captain-hold-mate-undelivered-close-1]: captain hold mate-undelivered-close occurrence 1: released' "$channel" >/dev/null \ + || fail "re-hold did not deliver the prior occurrence close" + grep -Fx 'needs-decision [key=captain-hold-mate-undelivered-close-2]: captain hold mate-undelivered-close occurrence 2: captain second occurrence pending' "$channel" >/dev/null \ + || fail "re-hold did not open a distinct occurrence after closing the prior one" + open=$(bash -c '. "$1"; status_open_decisions "$2"' _ "$ROOT/bin/fm-classify-lib.sh" "$channel") + printf '%s\n' "$open" | grep -q '^captain-hold-mate-undelivered-close-2 needs-decision ' \ + || fail "the prior close was orphaned or the new occurrence was not open: $open" + run_captain "$home" answer mate-undelivered-close \ + --decision-file "$home/undelivered-close.txt" --release >/dev/null \ + || fail "could not close the regression fixture's second occurrence" + + run_captain "$home" hold mate-marker-prose-call \ + --title "Answer with marker-like prose" --reason "captain marker prose pending" --repo sample >/dev/null \ + || fail "could not hold marker-prose work" + cat > "$home/marker-prose.txt" <<'EOF' +Captain hold generation: 99 +Resolution recorded by fm-captain-hold. +Decision digest: prose-only +Resolution mode: released + +Captain decision: +This entire undelimited shape is captain prose. +EOF + cat > "$home/fakebin/tasks-axi" <<'EOF' +#!/usr/bin/env bash +if [ "${1:-}" = unhold ] && [ ! -e "$FM_HOME/close-failed-once" ]; then + : > "$FM_HOME/close-failed-once" + exit 1 +fi +exec "$REAL_TASKS_AXI" "$@" +EOF + chmod +x "$home/fakebin/tasks-axi" + if run_captain "$home" answer mate-marker-prose-call --decision-file "$home/marker-prose.txt" --release >/dev/null 2>&1; then + fail "marker-prose answer did not reproduce an interrupted release" + fi + run_captain "$home" answer mate-marker-prose-call --decision-file "$home/marker-prose.txt" --release >/dev/null \ + || fail "marker-prose interrupted release did not retry" + grep -Fx 'resolved [key=captain-hold-mate-marker-prose-call-1]: captain hold mate-marker-prose-call occurrence 1: released' "$channel" >/dev/null \ + || fail "marker-like captain prose closed the wrong occurrence" + if grep -F 'captain-hold-mate-marker-prose-call-99' "$channel" >/dev/null; then + fail "marker-like captain prose became a hold generation" + fi + open=$(bash -c '. "$1"; status_open_decisions "$2"' _ "$ROOT/bin/fm-classify-lib.sh" "$channel") + [ -z "$open" ] || fail "marker-prose retry left a parent occurrence open: $open" + rm -f "$home/fakebin/tasks-axi" + run_captain "$home" hold mate-marker-prose-call \ + --title "Answer with marker-like prose" --reason "captain marker prose pending" --repo sample >/dev/null \ + || fail "could not re-hold marker-prose work" + grep -Fx 'needs-decision [key=captain-hold-mate-marker-prose-call-2]: captain hold mate-marker-prose-call occurrence 2: captain marker prose pending' "$channel" >/dev/null \ + || fail "free prose changed the next hold generation" + run_captain "$home" answer mate-marker-prose-call --decision-file "$home/marker-prose.txt" --release >/dev/null \ + || fail "could not release the second marker-prose occurrence" + [ "$(grep -c 'captain-hold-mate-marker-prose-call-2' "$channel")" = 2 ] \ + || fail "the second marker-prose occurrence did not open and close exactly once" + + run_captain "$home" hold mate-batch-call \ + --title "Choose batch route" --reason "captain batch choice pending" --repo sample >/dev/null \ + || fail "could not hold the batch-answer task" + mv "$home/.fm-secondmate-parent" "$home/.fm-secondmate-parent.saved" + printf 'mate-batch-call\tnorth\n' | run_captain "$home" answers --source "batch fixture" \ + > "$home/batch.out" 2> "$home/batch.err" \ + || fail "batch answer did not durably close while parent delivery was unavailable" + grep -F 'actionable:' "$home/batch.err" >/dev/null \ + || fail "batch answer discarded its parent delivery warning" + mv "$home/.fm-secondmate-parent.saved" "$home/.fm-secondmate-parent" + printf 'mate-batch-call\tnorth\n' | run_captain "$home" answers --source "batch fixture" \ + > "$home/batch-retry.out" 2> "$home/batch-retry.err" \ + || fail "idempotent batch answer retry failed" + grep -Fx 'resolved [key=captain-hold-mate-batch-call-1]: captain hold mate-batch-call occurrence 1: answered' "$channel" >/dev/null \ + || fail "idempotent batch retry did not publish the parent close" + + id='mate-scout' + tasks_in "$home" add "$id" "Investigate mate systems" --kind scout --repo sample --start >/dev/null \ + || fail "could not create the mate scout fixture" + write_origin_meta "$home" "$id" + mkdir -p "$home/data/$id" + printf '# findings\n' > "$home/data/$id/report.md" + printf 'working: drafting\ndone: report complete\n' > "$home/state/$id.status" + run_captain "$home" complete "$id" --none >/dev/null \ + || fail "mate scout completion gate failed" + run_teardown "$home" "$id" >/dev/null 2> "$home/teardown.err" \ + || fail "mate scout teardown failed: $(cat "$home/teardown.err")" + grep -F "done [key=mirror-10-$id-l" "$channel" | grep -F "mirror: child=$id report complete report=data/$id/report.md" >/dev/null \ + || fail "teardown did not deliver the scout's final line to the parent: $(cat "$channel")" + [ ! -e "$home/state/parent-mirror/$id.record" ] || fail "teardown did not retire the scout's mirror record" + [ ! -e "$home/state/$id.meta" ] || fail "teardown left the scout's record behind" + pass "a mate home publishes captain holds, their answers, and a scout's final line on the parent channel" +} + # The recorded-answer rule: answering closes with the captain's exact words, an # exact retry is idempotent, a drifted retry is rejected, dependent work routed # behind the answered task is released by the close, and the completion gate is @@ -1169,6 +1427,7 @@ EOF } test_uninventoried_report_decision_refuses_completion +test_secondmate_home_publishes_holds_and_final_outcomes test_completion_gate_attests_and_transfers test_answer_records_and_closes test_release_frees_held_work diff --git a/tests/fm-gotmp.test.sh b/tests/fm-gotmp.test.sh index 3b17c593c23..9ed7943654e 100755 --- a/tests/fm-gotmp.test.sh +++ b/tests/fm-gotmp.test.sh @@ -84,6 +84,8 @@ make_fake_root() { ln -s "$ROOT/bin/fm-x-lib.sh" "$fake/bin/fm-x-lib.sh" ln -s "$ROOT/bin/fm-secondmate-registry-lib.sh" "$fake/bin/fm-secondmate-registry-lib.sh" ln -s "$ROOT/bin/fm-secondmate-parent-lib.sh" "$fake/bin/fm-secondmate-parent-lib.sh" + ln -s "$ROOT/bin/fm-parent-channel-lib.sh" "$fake/bin/fm-parent-channel-lib.sh" + ln -s "$ROOT/bin/fm-parent-mirror-lib.sh" "$fake/bin/fm-parent-mirror-lib.sh" # Receiver-wake retirement sources the pending-reply library, which in turn # requires the marker helper even for this ordinary-task teardown fixture. ln -s "$ROOT/bin/fm-pending-reply-lib.sh" "$fake/bin/fm-pending-reply-lib.sh" @@ -177,6 +179,8 @@ test_teardown_skips_gracefully_without_tasktmp() { ln -s "$ROOT/bin/fm-x-lib.sh" "$fake/bin/fm-x-lib.sh" ln -s "$ROOT/bin/fm-secondmate-registry-lib.sh" "$fake/bin/fm-secondmate-registry-lib.sh" ln -s "$ROOT/bin/fm-secondmate-parent-lib.sh" "$fake/bin/fm-secondmate-parent-lib.sh" + ln -s "$ROOT/bin/fm-parent-channel-lib.sh" "$fake/bin/fm-parent-channel-lib.sh" + ln -s "$ROOT/bin/fm-parent-mirror-lib.sh" "$fake/bin/fm-parent-mirror-lib.sh" ln -s "$ROOT/bin/fm-pending-reply-lib.sh" "$fake/bin/fm-pending-reply-lib.sh" ln -s "$ROOT/bin/fm-marker-lib.sh" "$fake/bin/fm-marker-lib.sh" ln -s "$ROOT/bin/fm-operational-input.sh" "$fake/bin/fm-operational-input.sh" diff --git a/tests/fm-home-summary-refresh.test.sh b/tests/fm-home-summary-refresh.test.sh index 9f06f44672e..ec78f62e6fc 100755 --- a/tests/fm-home-summary-refresh.test.sh +++ b/tests/fm-home-summary-refresh.test.sh @@ -62,6 +62,7 @@ mkdir -p "$HOME_DIR/state" "$HOME_DIR/data" "$HOME_DIR/config" \ HOME_DIR=$(cd "$HOME_DIR" && pwd -P) printf '# Seeded Firstmate home\n' > "$HOME_DIR/AGENTS.md" printf 'mate\n' > "$HOME_DIR/.fm-secondmate-home" +printf 'schema=fm-secondmate-parent.v1\nroute=remote\n' > "$HOME_DIR/.fm-secondmate-parent" fm_git_init_commit "$HOME_DIR/projects/task" git -C "$HOME_DIR/projects/task" checkout -q -b fm/ledger-task cat > "$HOME_DIR/data/backlog.md" <<'EOF' @@ -162,6 +163,7 @@ mkdir -p "$CADENCE_HOME/state" "$CADENCE_HOME/data" "$CADENCE_HOME/config" \ "$CADENCE_HOME/projects" printf '# Seeded Firstmate home\n' > "$CADENCE_HOME/AGENTS.md" printf 'cadence\n' > "$CADENCE_HOME/.fm-secondmate-home" +printf 'schema=fm-secondmate-parent.v1\nroute=remote\n' > "$CADENCE_HOME/.fm-secondmate-parent" cat > "$CADENCE_HOME/data/backlog.md" <<'EOF' ## In flight @@ -524,6 +526,7 @@ mkdir -p "$COST_HOME/state" "$COST_HOME/data" "$COST_HOME/config" \ "$COST_HOME/projects/task" printf '# Seeded Firstmate home\n' > "$COST_HOME/AGENTS.md" printf 'cost\n' > "$COST_HOME/.fm-secondmate-home" +printf 'schema=fm-secondmate-parent.v1\nroute=remote\n' > "$COST_HOME/.fm-secondmate-parent" fm_git_init_commit "$COST_HOME/projects/task" cat > "$COST_HOME/data/backlog.md" <<'EOF' ## In flight @@ -578,6 +581,7 @@ mkdir -p "$REMOTE_HOME/state" "$REMOTE_HOME/data" "$REMOTE_HOME/config" \ "$REMOTE_HOME/projects" "$TMP_ROOT/sshbin" printf '# Seeded Firstmate home\n' > "$REMOTE_HOME/AGENTS.md" printf 'remote\n' > "$REMOTE_HOME/.fm-secondmate-home" +printf 'schema=fm-secondmate-parent.v1\nroute=remote\n' > "$REMOTE_HOME/.fm-secondmate-parent" cat > "$REMOTE_HOME/data/backlog.md" <<'EOF' ## In flight - [ ] rsm - Read remote current state (repo: firstmate) (kind: ship) (since 2026-08-28) @@ -641,6 +645,7 @@ mkdir -p "$BEAT_HOME/state" "$BEAT_HOME/data" "$BEAT_HOME/config" \ "$BEAT_HOME/projects" printf '# Seeded Firstmate home\n' > "$BEAT_HOME/AGENTS.md" printf 'beat\n' > "$BEAT_HOME/.fm-secondmate-home" +printf 'schema=fm-secondmate-parent.v1\nroute=remote\n' > "$BEAT_HOME/.fm-secondmate-parent" cat > "$BEAT_HOME/data/backlog.md" <<'EOF' ## In flight @@ -708,6 +713,7 @@ mkdir -p "$RESTART_HOME/state" "$RESTART_HOME/data" "$RESTART_HOME/config" \ "$RESTART_HOME/projects/task" printf '# Seeded Firstmate home\n' > "$RESTART_HOME/AGENTS.md" printf 'restart\n' > "$RESTART_HOME/.fm-secondmate-home" +printf 'schema=fm-secondmate-parent.v1\nroute=remote\n' > "$RESTART_HOME/.fm-secondmate-parent" fm_git_init_commit "$RESTART_HOME/projects/task" cat > "$RESTART_HOME/data/backlog.md" <<'EOF' ## In flight @@ -826,6 +832,7 @@ mkdir -p "$REPORT_HOME/state" "$REPORT_HOME/data" "$REPORT_HOME/config" \ "$REPORT_HOME/projects" printf '# Seeded Firstmate home\n' > "$REPORT_HOME/AGENTS.md" printf 'report\n' > "$REPORT_HOME/.fm-secondmate-home" +printf 'schema=fm-secondmate-parent.v1\nroute=remote\n' > "$REPORT_HOME/.fm-secondmate-parent" cat > "$REPORT_HOME/data/backlog.md" <<'EOF' ## In flight @@ -850,6 +857,7 @@ mkdir -p "$COMPAT_HOME/state" "$COMPAT_HOME/data" "$COMPAT_HOME/config" \ "$COMPAT_HOME/projects" printf '# Seeded Firstmate home\n' > "$COMPAT_HOME/AGENTS.md" printf 'compat\n' > "$COMPAT_HOME/.fm-secondmate-home" +printf 'schema=fm-secondmate-parent.v1\nroute=remote\n' > "$COMPAT_HOME/.fm-secondmate-parent" cat > "$COMPAT_HOME/data/backlog.md" <<'EOF' ## In flight @@ -896,6 +904,7 @@ mkdir -p "$ORDER_HOME/state" "$ORDER_HOME/data" "$ORDER_HOME/config" \ "$ORDER_HOME/projects" "$ORDER_DATE_BIN" printf '# Seeded Firstmate home\n' > "$ORDER_HOME/AGENTS.md" printf 'order\n' > "$ORDER_HOME/.fm-secondmate-home" +printf 'schema=fm-secondmate-parent.v1\nroute=remote\n' > "$ORDER_HOME/.fm-secondmate-parent" cat > "$ORDER_HOME/data/backlog.md" <<'EOF' ## In flight diff --git a/tests/fm-inactive-reconcile.test.sh b/tests/fm-inactive-reconcile.test.sh index 1dbe6dc5afc..779409d5893 100755 --- a/tests/fm-inactive-reconcile.test.sh +++ b/tests/fm-inactive-reconcile.test.sh @@ -140,9 +140,11 @@ test_main_direct_terminal_presentation_receipt() { pass "main direct terminal presentation has a durable receipt" } -# A secondmate independently reports a genuinely terminal inactive child. +# A secondmate independently reports a genuinely terminal inactive child whose +# ledger is silent about it: the authoritative current state says done while +# the last status line is still a progress note. test_local_secondmate_reports_terminal_child() { - make_world local; bind_secondmate local; write_child "$MATE" child 'done: PR https://example.test/owner/repo/pull/1 checks green' + make_world local; bind_secondmate local; write_child "$MATE" child 'working: validation running' FM_FAKE_CREW_STATE='done' run_reconcile "$MATE" --startup grep -Fq 'done [key=inactive-outcome-mate-child-done]:' "$MAIN/state/mate.status" \ || fail "secondmate did not append its durable parent report" @@ -150,11 +152,39 @@ test_local_secondmate_reports_terminal_child() { pass "secondmate reports its own inactive terminal child" } +# A child whose ledger already ends in a terminal verb is the parent mirror's +# evidence (bin/fm-parent-mirror-lib.sh); the inactive scan yields it rather +# than reporting the same outcome a second time on a slower clock. +test_local_secondmate_yields_terminal_ledger_to_mirror() { + make_world yield; bind_secondmate local + write_child "$MATE" child 'done: PR https://example.test/owner/repo/pull/1 checks green' + FM_FAKE_CREW_STATE='done' run_reconcile "$MATE" --startup + [ ! -e "$MAIN/state/mate.status" ] \ + || fail "inactive scan reported a terminal-verb ledger the mirror owns" + [ "$(outcome_count "$MATE" pending)" = 0 ] && [ "$(outcome_count "$MATE" reported)" = 0 ] \ + || fail "inactive scan created a receipt for a ledger the mirror owns" + + make_world yield-partial; bind_secondmate local + write_child "$MATE" child 'working: finishing' + printf 'done: incomplete outcome' >> "$MATE/state/child.status" + age "$MATE/state/child.status" + FM_FAKE_CREW_STATE='done' run_reconcile "$MATE" --startup + grep -Fq 'done [key=inactive-outcome-mate-child-done]:' "$MAIN/state/mate.status" \ + || fail "inactive scan yielded to an unterminated mirror line" + + # A main home keeps its own presentation for the same ledger shape. + make_world yield-main; write_child "$MAIN" child 'done: PR https://example.test/owner/repo/pull/1 checks green' + FM_FAKE_CREW_STATE='done' run_reconcile "$MAIN" --startup + [ "$(wake_count "$MAIN" 'inactive-outcome:')" = 1 ] \ + || fail "main home stopped presenting a terminal-verb ledger" + pass "secondmate inactive scan yields terminal-verb ledgers to the parent mirror" +} + test_local_secondmate_rejects_relative_parent_home() { make_world relative-parent; bind_secondmate local printf 'schema=fm-secondmate-parent.v1\nroute=local\nparent_home=relative-parent\n' \ > "$MATE/.fm-secondmate-parent" - write_child "$MATE" child 'failed: terminal' + write_child "$MATE" child 'working: still validating' (cd "$WORLD" && FM_FAKE_CREW_STATE='failed' run_reconcile "$MATE" --startup) [ ! -e "$WORLD/relative-parent/state/mate.status" ] \ || fail "relative parent home received a false durable report" @@ -173,7 +203,7 @@ test_invalid_secondmate_marker_blocks_routing() { local kind out target for kind in malformed symlink; do make_world "invalid-marker-$kind" - write_child "$MATE" child 'failed: terminal' + write_child "$MATE" child 'working: still validating' if [ "$kind" = malformed ]; then printf '../main\n' > "$MATE/.fm-secondmate-home" else @@ -199,7 +229,7 @@ test_invalid_secondmate_marker_blocks_routing() { # A remote child route writes the existing mirror input once even across restarts. test_remote_parent_reply_is_idempotent() { - make_world remote; bind_secondmate remote; write_child "$MATE" child 'done: green' + make_world remote; bind_secondmate remote; write_child "$MATE" child 'working: checks running' FM_FAKE_CREW_STATE='done' run_reconcile "$MATE" --startup FM_FAKE_CREW_STATE='done' run_reconcile "$MATE" --startup [ "$(grep -c 'inactive-outcome-mate-child-done' "$MATE/state/parent-replies.status")" = 1 ] \ @@ -212,10 +242,10 @@ test_remote_parent_reply_is_idempotent() { # when its terminal state and status text match the retired worker exactly. test_reused_task_id_reports_each_incarnation() { make_world reused-id; bind_secondmate remote - write_child "$MATE" child 'failed: terminal' spawn-one + write_child "$MATE" child 'working: still validating' spawn-one FM_FAKE_CREW_STATE='failed' run_reconcile "$MATE" --startup rm -f "$MATE/state/child.meta" "$MATE/state/child.status" "$MATE/state/child.turn-ended" - write_child "$MATE" child 'failed: terminal' spawn-two + write_child "$MATE" child 'working: still validating' spawn-two FM_FAKE_CREW_STATE='failed' run_reconcile "$MATE" --startup [ "$(outcome_count "$MATE" reported)" = 2 ] \ || fail "reused task id collided with the retired incarnation receipt" @@ -229,7 +259,7 @@ test_reused_task_id_reports_each_incarnation() { test_legacy_metadata_rewrite_keeps_receipt_identity() { local meta tmp make_world legacy-rewrite; bind_secondmate remote - write_child "$MATE" child 'failed: terminal' spawn-old + write_child "$MATE" child 'working: still validating' spawn-old meta="$MATE/state/child.meta" tmp="$MATE/state/.child.meta.legacy" awk '$0 !~ /^spawn_gen=/' "$meta" > "$tmp" @@ -255,7 +285,7 @@ test_legacy_metadata_rewrite_keeps_receipt_identity() { test_relaunch_cannot_replace_metadata_during_state_snapshot() { local recon_pid update_pid record i make_world relaunch-race; bind_secondmate remote - write_child "$MATE" child 'failed: terminal' spawn-old + write_child "$MATE" child 'working: still validating' spawn-old cat > "$WORLD/fakebin/fm-crew-state.sh" <<'SH' #!/usr/bin/env bash : > "${FM_RACE_WORLD:?}/state-started" @@ -422,7 +452,7 @@ test_missing_parent_binding_names_itself() { local out make_world missing-binding printf 'mate\n' > "$MATE/.fm-secondmate-home" - write_child "$MATE" child 'done: PR merged' + write_child "$MATE" child 'working: pipeline finishing' out=$(FM_FAKE_CREW_STATE='done' run_reconcile "$MATE" --startup) case "$out" in *"actionable: inactive terminal outcome needs parent report"*".fm-secondmate-parent"*) ;; @@ -437,7 +467,7 @@ test_notice_recovery_does_not_duplicate_wake() { local record err seq generation make_world notice-recovery; bind_secondmate remote printf 'schema=fm-secondmate-parent.v1\nroute=invalid\n' > "$MATE/.fm-secondmate-parent" - write_child "$MATE" child 'failed: terminal' + write_child "$MATE" child 'working: still validating' FM_FAKE_CREW_STATE='failed' run_reconcile "$MATE" --startup [ "$(wake_count "$MATE" 'inactive-reconcile:')" = 1 ] || fail "parent-report failure did not queue one notice" @@ -468,6 +498,7 @@ test_reconciliation_never_calls_forge() { test_main_direct_terminal_presentation_receipt test_local_secondmate_reports_terminal_child +test_local_secondmate_yields_terminal_ledger_to_mirror test_local_secondmate_rejects_relative_parent_home test_invalid_secondmate_marker_blocks_routing test_remote_parent_reply_is_idempotent diff --git a/tests/fm-parent-mirror.test.sh b/tests/fm-parent-mirror.test.sh new file mode 100755 index 00000000000..2b2d1fdaa87 --- /dev/null +++ b/tests/fm-parent-mirror.test.sh @@ -0,0 +1,949 @@ +#!/usr/bin/env bash +# Behavioral coverage for the secondmate parent-channel mirror +# (bin/fm-parent-mirror.sh, bin/fm-parent-mirror-lib.sh, +# bin/fm-parent-channel-lib.sh): every child ledger event a parent is owed +# reaches the parent channel without the mate model appending anything. +# docs/secondmate-parent-channel.md owns the contract. +set -u + +# shellcheck source=tests/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +MIRROR="$ROOT/bin/fm-parent-mirror.sh" +PR_CHECK="$ROOT/bin/fm-pr-check.sh" +WATCH="$ROOT/bin/fm-watch.sh" +TMP_ROOT=$(fm_test_tmproot fm-parent-mirror) +PR_URL=https://github.com/owner/repo/pull/7 + +make_tools() { # + local world=$1 fake tool + fake="$world/fakebin" + mkdir -p "$fake" "$world/root/bin" + cat > "$fake/tmux" <<'SH' +#!/usr/bin/env bash +case "${1:-}" in + display-message) printf '%%1\n' ;; + capture-pane) printf 'idle\n> \n' ;; +esac +SH + for tool in gh gh-axi curl glab; do + cat > "$fake/$tool" <<'SH' +#!/usr/bin/env bash +printf '%s\n' "$(basename "$0")" >> "${FM_FORGE_LOG:?}" +exit 97 +SH + done + cat > "$world/root/bin/fm-guard.sh" <<'SH' +#!/usr/bin/env bash +exit 0 +SH + chmod +x "$fake"/* "$world/root/bin/fm-guard.sh" +} + +make_world() { # + WORLD="$TMP_ROOT/$1" + MAIN="$WORLD/main" + MATE="$WORLD/mate" + mkdir -p "$MAIN"/{state,data,config,projects} "$MATE"/{state,data,config,projects,bin} + : > "$MATE/AGENTS.md" + make_tools "$WORLD" + : > "$WORLD/forge.log" +} + +bind_secondmate() { # + printf 'mate\n' > "$MATE/.fm-secondmate-home" + if [ "$1" = local ]; then + printf 'schema=fm-secondmate-parent.v1\nroute=local\nparent_home=%s\n' "$MAIN" > "$MATE/.fm-secondmate-parent" + else + printf 'schema=fm-secondmate-parent.v1\nroute=remote\n' > "$MATE/.fm-secondmate-parent" + fi +} + +write_child() { # [kind] [spawn-gen] + local home=$1 id=$2 kind=${3:-ship} spawn_gen=${4:-gen-one} + fm_write_meta "$home/state/$id.meta" \ + "window=firstmate:fm-$id" "worktree=$home/projects/missing-$id" "project=alpha" \ + 'harness=codex' "kind=$kind" 'mode=no-mistakes' 'yolo=off' "spawn_gen=$spawn_gen" +} + +ledger() { # ... + local home=$1 id=$2 + shift 2 + printf '%s\n' "$@" >> "$home/state/$id.status" +} + +sweep() { # [args...] + local home=$1 now=$2 + shift 2 + PATH="$WORLD/fakebin:$PATH" FM_ROOT_OVERRIDE="$WORLD/root" FM_HOME="$home" \ + FM_STATE_OVERRIDE="$home/state" FM_DATA_OVERRIDE="$home/data" FM_CONFIG_OVERRIDE="$home/config" \ + FM_PARENT_MIRROR_NOW="$now" FM_PARENT_MIRROR_OPEN_SECS=60 FM_FORGE_LOG="$WORLD/forge.log" \ + "$MIRROR" sweep "$@" +} + +parent_channel() { printf '%s/state/mate.status\n' "$MAIN"; } + +channel_count() { # + grep -c -F -- "$1" "$(parent_channel)" 2>/dev/null || true +} + +channel_lines() { wc -l < "$(parent_channel)" 2>/dev/null | tr -d ' ' || printf '0'; } + +parent_open_decisions() { + bash -c '. "$1"; status_open_decisions "$2"' _ "$ROOT/bin/fm-classify-lib.sh" "$(parent_channel)" +} + +record_field() { # + grep "^$2=" "$MATE/state/parent-mirror/$1.record" 2>/dev/null | cut -d= -f2- || true +} + +wake_count() { # + grep -c -F -- "$2" "$1/state/.wake-queue" 2>/dev/null || true +} + +prime_seen() { # + FM_STATE_OVERRIDE="$1" bash -c ' + . "$1" + fm_wake_status_mark_current "$2" "$3" + ' _ "$ROOT/bin/fm-wake-lib.sh" "$1" "$2" +} + +reap() { kill "$1" 2>/dev/null || true; wait "$1" 2>/dev/null || true; } + +# A child's done line reaches the parent on the next sweep, carrying the +# recorded PR, the delivery mode, and the merge posture; a repeat sweep +# appends nothing, so the delivery is exactly once. +test_done_line_delivered_once() { + local out size + make_world 'done'; bind_secondmate local + write_child "$MATE" child + printf 'pr=%s\n' "$PR_URL" >> "$MATE/state/child.meta" + ledger "$MATE" child 'working: setup complete' "done: PR $PR_URL checks green" + out=$(sweep "$MATE" 1000) || fail "sweep failed: $out" + [ -z "$out" ] || fail "a clean sweep printed output: $out" + [ "$(channel_count "done [key=mirror-5-child-l")" = 1 ] || fail "done line was not delivered once" + grep -F "mirror: child=child PR $PR_URL checks green pr=$PR_URL mode=no-mistakes yolo=off" "$(parent_channel)" >/dev/null \ + || fail "delivered line lacks the child's note or its recorded context: $(cat "$(parent_channel)")" + sweep "$MATE" 1001 >/dev/null || fail "second sweep failed" + [ "$(channel_lines)" = 1 ] || fail "second sweep duplicated the delivered line" + size=$(wc -c < "$MATE/state/child.status" | tr -d ' ') + [ "$(record_field child offset)" = "$size" ] || fail "record cursor did not advance to the ledger end" + pass "a child's done line is delivered once with its recorded context" +} + +test_incremental_sweep_reads_only_new_ledger_bytes() { + local reader trace size + make_world incremental-span; bind_secondmate local + write_child "$MATE" child + ledger "$MATE" child 'working: initial line' + reader="$WORLD/span-reader" + trace="$WORLD/span-trace" + cat > "$reader" <<'EOF' +#!/usr/bin/env bash +printf '%s\t%s\t%s\n' "$1" "$2" "$3" >> "$FM_SPAN_TRACE" +perl -MFcntl=:DEFAULT -e ' + my ($path, $start, $length) = @ARGV; + sysopen(my $file, $path, O_RDONLY | O_NOFOLLOW) or exit 1; + sysseek($file, $start, 0) == $start or exit 1; + while ($length > 0) { + my $read = sysread($file, my $chunk, $length); + defined($read) && $read > 0 or exit 1; + print $chunk or exit 1; + $length -= $read; + } +' "$@" +EOF + chmod +x "$reader" + FM_STATUS_SPAN_READER="$reader" FM_SPAN_TRACE="$trace" sweep "$MATE" 1000 >/dev/null \ + || fail "initial traced sweep failed" + size=$(wc -c < "$MATE/state/child.status" | tr -d ' ') + : > "$trace" + ledger "$MATE" child 'done: incremental outcome' + FM_STATUS_SPAN_READER="$reader" FM_SPAN_TRACE="$trace" sweep "$MATE" 1001 >/dev/null \ + || fail "incremental traced sweep failed" + awk -F '\t' -v file="$MATE/state/child.status" -v start="$size" ' + $1 == file && $2 == start { found = 1 } + $1 == file && $2 == 0 { bad = 1 } + END { exit !(found && !bad) } + ' "$trace" || fail "second sweep reread the ledger before its durable cursor" + grep -F 'mirror: child=child incremental outcome' "$(parent_channel)" >/dev/null \ + || fail "incremental span did not deliver its appended outcome" + pass "subsequent sweeps read only bytes after the durable cursor" +} + +# A scout's done line carries the report pointer so the parent can read the +# findings without entering the mate home. +test_scout_report_pointer() { + make_world scout; bind_secondmate local + write_child "$MATE" scout scout + mkdir -p "$MATE/data/scout" + printf '# findings\n' > "$MATE/data/scout/report.md" + ledger "$MATE" scout 'done: investigation complete, three findings' + sweep "$MATE" 1000 >/dev/null || fail "sweep failed" + grep -F "mirror: child=scout investigation complete, three findings report=data/scout/report.md" "$(parent_channel)" >/dev/null \ + || fail "scout delivery lacks the report pointer: $(cat "$(parent_channel)")" + pass "a scout's done line carries its report pointer" +} + +test_scratch_paths_refuse_symlink_redirection() { + local target rc + make_world scratch-symlink; bind_secondmate local + write_child "$MATE" child + ledger "$MATE" child 'done: safe scratch test' + mkdir -p "$MATE/state/parent-mirror" + target="$WORLD/external-target" + printf 'sentinel\n' > "$target" + rc=0 + FM_HOME="$MATE" FM_STATE_OVERRIDE="$MATE/state" FM_DATA_OVERRIDE="$MATE/data" bash -c ' + . "$1" + ln -s "$2" "$STATE/parent-mirror/child.record.capture.$$" + fm_parent_mirror_sweep_child_locked child + ' _ "$ROOT/bin/fm-parent-mirror-lib.sh" "$target" >/dev/null 2>&1 || rc=$? + [ "$rc" -eq 0 ] || fail "safe scratch allocation refused a harmless stale predictable symlink" + [ "$(cat "$target")" = sentinel ] || fail "predictable scratch symlink changed its target" + + make_world scratch-dir-symlink; bind_secondmate local + write_child "$MATE" child + ledger "$MATE" child 'done: unsafe directory test' + mkdir -p "$WORLD/external-dir" + printf 'sentinel\n' > "$WORLD/external-dir/target" + ln -s "$WORLD/external-dir" "$MATE/state/parent-mirror" + rc=0 + FM_HOME="$MATE" FM_STATE_OVERRIDE="$MATE/state" FM_DATA_OVERRIDE="$MATE/data" bash -c ' + . "$1"; fm_parent_mirror_sweep_child_locked child + ' _ "$ROOT/bin/fm-parent-mirror-lib.sh" >/dev/null 2>&1 || rc=$? + [ "$rc" -ne 0 ] || fail "mirror accepted a symlinked scratch directory" + [ "$(cat "$WORLD/external-dir/target")" = sentinel ] || fail "symlinked mirror directory changed external data" + [ ! -e "$WORLD/external-dir/child.record" ] || fail "symlinked mirror directory received a record" + pass "mirror scratch allocation refuses symlink redirection" +} + +# Only whole lines are delivered: a line still being appended waits for its +# newline, then is delivered on the next sweep. +test_partial_line_waits_for_newline() { + make_world partial; bind_secondmate local + write_child "$MATE" child + printf 'done: PR %s checks green' "$PR_URL" > "$MATE/state/child.status" + sweep "$MATE" 1000 >/dev/null || fail "sweep failed" + [ ! -e "$(parent_channel)" ] || fail "an unterminated line was delivered: $(cat "$(parent_channel)")" + printf '\n' >> "$MATE/state/child.status" + sweep "$MATE" 1001 >/dev/null || fail "second sweep failed" + [ "$(channel_count "done [key=mirror-5-child-l0]")" = 1 ] || fail "the completed line was not delivered" + pass "an unterminated ledger line waits for its newline" +} + +# Unterminated thresholded outcomes do not become standing state and do not +# begin aging until their terminating newline is captured. +test_orphan_retains_unterminated_tail() { + make_world orphan-tail; bind_secondmate local + write_child "$MATE" child + printf 'done: still being written' > "$MATE/state/child.status" + sweep "$MATE" 1000 >/dev/null || fail "tail priming sweep failed" + rm -f "$MATE/state/child.meta" + sweep "$MATE" 1001 >/dev/null || fail "partial orphan sweep failed" + [ "$(record_field child tail)" = 1 ] || fail "orphan record did not retain its incomplete tail" + [ ! -e "$(parent_channel)" ] || fail "partial orphan tail was delivered" + printf '\n' >> "$MATE/state/child.status" + sweep "$MATE" 1002 >/dev/null || fail "completed orphan sweep failed" + grep -F 'done [key=mirror-5-child-l0]: mirror: child=child still being written' "$(parent_channel)" >/dev/null \ + || fail "completed orphan tail was not delivered" + [ ! -e "$MATE/state/parent-mirror/child.record" ] || fail "completed orphan record was not retired" + pass "an orphan retains an unterminated tail until it completes" +} + +test_partial_thresholded_outcomes_do_not_age() { + make_world partial-failed; bind_secondmate local + write_child "$MATE" child + printf 'failed: build still writing' > "$MATE/state/child.status" + sweep "$MATE" 1000 >/dev/null || fail "partial failure first sweep failed" + sweep "$MATE" 1060 >/dev/null || fail "partial failure second sweep failed" + [ -z "$(record_field child terminal_line)" ] || fail "an unterminated failure became standing state" + [ ! -e "$(parent_channel)" ] || fail "an unterminated failure was delivered" + printf '\n' >> "$MATE/state/child.status" + sweep "$MATE" 1061 >/dev/null || fail "completed failure first sweep failed" + [ "$(record_field child terminal_first_seen)" = 1061 ] || fail "failure aged before its newline" + [ ! -e "$(parent_channel)" ] || fail "a newly completed failure skipped the threshold" + sweep "$MATE" 1121 >/dev/null || fail "completed failure threshold sweep failed" + grep -F 'failed [key=mirror-5-child-l0]: mirror: child=child build still writing (unhandled past 60s)' "$(parent_channel)" >/dev/null \ + || fail "the completed failure was not delivered after its full threshold" + + make_world partial-decision; bind_secondmate local + write_child "$MATE" child + printf 'needs-decision [key=api]: choose one' > "$MATE/state/child.status" + sweep "$MATE" 2000 >/dev/null || fail "partial decision first sweep failed" + sweep "$MATE" 2060 >/dev/null || fail "partial decision second sweep failed" + [ -z "$(record_field child open)" ] || fail "an unterminated decision became open state" + [ ! -e "$(parent_channel)" ] || fail "an unterminated decision was delivered" + printf '\n' >> "$MATE/state/child.status" + sweep "$MATE" 2061 >/dev/null || fail "completed decision first sweep failed" + [ ! -e "$(parent_channel)" ] || fail "a newly completed decision skipped the threshold" + sweep "$MATE" 2121 >/dev/null || fail "completed decision threshold sweep failed" + grep -F 'needs-decision [key=mirror-5-child-api]' "$(parent_channel)" >/dev/null \ + || fail "the completed decision was not delivered after its full threshold" + pass "unterminated failures and decisions neither stand nor age" +} + +# An open decision is raised only after it stands past the threshold, is +# keyed so the parent's own fold opens it, and is closed on the parent when +# the child's fold closes it. +test_open_decision_thresholded_then_closed() { + make_world decision; bind_secondmate local + write_child "$MATE" child + ledger "$MATE" child 'needs-decision [key=api]: choose A or B' + sweep "$MATE" 1000 >/dev/null || fail "first sweep failed" + [ ! -e "$(parent_channel)" ] || fail "a fresh decision was raised before the threshold: $(cat "$(parent_channel)")" + sweep "$MATE" 1030 >/dev/null || fail "second sweep failed" + [ ! -e "$(parent_channel)" ] || fail "a decision inside the threshold was raised" + sweep "$MATE" 1060 >/dev/null || fail "third sweep failed" + grep -F 'needs-decision [key=mirror-5-child-api]: mirror: child=child decision api (opened at line 1) open past 60s without an answer or a captain hold: choose A or B' "$(parent_channel)" >/dev/null \ + || fail "the standing decision was not raised: $(cat "$(parent_channel)")" + parent_open_decisions | grep -q "^mirror-5-child-api needs-decision " \ + || fail "the parent's fold did not open the mirrored decision" + sweep "$MATE" 1200 >/dev/null || fail "fourth sweep failed" + [ "$(channel_lines)" = 1 ] || fail "a standing decision was raised twice" + ledger "$MATE" child 'resolved [key=api]: chose A' + sweep "$MATE" 1300 >/dev/null || fail "closing sweep failed" + grep -F 'resolved [key=mirror-5-child-api]: mirror: child=child decision api (opened at line 1) closed' "$(parent_channel)" >/dev/null \ + || fail "the closed decision was not closed on the parent: $(cat "$(parent_channel)")" + [ -z "$(parent_open_decisions)" ] || fail "the parent's fold still holds the closed decision: $(parent_open_decisions)" + pass "an open decision is raised past the threshold and closed with the child's own close" +} + +test_open_decision_updates_retain_occurrence() { + make_world decision-updates; bind_secondmate local + write_child "$MATE" child + ledger "$MATE" child 'needs-decision [key=api]: initial choice' + sweep "$MATE" 1000 >/dev/null || fail "initial decision observation failed" + ledger "$MATE" child 'needs-decision [key=api]: refined choice' + sweep "$MATE" 1020 >/dev/null || fail "first decision update failed" + ledger "$MATE" child 'blocked [key=api]: latest blocking detail' + sweep "$MATE" 1040 >/dev/null || fail "second decision update failed" + [ ! -e "$(parent_channel)" ] || fail "an updated decision was raised before its original threshold" + sweep "$MATE" 1060 >/dev/null || fail "updated decision delivery failed" + grep -F 'blocked [key=mirror-5-child-api]: mirror: child=child decision api (opened at line 1) open past 60s without an answer or a captain hold: latest blocking detail' "$(parent_channel)" >/dev/null \ + || fail "updated decision did not retain its origin and latest state: $(cat "$(parent_channel)")" + [ "$(channel_lines)" = 1 ] || fail "updated decision was raised more than once" + parent_open_decisions | grep -q '^mirror-5-child-api' \ + || fail "the updated decision was not open on the parent" + ledger "$MATE" child 'needs-decision [key=api]: final open detail' + sweep "$MATE" 1061 >/dev/null || fail "mirrored decision update failed" + [ "$(channel_lines)" = 1 ] || fail "a mirrored decision update emitted a false close or reopen" + parent_open_decisions | grep -q '^mirror-5-child-api' \ + || fail "a mirrored decision update falsely closed the parent decision" + ledger "$MATE" child 'resolved [key=api]: explicit answer' + sweep "$MATE" 1070 >/dev/null || fail "updated decision close failed" + [ "$(channel_count 'resolved [key=mirror-5-child-api]')" = 1 ] \ + || fail "updated decision did not close exactly once" + [ -z "$(parent_open_decisions)" ] || fail "explicit close left the updated parent decision open" + pass "decision updates retain their original occurrence and latest state" +} + +test_ledger_reset_closes_decisions_and_clears_failures() { + make_world reset-decision; bind_secondmate local + write_child "$MATE" child + ledger "$MATE" child 'needs-decision [key=api]: choose before reset' + sweep "$MATE" 1000 >/dev/null || fail "reset decision observation failed" + sweep "$MATE" 1060 >/dev/null || fail "reset decision delivery failed" + parent_open_decisions | grep -q '^mirror-5-child-api' \ + || fail "reset fixture did not open its parent decision" + printf 'working: replacement generation\n' > "$MATE/state/child.status.next" + mv "$MATE/state/child.status.next" "$MATE/state/child.status" \ + || fail "could not replace the decision ledger" + sweep "$MATE" 1070 >/dev/null || fail "replacement decision sweep failed" + grep -F 'resolved [key=mirror-5-child-api]: mirror: child=child decision api (opened at line 1) closed' "$(parent_channel)" >/dev/null \ + || fail "ledger replacement did not close the mirrored decision" + [ -z "$(parent_open_decisions)" ] || fail "ledger replacement left the parent decision open" + + make_world reset-failure; bind_secondmate local + write_child "$MATE" child + ledger "$MATE" child 'failed: old generation failure' + sweep "$MATE" 2000 >/dev/null || fail "reset failure observation failed" + printf 'working: replacement generation\n' > "$MATE/state/child.status.next" + mv "$MATE/state/child.status.next" "$MATE/state/child.status" \ + || fail "could not replace the failed ledger" + sweep "$MATE" 2010 >/dev/null || fail "replacement failure sweep failed" + sweep "$MATE" 2060 >/dev/null || fail "old failure threshold sweep failed" + [ ! -e "$(parent_channel)" ] || fail "old failure survived ledger replacement" + ledger "$MATE" child 'failed: replacement generation failure' + sweep "$MATE" 2070 >/dev/null || fail "replacement failure observation failed" + [ ! -e "$(parent_channel)" ] || fail "replacement failure skipped its own threshold" + sweep "$MATE" 2130 >/dev/null || fail "replacement failure delivery failed" + grep -F 'replacement generation failure (unhandled past 60s)' "$(parent_channel)" >/dev/null \ + || fail "new failure after ledger replacement was not delivered" + if grep -F 'old generation failure' "$(parent_channel)" >/dev/null; then + fail "stale failure from the replaced ledger was delivered" + fi + pass "ledger resets close decisions and discard stale failures" +} + +test_failed_decision_close_retries_without_child_change() { + local rc=0 + make_world close-retry; bind_secondmate local + write_child "$MATE" child + ledger "$MATE" child 'needs-decision [key=api]: choose A or B' + sweep "$MATE" 1000 >/dev/null || fail "first sweep failed" + sweep "$MATE" 1060 >/dev/null || fail "opening sweep failed" + ledger "$MATE" child 'resolved [key=api]: chose A' + chmod 0400 "$(parent_channel)" + sweep "$MATE" 1100 >/dev/null 2>&1 || rc=$? + chmod 0600 "$(parent_channel)" + [ "$rc" -ne 0 ] || fail "an unwritable parent close reported success" + [ "$(channel_count 'resolved [key=mirror-5-child-api]')" = 0 ] || fail "the failed close was unexpectedly delivered" + sweep "$MATE" 1101 >/dev/null || fail "unchanged retry sweep failed" + [ "$(channel_count 'resolved [key=mirror-5-child-api]')" = 1 ] || fail "the failed close was not retried" + [ -z "$(parent_open_decisions)" ] || fail "the retried close left the parent decision open" + pass "a failed decision close retries without another child append" +} + +# The same key re-opened after a close, even with the same note, is a new +# opening: it is raised and closed again rather than lost to deduplication. +test_reopened_decision_is_raised_again() { + make_world reopen; bind_secondmate local + write_child "$MATE" child + ledger "$MATE" child 'needs-decision [key=api]: choose A or B' + sweep "$MATE" 1000 >/dev/null || fail "first sweep failed" + sweep "$MATE" 1060 >/dev/null || fail "second sweep failed" + ledger "$MATE" child 'resolved [key=api]: chose A' + sweep "$MATE" 1100 >/dev/null || fail "closing sweep failed" + ledger "$MATE" child 'needs-decision [key=api]: choose A or B' + sweep "$MATE" 1200 >/dev/null || fail "re-open sweep failed" + sweep "$MATE" 1260 >/dev/null || fail "re-open threshold sweep failed" + grep -F 'decision api (opened at line 3) open past 60s' "$(parent_channel)" >/dev/null \ + || fail "the re-opened decision was not raised: $(cat "$(parent_channel)")" + parent_open_decisions | grep -q "^mirror-5-child-api needs-decision " \ + || fail "the parent's fold did not re-open the decision" + ledger "$MATE" child 'resolved [key=api]: chose B after all' + sweep "$MATE" 1300 >/dev/null || fail "second closing sweep failed" + grep -F 'decision api (opened at line 3) closed' "$(parent_channel)" >/dev/null \ + || fail "the re-opened decision was not closed again: $(cat "$(parent_channel)")" + [ "$(channel_lines)" = 4 ] || fail "expected two openings and two closes, got: $(cat "$(parent_channel)")" + [ -z "$(parent_open_decisions)" ] || fail "the parent's fold still holds the re-closed decision" + pass "a decision re-opened under the same key is raised and closed again" +} + +# A decision the mate answers, or transfers to a captain hold, inside the +# threshold is never raised: the mate stays the first responder. +test_decision_handled_inside_threshold_is_silent() { + make_world handled; bind_secondmate local + write_child "$MATE" child + ledger "$MATE" child 'blocked [key=creds]: need the deploy token' + sweep "$MATE" 1000 >/dev/null || fail "first sweep failed" + ledger "$MATE" child 'resolved [key=creds]: token provided' + sweep "$MATE" 1030 >/dev/null || fail "second sweep failed" + sweep "$MATE" 2000 >/dev/null || fail "third sweep failed" + [ ! -e "$(parent_channel)" ] || fail "an answered blocker was raised: $(cat "$(parent_channel)")" + ledger "$MATE" child 'needs-decision [key=scope]: widen the migration?' + sweep "$MATE" 2001 >/dev/null || fail "fourth sweep failed" + ledger "$MATE" child 'captain-held [key=scope]: transferred to task scope-call' + sweep "$MATE" 3000 >/dev/null || fail "fifth sweep failed" + [ ! -e "$(parent_channel)" ] || fail "a captain-held transfer inside the threshold was raised: $(cat "$(parent_channel)")" + pass "decisions and blockers handled inside the threshold stay silent" +} + +# A failed line that keeps standing as the last line is raised past the +# threshold with deterministic text; one superseded by a relaunch is not. +test_failed_line_thresholded_and_superseded() { + make_world 'failed'; bind_secondmate local + write_child "$MATE" child + ledger "$MATE" child 'failed: build broke on main' + sweep "$MATE" 1000 >/dev/null || fail "first sweep failed" + [ ! -e "$(parent_channel)" ] || fail "a fresh failure was raised before the threshold" + sweep "$MATE" 1060 >/dev/null || fail "second sweep failed" + grep -F 'failed [key=mirror-5-child-l0]: mirror: child=child build broke on main (unhandled past 60s) mode=no-mistakes yolo=off' "$(parent_channel)" >/dev/null \ + || fail "the standing failure was not raised: $(cat "$(parent_channel)")" + sweep "$MATE" 1200 >/dev/null || fail "third sweep failed" + [ "$(channel_lines)" = 1 ] || fail "a standing failure was raised twice" + + make_world trailing-blank-failure; bind_secondmate local + write_child "$MATE" child + ledger "$MATE" child 'failed: blank lines followed' '' + FM_HOME="$MATE" FM_STATE_OVERRIDE="$MATE/state" FM_DATA_OVERRIDE="$MATE/data" bash -c ' + . "$1"; fm_parent_mirror_owns_ledger "$2" child + ' _ "$ROOT/bin/fm-parent-mirror-lib.sh" "$MATE/state" \ + || fail "inactive reconciliation did not yield the trailing-blank failure to the mirror" + sweep "$MATE" 1000 >/dev/null || fail "trailing-blank failure observation failed" + [ ! -e "$(parent_channel)" ] || fail "trailing-blank failure skipped its threshold" + sweep "$MATE" 1060 >/dev/null || fail "trailing-blank failure threshold failed" + grep -F 'failed [key=mirror-5-child-l0]: mirror: child=child blank lines followed (unhandled past 60s)' "$(parent_channel)" >/dev/null \ + || fail "trailing blank lines suppressed the standing failure" + sweep "$MATE" 1200 >/dev/null || fail "trailing-blank failure repeat sweep failed" + [ "$(channel_count 'failed [key=mirror-5-child-l0]')" = 1 ] \ + || fail "trailing-blank failure was not delivered exactly once" + + make_world relaunched; bind_secondmate local + write_child "$MATE" child + ledger "$MATE" child 'failed: build broke on main' + sweep "$MATE" 1000 >/dev/null || fail "relaunch first sweep failed" + ledger "$MATE" child 'working: relaunched on a clean base' + sweep "$MATE" 1060 >/dev/null || fail "relaunch second sweep failed" + sweep "$MATE" 2000 >/dev/null || fail "relaunch third sweep failed" + [ ! -e "$(parent_channel)" ] || fail "a superseded failure was raised: $(cat "$(parent_channel)")" + + make_world incarnation-relaunch; bind_secondmate local + write_child "$MATE" child + ledger "$MATE" child 'failed: prior generation failed' + sweep "$MATE" 3000 >/dev/null || fail "incarnation first observation failed" + sed 's/^spawn_gen=.*/spawn_gen=gen-two/' "$MATE/state/child.meta" > "$MATE/state/child.meta.next" \ + || fail "could not prepare the relaunch generation" + mv "$MATE/state/child.meta.next" "$MATE/state/child.meta" \ + || fail "could not record the relaunch generation" + sweep "$MATE" 3010 >/dev/null || fail "incarnation change sweep failed" + sweep "$MATE" 3060 >/dev/null || fail "incarnation old-threshold sweep failed" + [ ! -e "$(parent_channel)" ] || fail "a prior-generation failure survived relaunch handling" + ledger "$MATE" child 'failed: new generation failed' + sweep "$MATE" 3070 >/dev/null || fail "new-generation failure observation failed" + [ ! -e "$(parent_channel)" ] || fail "a new-generation failure skipped its own threshold" + sweep "$MATE" 3130 >/dev/null || fail "new-generation failure threshold failed" + [ "$(channel_count 'failed [key=mirror-5-child-l')" = 1 ] \ + || fail "the new-generation failure was not delivered exactly once" + grep -F 'new generation failed (unhandled past 60s)' "$(parent_channel)" >/dev/null \ + || fail "the delivered failure did not belong to the new generation" + + make_world recurring; bind_secondmate local + write_child "$MATE" child + ledger "$MATE" child 'failed: same failure' + sweep "$MATE" 3000 >/dev/null || fail "recurring first observation failed" + sweep "$MATE" 3060 >/dev/null || fail "recurring first delivery failed" + ledger "$MATE" child 'working: retrying' 'failed: same failure' + sweep "$MATE" 3070 >/dev/null || fail "recurring second observation failed" + sweep "$MATE" 3130 >/dev/null || fail "recurring second delivery failed" + [ "$(channel_count 'failed [key=mirror-5-child-l')" = 2 ] \ + || fail "an identical recurring failure was not delivered as a new event" + pass "standing failures are offset-identified and superseded failures stay silent" +} + +# A decision line the fold cannot track is delivered at once under the done +# verb, so it surfaces without opening a parent decision nothing can close. +test_untracked_decision_line_is_delivered_as_done() { + make_world untracked; bind_secondmate local + write_child "$MATE" child + ledger "$MATE" child 'needs-decision [key=bad key]: which one' + sweep "$MATE" 1000 >/dev/null || fail "sweep failed" + grep -F 'done [key=mirror-5-child-l0]: mirror: child=child untracked needs-decision line: which one' "$(parent_channel)" >/dev/null \ + || fail "the untracked decision line was not delivered: $(cat "$(parent_channel)")" + [ -z "$(parent_open_decisions)" ] || fail "an untracked decision opened a parent decision" + pass "an untrackable decision line is delivered without opening a parent decision" +} + +# A remote route writes the mate's own parent-replies log, the input the +# parent's remote reply adapter already mirrors. +test_injective_keys_and_concurrent_append() { + local i pid pids='' failures=0 + make_world injective; bind_secondmate local + write_child "$MATE" a-b + write_child "$MATE" a + ledger "$MATE" a-b 'needs-decision [key=c]: first choice' + ledger "$MATE" a 'needs-decision [key=b-c]: second choice' + sweep "$MATE" 1000 >/dev/null || fail "injective first sweep failed" + sweep "$MATE" 1060 >/dev/null || fail "injective threshold sweep failed" + parent_open_decisions | grep -q '^mirror-3-a-b-c' || fail "first child decision key was not preserved" + parent_open_decisions | grep -q '^mirror-1-a-b-c' || fail "second child decision key was not preserved" + ledger "$MATE" a-b 'resolved [key=c]: first chosen' + sweep "$MATE" 1070 >/dev/null || fail "injective close sweep failed" + parent_open_decisions | grep -q '^mirror-1-a-b-c' || fail "closing one child closed the colliding sibling" + + make_world concurrent; bind_secondmate local + i=0 + while [ "$i" -lt 12 ]; do + FM_HOME="$MATE" FM_STATE_OVERRIDE="$MATE/state" bash -c ' + . "$1"; fm_parent_channel_report "$2" "$3" "done [key=concurrent]: one outcome" + ' _ "$ROOT/bin/fm-parent-channel-lib.sh" "$MATE" "$MATE/state" & + pid=$! + pids="${pids:-} $pid" + i=$((i + 1)) + done + for pid in $pids; do wait "$pid" || failures=$((failures + 1)); done + [ "$failures" -eq 0 ] || fail "a concurrent parent publication failed" + [ "$(channel_count 'done [key=concurrent]: one outcome')" = 1 ] \ + || fail "concurrent exact-line publications duplicated the outcome" + [ -z "$(find "$MAIN/state" -name '.parent-channel-*' -print -quit)" ] \ + || fail "a destination lock artifact landed in the parent home" + pass "mirrored keys are injective and concurrent appends stay idempotent" +} + +test_parent_channel_repairs_unterminated_tail() { + local pid rc i + make_world channel-tail; bind_secondmate local + printf 'done [key=tail]: intended outcome' > "$(parent_channel)" + FM_HOME="$MATE" FM_STATE_OVERRIDE="$MATE/state" bash -c ' + . "$1"; fm_parent_channel_report "$2" "$3" "done [key=tail]: intended outcome" + ' _ "$ROOT/bin/fm-parent-channel-lib.sh" "$MATE" "$MATE/state" \ + || fail "retrying an unterminated intended line failed" + [ "$(channel_lines)" = 1 ] || fail "unterminated intended text was duplicated" + grep -Fx 'done [key=tail]: intended outcome' "$(parent_channel)" >/dev/null \ + || fail "unterminated intended text was not repaired as a complete record" + + printf 'partial fragment' > "$(parent_channel)" + FM_HOME="$MATE" FM_STATE_OVERRIDE="$MATE/state" bash -c ' + . "$1"; fm_parent_channel_report "$2" "$3" "done [key=next]: next outcome" + ' _ "$ROOT/bin/fm-parent-channel-lib.sh" "$MATE" "$MATE/state" \ + || fail "appending after an unterminated fragment failed" + [ "$(channel_lines)" = 2 ] || fail "the repaired fragment and next outcome were concatenated" + grep -Fx 'partial fragment' "$(parent_channel)" >/dev/null \ + || fail "the unterminated fragment was not delimited" + grep -Fx 'done [key=next]: next outcome' "$(parent_channel)" >/dev/null \ + || fail "the next outcome was malformed after tail repair" + + make_world channel-fifo; bind_secondmate local + mkfifo "$(parent_channel)" || fail "could not create the channel FIFO fixture" + FM_HOME="$MATE" FM_STATE_OVERRIDE="$MATE/state" bash -c ' + . "$1"; fm_parent_channel_report "$2" "$3" "done [key=fifo]: blocked outcome" + ' _ "$ROOT/bin/fm-parent-channel-lib.sh" "$MATE" "$MATE/state" >/dev/null 2>&1 & + pid=$! + i=0 + while kill -0 "$pid" 2>/dev/null && [ "$i" -lt 30 ]; do sleep 0.1; i=$((i + 1)); done + if kill -0 "$pid" 2>/dev/null; then + kill "$pid" 2>/dev/null || true + wait "$pid" 2>/dev/null || true + fail "parent append blocked on a FIFO destination" + fi + rc=0 + wait "$pid" || rc=$? + [ "$rc" -ne 0 ] || fail "parent append accepted a FIFO destination" + [ -p "$(parent_channel)" ] || fail "parent append replaced or removed the FIFO" + pass "parent appends repair tails and reject non-regular destinations" +} + +test_remote_route_writes_parent_replies() { + make_world remote; bind_secondmate remote + write_child "$MATE" child + ledger "$MATE" child "done: PR $PR_URL checks green" + sweep "$MATE" 1000 >/dev/null || fail "sweep failed" + grep -F "done [key=mirror-5-child-l0]: mirror: child=child PR $PR_URL checks green" "$MATE/state/parent-replies.status" >/dev/null \ + || fail "remote route did not write parent-replies.status" + [ ! -e "$MAIN/state/mate.status" ] || fail "remote route wrote a local parent file" + pass "a remote route delivers into the mate's parent-replies log" +} + +# A PR registration delivers the child's ready line at registration time, with +# the canonical pr= just recorded, and leaves nothing for the next sweep. +test_pr_check_registration_delivers_now() { + local out size holder i lock + make_world pr-check; bind_secondmate local + write_child "$MATE" child + ledger "$MATE" child "done: PR $PR_URL checks green" + out=$(PATH="$WORLD/fakebin:$PATH" FM_ROOT_OVERRIDE="$WORLD/root" FM_HOME="$MATE" \ + FM_STATE_OVERRIDE="$MATE/state" FM_DATA_OVERRIDE="$MATE/data" FM_FORGE_LOG="$WORLD/forge.log" \ + "$PR_CHECK" child "$PR_URL" 2>&1) || fail "fm-pr-check failed: $out" + case "$out" in *"armed: state/child.check.sh"*) ;; *) fail "fm-pr-check did not arm: $out" ;; esac + case "$out" in *actionable:*) fail "fm-pr-check reported a delivery problem: $out" ;; esac + grep -F "done [key=mirror-5-child-l0]: mirror: child=child PR $PR_URL checks green pr=$PR_URL" "$(parent_channel)" >/dev/null \ + || fail "registration did not deliver the ready line with pr=: $(cat "$(parent_channel)" 2>/dev/null)" + size=$(wc -c < "$MATE/state/child.status" | tr -d ' ') + [ "$(record_field child offset)" = "$size" ] || fail "registration left the ledger for the next sweep" + sweep "$MATE" 1000 >/dev/null || fail "sweep after registration failed" + [ "$(channel_lines)" = 1 ] || fail "the next sweep duplicated the registration delivery" + + make_world pr-check-contention; bind_secondmate local + write_child "$MATE" child + ledger "$MATE" child "done: PR $PR_URL checks green" + lock="$MATE/state/.parent-mirror.lock" + FM_STATE_OVERRIDE="$MATE/state" bash -c ' + . "$1/bin/fm-wake-lib.sh" + fm_lock_acquire_wait "$2" + : > "$3" + sleep 30 + ' _ "$ROOT" "$lock" "$WORLD/sweep-held" & + holder=$! + i=0 + while [ "$i" -lt 50 ] && [ ! -e "$WORLD/sweep-held" ]; do sleep 0.1; i=$((i + 1)); done + [ -e "$WORLD/sweep-held" ] || fail "registration sweep lock holder did not start" + out=$(PATH="$WORLD/fakebin:$PATH" FM_ROOT_OVERRIDE="$WORLD/root" FM_HOME="$MATE" \ + FM_STATE_OVERRIDE="$MATE/state" FM_DATA_OVERRIDE="$MATE/data" FM_FORGE_LOG="$WORLD/forge.log" \ + FM_PARENT_MIRROR_LOCK_WAIT_SECS=1 "$PR_CHECK" child "$PR_URL" 2>&1) \ + || fail "contended fm-pr-check failed: $out" + reap "$holder" + case "$out" in *"armed: state/child.check.sh"*) ;; *) fail "contended registration did not arm: $out" ;; esac + case "$out" in *"actionable:"*"rc=5"*) ;; *) fail "contended registration was not actionable: $out" ;; esac + [ ! -e "$(parent_channel)" ] || fail "contended registration unexpectedly delivered" + pass "a PR registration delivers now or reports contention as actionable" +} + +# The retire path a teardown runs: the child's final line is delivered before +# its record goes, its mirror state is retired, and when delivery fails the +# record is kept as an orphan that a later sweep still delivers. +test_retire_delivers_then_orphan_retries() { + local rc + make_world retire; bind_secondmate local + write_child "$MATE" scout scout + ledger "$MATE" scout 'done: report ready' + FM_HOME="$MATE" FM_STATE_OVERRIDE="$MATE/state" FM_DATA_OVERRIDE="$MATE/data" bash -c ' + . "$1"; fm_parent_mirror_retire_locked scout + ' _ "$ROOT/bin/fm-parent-mirror-lib.sh" || fail "retire failed" + grep -F 'mirror: child=scout report ready' "$(parent_channel)" >/dev/null || fail "retire did not deliver the final line" + [ ! -e "$MATE/state/parent-mirror/scout.record" ] || fail "retire did not remove the mirror record" + + make_world orphan; bind_secondmate local + write_child "$MATE" scout scout + printf 'pr=%s\n' "$PR_URL" >> "$MATE/state/scout.meta" + mkdir -p "$MATE/data/scout" + printf '# retained report\n' > "$MATE/data/scout/report.md" + ledger "$MATE" scout 'done: report ready' + mkdir -p "$MAIN/state" + chmod 0500 "$MAIN/state" + rc=0 + FM_HOME="$MATE" FM_STATE_OVERRIDE="$MATE/state" FM_DATA_OVERRIDE="$MATE/data" bash -c ' + . "$1"; fm_parent_mirror_retire_locked scout + ' _ "$ROOT/bin/fm-parent-mirror-lib.sh" 2>/dev/null || rc=$? + chmod 0755 "$MAIN/state" + [ "$rc" -ne 0 ] || fail "retire reported success while the parent channel was unwritable" + [ "$(record_field scout orphan)" = 1 ] || fail "a failed retire did not keep an orphan record" + rm -f "$MATE/state/scout.meta" + sweep "$MATE" 1000 >/dev/null || fail "orphan sweep failed" + grep -F "mirror: child=scout report ready pr=$PR_URL report=data/scout/report.md mode=no-mistakes yolo=off" "$(parent_channel)" >/dev/null \ + || fail "the orphan retry lost its recorded metadata context" + [ ! -e "$MATE/state/parent-mirror/scout.record" ] || fail "a delivered orphan record was not removed" + make_world retire-tail; bind_secondmate local + write_child "$MATE" child + printf 'done: teardown tail' > "$MATE/state/child.status" + FM_HOME="$MATE" FM_STATE_OVERRIDE="$MATE/state" FM_DATA_OVERRIDE="$MATE/data" bash -c ' + . "$1"; fm_parent_mirror_retire_locked child + ' _ "$ROOT/bin/fm-parent-mirror-lib.sh" || fail "partial-tail retire failed" + [ "$(record_field child orphan)" = 1 ] && [ "$(record_field child tail)" = 1 ] \ + || fail "retire did not preserve its incomplete tail as an orphan" + rm -f "$MATE/state/child.meta" + printf '\n' >> "$MATE/state/child.status" + sweep "$MATE" 1001 >/dev/null || fail "retired-tail retry failed" + grep -F 'mirror: child=child teardown tail' "$(parent_channel)" >/dev/null \ + || fail "retired tail was not delivered after completion" + [ ! -e "$MATE/state/parent-mirror/child.record" ] || fail "retired-tail record was not removed" + pass "retire and orphan paths preserve every undelivered byte" +} + +# Retirement ends first-responder authority, so thresholded outcomes are due +# immediately and their record survives until delivery succeeds. +test_retired_and_orphaned_outcomes_deliver_immediately() { + make_world retire-failed; bind_secondmate local + write_child "$MATE" child + ledger "$MATE" child 'failed: build broke before teardown' + FM_HOME="$MATE" FM_STATE_OVERRIDE="$MATE/state" FM_DATA_OVERRIDE="$MATE/data" \ + FM_PARENT_MIRROR_NOW=1000 FM_PARENT_MIRROR_OPEN_SECS=60 bash -c ' + . "$1"; fm_parent_mirror_retire_locked child + ' _ "$ROOT/bin/fm-parent-mirror-lib.sh" || fail "failed-child retire failed" + grep -F 'failed [key=mirror-5-child-l0]: mirror: child=child build broke before teardown (unhandled when child retired)' "$(parent_channel)" >/dev/null \ + || fail "retire did not immediately deliver the standing failure" + [ ! -e "$MATE/state/parent-mirror/child.record" ] || fail "retire kept a delivered failure record" + + make_world retire-decision; bind_secondmate local + write_child "$MATE" child + ledger "$MATE" child 'needs-decision [key=ship]: choose release train' + FM_HOME="$MATE" FM_STATE_OVERRIDE="$MATE/state" FM_DATA_OVERRIDE="$MATE/data" \ + FM_PARENT_MIRROR_NOW=1000 FM_PARENT_MIRROR_OPEN_SECS=60 bash -c ' + . "$1"; fm_parent_mirror_retire_locked child + ' _ "$ROOT/bin/fm-parent-mirror-lib.sh" || fail "decision-child retire failed" + grep -F 'needs-decision [key=mirror-5-child-ship]: mirror: child=child decision ship (opened at line 1) open when child retired' "$(parent_channel)" >/dev/null \ + || fail "retire did not immediately deliver the open decision" + [ ! -e "$MATE/state/parent-mirror/child.record" ] || fail "retire kept a delivered decision record" + + make_world orphan-failed; bind_secondmate local + write_child "$MATE" child + ledger "$MATE" child 'failed: orphan build broke' + sweep "$MATE" 2000 >/dev/null || fail "orphan failure priming sweep failed" + rm -f "$MATE/state/child.meta" + sweep "$MATE" 2001 >/dev/null || fail "orphan failure sweep failed" + grep -F 'orphan build broke (unhandled when child retired)' "$(parent_channel)" >/dev/null \ + || fail "orphan sweep did not immediately deliver the standing failure" + [ ! -e "$MATE/state/parent-mirror/child.record" ] || fail "orphan failure record was removed too late" + + make_world orphan-decision; bind_secondmate local + write_child "$MATE" child + ledger "$MATE" child 'blocked [key=token]: need deploy token' + sweep "$MATE" 3000 >/dev/null || fail "orphan decision priming sweep failed" + rm -f "$MATE/state/child.meta" + sweep "$MATE" 3001 >/dev/null || fail "orphan decision sweep failed" + grep -F 'blocked [key=mirror-5-child-token]: mirror: child=child decision token (opened at line 1) open when child retired' "$(parent_channel)" >/dev/null \ + || fail "orphan sweep did not immediately deliver the open decision" + [ ! -e "$MATE/state/parent-mirror/child.record" ] || fail "orphan decision record was removed too late" + pass "retired and orphaned outcomes bypass the grace threshold" +} + +# Lock discipline: a child whose record is held (a teardown or relaunch in +# progress) is skipped within the bounded wait rather than waited on, so the +# watcher's beacon is never held hostage, and a retire under a held meta lock +# takes no sweep lock, so a concurrent sweep can never deadlock a teardown. +test_busy_locks_are_skipped_not_waited() { + local holder started elapsed i lock rc + make_world locks; bind_secondmate local + write_child "$MATE" child + ledger "$MATE" child "done: PR $PR_URL checks green" + lock="$MATE/state/.meta-child.lock" + FM_STATE_OVERRIDE="$MATE/state" bash -c ' + . "$1/bin/fm-wake-lib.sh" + fm_lock_acquire_wait "$2" + : > "$3" + sleep 30 + ' _ "$ROOT" "$lock" "$WORLD/lock-held" & + holder=$! + i=0 + while [ "$i" -lt 50 ] && [ ! -e "$WORLD/lock-held" ]; do sleep 0.1; i=$((i + 1)); done + [ -e "$WORLD/lock-held" ] || fail "meta lock holder did not start" + started=$(date +%s) + FM_PARENT_MIRROR_LOCK_WAIT_SECS=1 sweep "$MATE" 1000 >/dev/null || fail "sweep against a busy child failed" + elapsed=$(( $(date +%s) - started )) + [ "$elapsed" -le 6 ] || fail "sweep waited on a busy child (${elapsed}s)" + [ ! -e "$(parent_channel)" ] || fail "a busy child was examined under someone else's lock" + rc=0 + FM_PARENT_MIRROR_LOCK_WAIT_SECS=1 sweep "$MATE" 1000 --child child >/dev/null 2>&1 || rc=$? + [ "$rc" -eq 5 ] || fail "targeted child contention did not return 5 (rc=$rc)" + reap "$holder" + sweep "$MATE" 1001 >/dev/null || fail "sweep after release failed" + [ "$(channel_count "done [key=mirror-5-child-l0]")" = 1 ] || fail "the child was not delivered once its lock was free" + + make_world retire-lock; bind_secondmate local + write_child "$MATE" scout scout + ledger "$MATE" scout 'done: report ready' + lock="$MATE/state/.parent-mirror.lock" + FM_STATE_OVERRIDE="$MATE/state" bash -c ' + . "$1/bin/fm-wake-lib.sh" + fm_lock_acquire_wait "$2" + : > "$3" + sleep 30 + ' _ "$ROOT" "$lock" "$WORLD/sweep-held" & + holder=$! + i=0 + while [ "$i" -lt 50 ] && [ ! -e "$WORLD/sweep-held" ]; do sleep 0.1; i=$((i + 1)); done + [ -e "$WORLD/sweep-held" ] || fail "sweep lock holder did not start" + started=$(date +%s) + FM_HOME="$MATE" FM_STATE_OVERRIDE="$MATE/state" FM_DATA_OVERRIDE="$MATE/data" bash -c ' + . "$1"; fm_parent_mirror_retire_locked scout + ' _ "$ROOT/bin/fm-parent-mirror-lib.sh" || fail "retire under a held sweep lock failed" + elapsed=$(( $(date +%s) - started )) + reap "$holder" + [ "$elapsed" -le 6 ] || fail "retire waited on the sweep lock (${elapsed}s)" + grep -F 'mirror: child=scout report ready' "$(parent_channel)" >/dev/null || fail "retire did not deliver while a sweep held its lock" + pass "busy locks are skipped within the bound and a retire never waits on a sweep" +} + +# A main home has no parent channel: the sweep is silent, writes nothing, and +# exits zero. +test_main_home_is_inert() { + local out + make_world main + write_child "$MAIN" child + ledger "$MAIN" child "done: PR $PR_URL checks green" + out=$(sweep "$MAIN" 1000) || fail "main-home sweep exited non-zero" + [ -z "$out" ] || fail "main-home sweep printed: $out" + [ ! -e "$MAIN/state/parent-mirror" ] || fail "main-home sweep created mirror state" + pass "a main home is inert" +} + +# A mate whose parent binding is unreadable is loud once per episode: one +# durable wake naming the binding, printed only when newly queued. +test_unreadable_binding_is_loud_once() { + local out rc=0 holder i started elapsed pids='' pid rows actions + make_world unbound + printf 'mate\n' > "$MATE/.fm-secondmate-home" + write_child "$MATE" child + ledger "$MATE" child "done: PR $PR_URL checks green" + out=$(sweep "$MATE" 1000 2>&1) || rc=$? + [ "$rc" -eq 3 ] || fail "unreadable binding did not return 3 (rc=$rc): $out" + case "$out" in *actionable:*".fm-secondmate-parent"*) ;; *) fail "the missing binding was not named: $out" ;; esac + [ "$(wake_count "$MATE" 'parent-mirror-diagnostic:channel')" = 1 ] || fail "the diagnostic was not queued once" + rc=0 + out=$(sweep "$MATE" 1001 2>&1) || rc=$? + [ "$rc" -eq 3 ] || fail "second sweep did not keep returning 3" + [ -z "$out" ] || fail "a still-queued diagnostic was printed again: $out" + [ "$(wake_count "$MATE" 'parent-mirror-diagnostic:channel')" = 1 ] || fail "the diagnostic was queued twice" + + make_world diagnostic-concurrent + printf 'mate\n' > "$MATE/.fm-secondmate-home" + write_child "$MATE" child + ledger "$MATE" child "done: PR $PR_URL checks green" + i=0 + while [ "$i" -lt 8 ]; do + sweep "$MATE" 1000 > "$WORLD/diag-$i.out" 2> "$WORLD/diag-$i.err" & + pid=$! + pids="$pids $pid" + i=$((i + 1)) + done + for pid in $pids; do wait "$pid" 2>/dev/null || true; done + rows=$(grep -c 'parent-mirror-diagnostic:channel' "$MATE/state/.wake-queue" 2>/dev/null || true) + actions=$(grep -h -c 'actionable: parent channel unavailable' "$WORLD"/diag-*.out | awk '{ total += $1 } END { print total + 0 }') + [ "$rows" -eq 1 ] || fail "concurrent diagnostics queued $rows rows" + [ "$actions" -eq 1 ] || fail "concurrent diagnostics emitted $actions actionable wakes" + + make_world diagnostic-lock + pids='' + printf 'mate\n' > "$MATE/.fm-secondmate-home" + write_child "$MATE" child + ledger "$MATE" child "done: PR $PR_URL checks green" + FM_STATE_OVERRIDE="$MATE/state" bash -c ' + . "$1/bin/fm-wake-lib.sh" + fm_lock_acquire_wait "$FM_WAKE_QUEUE_LOCK" + : > "$2" + sleep 30 + ' _ "$ROOT" "$WORLD/wake-held" & + holder=$! + i=0 + while [ "$i" -lt 50 ] && [ ! -e "$WORLD/wake-held" ]; do sleep 0.1; i=$((i + 1)); done + started=$(date +%s) + rc=0 + out=$(FM_PARENT_MIRROR_LOCK_WAIT_SECS=1 sweep "$MATE" 1000 2>&1) || rc=$? + elapsed=$(( $(date +%s) - started )) + reap "$holder" + [ "$rc" -eq 3 ] || fail "contended diagnostic changed the channel error (rc=$rc)" + [ "$elapsed" -le 4 ] || fail "diagnostic blocked on the wake queue (${elapsed}s)" + case "$out" in *"parent channel unavailable"*) ;; *) fail "contended diagnostic did not degrade to stderr" ;; esac + [ ! -e "$MATE/state/.wake-queue" ] \ + || fail "contended diagnostic wrote through a held queue" + pass "parent diagnostics are once-only and bounded by the wake queue" +} + +# The real watcher poll delivers a terminal child within one poll, so the +# parent's own watcher can wake on it, without the mate itself being woken. +test_watcher_poll_delivers_terminal_child() { + local pid i + make_world watcher; bind_secondmate local + write_child "$MATE" child + ledger "$MATE" child "done: PR $PR_URL checks green" + printf 'do not truncate\n' > "$WORLD/diagnostic-target" + ln -s "$WORLD/diagnostic-target" "$MATE/state/.parent-mirror.stderr" + prime_seen "$MATE/state" "$MATE/state/child.status" + PATH="$WORLD/fakebin:$PATH" FM_ROOT_OVERRIDE="$WORLD/root" FM_HOME="$MATE" FM_STATE_OVERRIDE="$MATE/state" \ + FM_DATA_OVERRIDE="$MATE/data" FM_CONFIG_OVERRIDE="$MATE/config" FM_FORGE_LOG="$WORLD/forge.log" \ + FM_POLL=1 FM_SIGNAL_GRACE=1 FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 FM_INACTIVE_RECONCILE_SECS=1800 \ + "$WATCH" > "$WORLD/watch.out" 2>&1 & + pid=$! + i=0 + while [ "$i" -lt 60 ]; do + grep -F 'mirror: child=child' "$(parent_channel)" >/dev/null 2>&1 && break + kill -0 "$pid" 2>/dev/null || break + sleep 0.25 + i=$((i + 1)) + done + reap "$pid" + [ "$(cat "$WORLD/diagnostic-target")" = 'do not truncate' ] \ + || fail "the watcher followed the predictable diagnostic symlink" + grep -F "done [key=mirror-5-child-l0]: mirror: child=child PR $PR_URL checks green" "$(parent_channel)" >/dev/null \ + || fail "the watcher poll did not deliver the terminal child: $(cat "$WORLD/watch.out")" + ! grep -F 'check: parent-mirror' "$WORLD/watch.out" >/dev/null || fail "a clean delivery woke the mate: $(cat "$WORLD/watch.out")" + pass "the watcher poll delivers a terminal child to the parent channel" +} + +test_done_line_delivered_once +test_incremental_sweep_reads_only_new_ledger_bytes +test_scout_report_pointer +test_scratch_paths_refuse_symlink_redirection +test_partial_line_waits_for_newline +test_orphan_retains_unterminated_tail +test_partial_thresholded_outcomes_do_not_age +test_open_decision_thresholded_then_closed +test_open_decision_updates_retain_occurrence +test_ledger_reset_closes_decisions_and_clears_failures +test_failed_decision_close_retries_without_child_change +test_reopened_decision_is_raised_again +test_decision_handled_inside_threshold_is_silent +test_failed_line_thresholded_and_superseded +test_untracked_decision_line_is_delivered_as_done +test_injective_keys_and_concurrent_append +test_parent_channel_repairs_unterminated_tail +test_remote_route_writes_parent_replies +test_pr_check_registration_delivers_now +test_retire_delivers_then_orphan_retries +test_retired_and_orphaned_outcomes_deliver_immediately +test_busy_locks_are_skipped_not_waited +test_main_home_is_inert +test_unreadable_binding_is_loud_once +test_watcher_poll_delivers_terminal_child + +echo "all parent mirror tests passed" diff --git a/tests/fm-pr-merge.test.sh b/tests/fm-pr-merge.test.sh index f75bc58c07b..8392c2a33e9 100755 --- a/tests/fm-pr-merge.test.sh +++ b/tests/fm-pr-merge.test.sh @@ -2067,8 +2067,14 @@ test_secondmate_without_parent_binding_is_loud() { expect_code 0 "$rc" "unbound-secondmate: the merge itself landed and must not be reported as failed" assert_grep 'could not report it upward' "$case_dir/stderr" \ "unbound-secondmate: a merge that could not be reported upward said nothing about it" - assert_absent "$case_dir/state/.wake-queue" \ - "unbound-secondmate: a secondmate home fell back to the main-home record" + # The merge outcome must not fall back to the main-home record. The + # registration's parent-channel sweep may queue its own channel diagnostic + # here, which names the missing binding rather than the merge. + if grep -F "$url" "$case_dir/state/.wake-queue" >/dev/null 2>&1; then + fail "unbound-secondmate: a secondmate home fell back to the main-home record" + fi + grep -F 'parent-mirror-diagnostic:channel' "$case_dir/state/.wake-queue" >/dev/null 2>&1 \ + || fail "unbound-secondmate: the broken parent channel was not queued as a durable diagnostic" pass "a secondmate home that cannot report upward says so instead of merging in silence" } diff --git a/tests/fm-public-followup.test.sh b/tests/fm-public-followup.test.sh index c0d5a7779da..bcc72c82a28 100755 --- a/tests/fm-public-followup.test.sh +++ b/tests/fm-public-followup.test.sh @@ -1049,6 +1049,8 @@ test_relay_disabled_parent_allows_marked_child_teardown() { child=$(make_home teardown-disabled-child relay-off) fm_git_init_commit "$child/projects/worktree" printf '%s\n' disabled-mate > "$child/.fm-secondmate-home" + printf 'schema=fm-secondmate-parent.v1\nroute=local\nparent_home=%s\n' "$parent" \ + > "$child/.fm-secondmate-parent" printf -- '- disabled-mate - synthetic (home: %s; scope: synthetic; projects: ; added 2026-07-30)\n' \ "$child" > "$parent/data/secondmates.md" fm_write_meta "$parent/state/disabled-mate.meta" "kind=secondmate" "home=$child" diff --git a/tests/fm-teardown.test.sh b/tests/fm-teardown.test.sh index fe0131ce479..ac45ea08449 100755 --- a/tests/fm-teardown.test.sh +++ b/tests/fm-teardown.test.sh @@ -544,6 +544,75 @@ seed_backlog_in_flight() { tasks-axi start task-x1 --file "$case_dir/data/backlog.md" >/dev/null } +test_parent_mirror_requires_durable_orphan_before_record_removal() { + local case_dir rc target mirror + case_dir=$(make_case mirror-orphan-refuse) + write_meta "$case_dir" local-only ship + seed_backlog_in_flight "$case_dir" + printf 'mate\n' > "$case_dir/.fm-secondmate-home" + printf 'invalid binding\n' > "$case_dir/.fm-secondmate-parent" + printf 'done: outcome awaiting parent\n' > "$case_dir/state/task-x1.status" + target="$case_dir/unsafe-parent-mirror" + mkdir -p "$target" + ln -s "$target" "$case_dir/state/parent-mirror" + rc=0 + FM_HOME="$case_dir" run_teardown "$case_dir" --force > "$case_dir/stdout" 2> "$case_dir/stderr" || rc=$? + [ "$rc" -ne 0 ] || fail "teardown removed a child without a durable orphan record" + [ -f "$case_dir/state/task-x1.meta" ] || fail "refused teardown removed the discoverable child record" + assert_grep "no durable orphan record exists" "$case_dir/stderr" \ + "refused teardown did not explain the missing durable orphan" + + case_dir=$(make_case mirror-orphan-confirmed) + write_meta "$case_dir" local-only ship + seed_backlog_in_flight "$case_dir" + printf 'mate\n' > "$case_dir/.fm-secondmate-home" + printf 'invalid binding\n' > "$case_dir/.fm-secondmate-parent" + printf 'done: outcome awaiting parent\n' > "$case_dir/state/task-x1.status" + printf 'task-x1\tfixture-ident\t0\t0\n' > "$case_dir/state/.status-presentation-cursor" + rc=0 + FM_HOME="$case_dir" run_teardown "$case_dir" --force > "$case_dir/stdout" 2> "$case_dir/stderr" || rc=$? + expect_code 0 "$rc" "teardown should proceed after confirming a durable orphan" + [ ! -e "$case_dir/state/task-x1.meta" ] || fail "verified orphan did not permit child-record removal" + [ -f "$case_dir/state/parent-mirror/task-x1.record" ] \ + || fail "teardown did not preserve its verified orphan record" + [ -f "$case_dir/state/task-x1.status" ] || fail "teardown removed an orphan's undelivered ledger" + grep -q '^task-x1' "$case_dir/state/.status-presentation-cursor" \ + || fail "teardown retired an orphan's presentation row before delivery" + printf 'schema=fm-secondmate-parent.v1\nroute=remote\n' > "$case_dir/.fm-secondmate-parent" + mirror="$ROOT/bin/fm-parent-mirror.sh" + FM_ROOT_OVERRIDE="$ROOT" FM_HOME="$case_dir" FM_STATE_OVERRIDE="$case_dir/state" \ + FM_DATA_OVERRIDE="$case_dir/data" "$mirror" sweep >/dev/null \ + || fail "restored parent channel did not sweep the orphan" + assert_grep 'mirror: child=task-x1 outcome awaiting parent' "$case_dir/state/parent-replies.status" \ + "orphan sweep did not deliver the retained ledger" + [ ! -e "$case_dir/state/task-x1.status" ] || fail "delivered orphan retained its ledger" + [ ! -e "$case_dir/state/parent-mirror/task-x1.record" ] || fail "delivered orphan retained its mirror record" + if grep -q '^task-x1' "$case_dir/state/.status-presentation-cursor"; then + fail "delivered orphan retained its presentation row" + fi + + case_dir=$(make_case mirror-orphan-tail) + write_meta "$case_dir" local-only ship + seed_backlog_in_flight "$case_dir" + printf 'mate\n' > "$case_dir/.fm-secondmate-home" + printf 'schema=fm-secondmate-parent.v1\nroute=remote\n' > "$case_dir/.fm-secondmate-parent" + printf 'done: tail awaiting completion' > "$case_dir/state/task-x1.status" + rc=0 + FM_HOME="$case_dir" run_teardown "$case_dir" --force > "$case_dir/stdout" 2> "$case_dir/stderr" || rc=$? + expect_code 0 "$rc" "teardown should retain an orphan with an incomplete tail" + [ -f "$case_dir/state/task-x1.status" ] || fail "teardown removed an incomplete orphan ledger" + [ -f "$case_dir/state/parent-mirror/task-x1.record" ] || fail "teardown lost its incomplete orphan record" + printf '\n' >> "$case_dir/state/task-x1.status" + FM_ROOT_OVERRIDE="$ROOT" FM_HOME="$case_dir" FM_STATE_OVERRIDE="$case_dir/state" \ + FM_DATA_OVERRIDE="$case_dir/data" "$mirror" sweep >/dev/null \ + || fail "completed orphan tail did not sweep" + assert_grep 'mirror: child=task-x1 tail awaiting completion' "$case_dir/state/parent-replies.status" \ + "completed orphan tail was not delivered" + [ ! -e "$case_dir/state/task-x1.status" ] || fail "delivered tail retained its ledger" + [ ! -e "$case_dir/state/parent-mirror/task-x1.record" ] || fail "delivered tail retained its mirror record" + pass "teardown preserves orphan ledgers through final delivery" +} + backlog_row_state() { local case_dir=$1 tasks-axi show task-x1 --file "$case_dir/data/backlog.md" 2>/dev/null | @@ -2605,6 +2674,7 @@ EOF pass "the run abort and the leaked-process reap both complete before the destructive worktree return" } +test_parent_mirror_requires_durable_orphan_before_record_removal test_local_only_fork_remote_allows test_teardown_closes_the_backlog_item_itself test_teardown_manual_backend_leaves_the_backlog_to_the_operator diff --git a/tests/fm-wake-queue.test.sh b/tests/fm-wake-queue.test.sh index c1f86a59c0d..17884238ce0 100755 --- a/tests/fm-wake-queue.test.sh +++ b/tests/fm-wake-queue.test.sh @@ -18,6 +18,86 @@ GRANT="$ROOT/bin/fm-wake-grant.sh" TMP_ROOT=$(fm_test_tmproot fm-wake-tests) +test_bounded_wake_queue_operations() { + local dir state holder i started elapsed rc keys pids='' pid failures=0 count + dir=$(make_case bounded-queue) + state="$dir/state" + FM_STATE_OVERRIDE="$state" bash -c ' + . "$1" + fm_lock_acquire_wait "$FM_WAKE_QUEUE_LOCK" + : > "$2" + sleep 30 + ' _ "$ROOT/bin/fm-wake-lib.sh" "$dir/held" & + holder=$! + i=0 + while [ "$i" -lt 50 ] && [ ! -e "$dir/held" ]; do sleep 0.1; i=$((i + 1)); done + [ -e "$dir/held" ] || fail "wake queue holder did not start" + started=$(date +%s) + rc=0 + FM_STATE_OVERRIDE="$state" bash -c '. "$1"; fm_wake_queued_keys_bounded check 1' \ + _ "$ROOT/bin/fm-wake-lib.sh" >/dev/null 2>&1 || rc=$? + [ "$rc" -ne 0 ] || fail "bounded queued-keys entered a held queue" + rc=0 + FM_STATE_OVERRIDE="$state" bash -c '. "$1"; fm_wake_append_bounded check key payload 1' \ + _ "$ROOT/bin/fm-wake-lib.sh" >/dev/null 2>&1 || rc=$? + elapsed=$(( $(date +%s) - started )) + [ "$rc" -ne 0 ] || fail "bounded append entered a held queue" + [ "$elapsed" -le 6 ] || fail "bounded queue operations exceeded their deadlines" + kill "$holder" 2>/dev/null || true + wait "$holder" 2>/dev/null || true + FM_STATE_OVERRIDE="$state" bash -c '. "$1"; fm_wake_append_bounded check key payload 2' \ + _ "$ROOT/bin/fm-wake-lib.sh" || fail "bounded append failed after release" + keys=$(FM_STATE_OVERRIDE="$state" bash -c '. "$1"; fm_wake_queued_keys_bounded check 2' \ + _ "$ROOT/bin/fm-wake-lib.sh") || fail "bounded queued-keys failed after release" + [ "$keys" = key ] || fail "bounded queue read lost the appended key" + i=0 + while [ "$i" -lt 12 ]; do + FM_STATE_OVERRIDE="$state" bash -c ' + . "$1"; fm_wake_append_if_key_absent_bounded check atomic-key "one diagnostic" 3 + ' _ "$ROOT/bin/fm-wake-lib.sh" >/dev/null 2>&1 & + pid=$! + pids="$pids $pid" + i=$((i + 1)) + done + for pid in $pids; do + wait "$pid" || { [ "$?" -eq 3 ] || failures=$((failures + 1)); } + done + [ "$failures" -eq 0 ] || fail "atomic append-if-absent had an unexpected failure" + count=$(awk -F '\t' '$3 == "check" && $4 == "atomic-key" { count++ } END { print count + 0 }' "$state/.wake-queue") + [ "$count" -eq 1 ] || fail "atomic append-if-absent queued $count duplicate rows" + + FM_STATE_OVERRIDE="$state" bash -c ' + . "$1" + fm_lock_acquire_wait "$STATE/.watcher-down.lock" + : > "$2" + sleep 30 + ' _ "$ROOT/bin/fm-wake-lib.sh" "$dir/recovery-held" & + holder=$! + i=0 + while [ "$i" -lt 50 ] && [ ! -e "$dir/recovery-held" ]; do sleep 0.1; i=$((i + 1)); done + [ -e "$dir/recovery-held" ] || fail "recovery marker holder did not start" + started=$(date +%s) + rc=0 + FM_STATE_OVERRIDE="$state" bash -c ' + . "$1"; fm_wake_append_if_key_absent_bounded check recovery-key payload 1 + ' _ "$ROOT/bin/fm-wake-lib.sh" >/dev/null 2>&1 || rc=$? + elapsed=$(( $(date +%s) - started )) + [ "$rc" -ne 0 ] || fail "bounded append entered a held recovery marker" + [ "$elapsed" -le 3 ] || fail "bounded append wedged on the recovery marker" + FM_STATE_OVERRIDE="$state" bash -c ' + . "$1" + fm_lock_acquire_wait_bounded "$FM_WAKE_QUEUE_LOCK" 1 + fm_lock_release "$FM_WAKE_QUEUE_LOCK" + ' _ "$ROOT/bin/fm-wake-lib.sh" || fail "failed bounded append retained the queue lock" + kill "$holder" 2>/dev/null || true + wait "$holder" 2>/dev/null || true + FM_STATE_OVERRIDE="$state" bash -c ' + . "$1"; fm_wake_append_if_key_absent_bounded check recovery-key payload 2 + ' _ "$ROOT/bin/fm-wake-lib.sh" >/dev/null \ + || fail "bounded append did not recover after the marker lock released" + pass "bounded wake queue operations bound both queue and recovery locks" +} + test_concurrent_append_and_drain() { local dir state out1 out2 pids i pid count unique malformed sequence generation dir=$(make_case concurrent) @@ -1438,6 +1518,7 @@ test_historical_annotation_skips_announced_status() { } test_self_held_lock_reclaims_instead_of_deadlocking +test_bounded_wake_queue_operations test_bounded_lock_handoff_after_contention test_live_presentation_holder_is_deadlined_without_weakening_ack test_malformed_presentation_lock_reports_acquire_failure