diff --git a/.gitignore b/.gitignore index dd0a8f1df1..d8c31963f9 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ data/ scratchpad* .no-mistakes/ .lavish/ +.squish/ .fm-secondmate-home .fm-secondmate-parent .DS_Store diff --git a/AGENTS.md b/AGENTS.md index d2ad7a7438..dac38575e9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -131,6 +131,7 @@ state/ runtime records and signals; gitignored ..open-decisions-cursor per-task byte cursor and folded open-decision set bounding the OPEN DECISIONS scan's cost to new status-log appends; written only by fm-classify-lib.sh's status_open_decisions_incremental, removed by teardown, safe to delete (forces one full re-fold) .status-presentation-cursor .status-presentation-lock fleet-wide per-task status identity plus independent annotation and outcome-backstop byte offsets, with a serialization lock preventing already-presented lines from replaying while preserving delayed signal annotations; owned by fm-classify-lib.sh, with each task's row retired by teardown .afk durable away-mode flag; present = sub-supervisor may inject escalations (set by /afk, cleared on user return) + .resgate-cap-work .resgate-cap-home durable manual resource-cap override markers forcing the 50% weekly-schedule cap immediately on the work or home PC regardless of the clock window; set and cleared by firstmate on the section 8 "Kappung" trigger (docs/configuration.md "Fleet resource governance") .watch.lock .wake-queue.lock watcher singleton and queue serialization locks .claude-autoarm.lock .claude-autoarm-epoch .claude-autoarm-failure-notified .claude-autoarm-failure-alarmed .turnend-claude-blocks .turnend-claude-blocks.lock Claude Stop auto-arm single-flight, epoch, failure-episode, attended-alarm, guard-budget, and budget-lock records; never touch .cursor-park-owner .cursor-park-owner.lock .turnend-cursor-blocks Cursor stop-hook owner record, publication and commit lock, and bounded repair-nag budget; never touch @@ -443,6 +444,12 @@ The skill owns the daemon procedure; these safety facts remain inline: - Away mode never expands approval authority for merges, ask-user findings, destructive actions, irreversible actions, or security-sensitive choices. - Bias ambiguous input toward exit because a present captain takes precedence. +### Resource-cap trigger + +Arm `state/.resgate-cap-` the moment the captain says "Kappung" in chat, matched case-insensitively: bare "Kappung" arms both roles, `Kappung ` arms only that one. +"Kappung auf" releases the same way, bare for both markers and `Kappung auf ` for that one alone. +docs/configuration.md "Fleet resource governance" owns the marker paths, the release-form-first and longest-hostname-first matching precedence, and the rest of the mechanism. + ### Stuck-worker trigger Load `stuck-crewmate-recovery` after a stale wake, looping or confused pane, answered-by-brief question, unresponsive worker, or failed steer. diff --git a/bin/fm-resgate-lib.sh b/bin/fm-resgate-lib.sh new file mode 100644 index 0000000000..a8886d0722 --- /dev/null +++ b/bin/fm-resgate-lib.sh @@ -0,0 +1,557 @@ +# shellcheck shell=bash +# fm-resgate-lib.sh - weekly clock-window resource governance for the captain's +# two Windows hosts, plus GPU exclusivity between Qwen and the JARVIS voice +# worker on the home PC. +# +# Sourced, never executed. bin/fm-resgate.sh is the CLI. +# +# Two fixed roles, not a configurable host list: `work` is the Arbeits-PC +# (default SSH alias Valentino-Arbeit) and `home` is the Heim-PC, RTX 4080 +# Super (default SSH alias Valentino). FM_RESGATE_WORK_SSH / FM_RESGATE_HOME_SSH +# override the alias for tests or a renamed host. +# +# Captain's policy (verbatim intent, data/captain.md 02.09.2026): +# Arbeits-PC: Mo-Fr 19:30-10:00 and the whole weekend fully free for the +# fleet; Mo-Fr 10:00-19:30 at most 50% of resources. +# Heim-PC: Mo-Fr 04:00-19:00 free; Mo-Fr 19:00-04:00 and the whole weekend +# at most 50%. +# Both windows are same-calendar-day spans, so the schedule is expressed as the +# CAPPED window for work (weekday 10:00-19:30) and the FREE window for home +# (weekday 04:00-19:00), with the opposite state as the default. That default +# is what makes the Friday-evening-through-Monday-morning free span on work, +# and the symmetric always-capped weekend on home, fall out of the two small +# per-weekday windows below without separate weekend-boundary code: nothing +# outside a weekday's stated window is ever in either list, weekday or not. +# +# Authoritative clock: this library never asks a remote host for its own +# clock. The schedule decision is computed once, here, against THIS host's +# wall clock forced into Europe/Berlin regardless of the host's configured +# default zone, and only the resulting capped/uncapped state and percentage +# travel to whichever host is being gated. A remote host's local clock can be +# wrong or drifted and must never be able to loosen or defeat the gate. +# FM_RESGATE_NOW_OVERRIDE=" " replaces the `date` +# read for tests. +# +# Fail-closed discipline: every measurement this library cannot read - the +# clock, an SSH probe, a port check, a GPU query - yields the MOST restrictive +# answer, never a guess and never "permissive by default". For the schedule +# gate that is capacity_pct=0 ("blocked", stricter than the ordinary 50% cap). +# For the GPU exclusivity check that is "not available" for either workload. +# +# Manual override: state/.resgate-cap- is a plain presence-based marker, +# written the same way state/.afk is (mktemp + mv, so a reader never observes +# a half-written file). Its presence forces capped state immediately, +# independent of which window the clock lands in - but strictly BELOW the +# fail-closed rule above: an unreadable clock stays blocked at 0% even with the +# marker armed, because a control whose whole purpose is to restrict must never +# be able to hand out capacity no measurement supports. docs/configuration.md +# "Fleet resource governance" owns the exact marker paths, the "Kappung" / +# "Kappung auf" trigger words, and which host each form arms. +# +# GPU exclusivity (home PC only): JARVIS voice is detected by its gateway PORT +# (currently 7414, data/learnings.md 23.08.2026), never by process name - +# process-name detection has broken this fleet's integration before. That port +# reading is AUTHORITATIVE and is decided first: a listening gateway means the +# voice worker owns the card, full stop, and the Qwen signals below are never +# even consulted. The aggregate memory reading cannot say WHOSE memory it is, +# so an idle-but-resident Qwen service plus the voice worker's own VRAM would +# otherwise read as contention and refuse the very workload that legitimately +# holds the card - blocking JARVIS voice against itself. Deciding on the port +# first is what makes that misreading impossible rather than merely unlikely. +# +# Qwen detection does NOT use `nvidia-smi --query-compute-apps`, even though +# that looks like the stabler per-process signal on paper. Live-verified +# against the real home host: on Windows/WDDM that query lists every process +# holding an ordinary desktop GPU context - dwm.exe, explorer.exe, every open +# browser - not just genuine compute workloads, and reports `[N/A]` for their +# per-process memory, so there is no field left to filter the noise out by. +# Treating any non-empty row as "Qwen is active" would read as busy any time +# the desktop itself is on. Qwen is instead detected by a named-process check +# (`Get-Process -Name ollama`, the exact identity live-confirmed for today's +# Qwen work; a fresh probe against another engine's name is a config change, +# not a design change) corroborated by AGGREGATE GPU memory +# (`nvidia-smi --query-gpu=memory.used`) clearing FM_RESGATE_GPU_BUSY_MB +# (default 4096 MiB): live-verified idle-ish desktop baseline was ~9 GiB used +# with a loaded model and process present, comfortably clear of ordinary +# desktop compositing. The process check alone would treat an installed-but- +# idle service as "active"; the memory check alone cannot name what is using +# the card; requiring both is the more stable combination the process name +# check alone is not. All three readings (port, process, aggregate memory) +# come from one bounded SSH round trip so a slow host cannot multiply the +# timeout. +# +# Overrides (tests and session pins): +# FM_RESGATE_WORK_SSH, FM_RESGATE_HOME_SSH +# FM_RESGATE_NOW_OVERRIDE +# FM_RESGATE_VOICE_PORT (default 7414) +# FM_RESGATE_GPU_BUSY_MB (default 4096) +# FM_RESGATE_SSH_TIMEOUT (seconds, default 5; must be a positive integer) +# FM_RESGATE_SKIP_REMOTE=1 (never probe, always fail closed - tests only) + +_FM_RESGATE_LIB_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=bin/fm-timeout-lib.sh +. "$_FM_RESGATE_LIB_DIR/fm-timeout-lib.sh" + +FM_RESGATE_WORK_SSH_DEFAULT=Valentino-Arbeit +FM_RESGATE_HOME_SSH_DEFAULT=Valentino +FM_RESGATE_VOICE_PORT_DEFAULT=7414 +FM_RESGATE_GPU_BUSY_MB_DEFAULT=4096 +FM_RESGATE_SSH_TIMEOUT_DEFAULT=5 +FM_RESGATE_GPU_PROCESS_NAME=ollama +FM_RESGATE_WORK_CAP_START_MIN=$((10 * 60)) # 10:00 +FM_RESGATE_WORK_CAP_END_MIN=$((19 * 60 + 30)) # 19:30 +FM_RESGATE_HOME_FREE_START_MIN=$((4 * 60)) # 04:00 +FM_RESGATE_HOME_FREE_END_MIN=$((19 * 60)) # 19:00 +FM_RESGATE_CAPPED_PCT=50 +FM_RESGATE_UNCAPPED_PCT=100 +FM_RESGATE_BLOCKED_PCT=0 +FM_RESGATE_BERLIN_STANDARD='CET +0100' +FM_RESGATE_BERLIN_SUMMER='CEST +0200' + +fm_resgate_role_ok() { + case "${1:-}" in work | home) return 0 ;; esac + return 1 +} + +fm_resgate_ssh_alias() { # + case "$1" in + work) printf '%s\n' "${FM_RESGATE_WORK_SSH:-$FM_RESGATE_WORK_SSH_DEFAULT}" ;; + home) printf '%s\n' "${FM_RESGATE_HOME_SSH:-$FM_RESGATE_HOME_SSH_DEFAULT}" ;; + esac +} + +fm_resgate_is_uint() { + case "${1:-}" in '' | *[!0-9]*) return 1 ;; esac + return 0 +} + +# A plausible TCP port. Every tunable that reaches a measurement is validated +# before it is used, so a mistyped session pin cannot silently turn into a +# permissive reading (an invalid threshold that makes the -ge comparison error +# out, or an invalid port that probes something other than the gateway). +fm_resgate_is_port() { + fm_resgate_is_uint "${1:-}" || return 1 + [ "${#1}" -le 5 ] || return 1 + [ "$1" -ge 1 ] && [ "$1" -le 65535 ] +} + +# Sets FM_RESGATE_NOW_DOW (1=Monday..7=Sunday) and FM_RESGATE_NOW_MOD (minutes +# since local midnight, 0-1439), read from ONE `date` call so a rollover +# between day-of-week and time-of-day cannot be observed straddling midnight. +# Returns 1 on any unreadable or malformed clock; callers must fail closed on +# that, never fall back to a default time. +# +# The same read also carries the resolved zone abbreviation and UTC offset, +# because a non-zero exit is NOT the failure mode that matters here: `date` +# never fails on a zone it cannot resolve, it silently falls back to UTC and +# exits 0. On a host without tzdata the gate would then read Berlin 10:30 CEST +# as 08:30, land outside the work PC's 10:00-19:30 window, and hand the fleet +# 100% of the captain's machine in the middle of his working hours - a wrong +# permissive answer from an effectively unreadable clock. Requiring the +# abbreviation and offset to be a matching Europe/Berlin pair proves the zone +# database entry actually loaded; anything else is unreadable and fails closed. +fm_resgate_now_fields() { + local raw dow hh mm + FM_RESGATE_NOW_DOW= + FM_RESGATE_NOW_MOD= + if [ -n "${FM_RESGATE_NOW_OVERRIDE:-}" ]; then + raw=$FM_RESGATE_NOW_OVERRIDE + else + raw=$(TZ=Europe/Berlin date +'%u %H %M %Z %z' 2>/dev/null) || return 1 + # shellcheck disable=SC2086 + set -- $raw + [ "$#" -eq 5 ] || return 1 + case "$4 $5" in + "$FM_RESGATE_BERLIN_STANDARD" | "$FM_RESGATE_BERLIN_SUMMER") ;; + *) return 1 ;; + esac + raw="$1 $2 $3" + fi + # shellcheck disable=SC2086 + set -- $raw + [ "$#" -eq 3 ] || return 1 + dow=$1 hh=$2 mm=$3 + fm_resgate_is_uint "$dow" || return 1 + fm_resgate_is_uint "$hh" || return 1 + fm_resgate_is_uint "$mm" || return 1 + [ "$dow" -ge 1 ] && [ "$dow" -le 7 ] || return 1 + [ "$hh" -le 23 ] || return 1 + [ "$mm" -le 59 ] || return 1 + FM_RESGATE_NOW_DOW=$dow + FM_RESGATE_NOW_MOD=$((10#$hh * 60 + 10#$mm)) + return 0 +} + +# Schedule state for at the current moment, ignoring any manual +# override. Sets FM_RESGATE_SCHEDULE_STATE to one of: +# uncapped - full resources +# capped - the 50% window applies +# blocked - the authoritative clock could not be read; fail closed to 0%, +# stricter than an ordinary capped window, because a schedule +# decision cannot be made at all. +# and FM_RESGATE_SCHEDULE_REASON to a short human-readable reason. +fm_resgate_schedule_state() { # + local role=$1 dow mod weekday + fm_resgate_role_ok "$role" || { + FM_RESGATE_SCHEDULE_STATE=blocked + FM_RESGATE_SCHEDULE_REASON="unknown role: $role" + return 1 + } + if ! fm_resgate_now_fields; then + FM_RESGATE_SCHEDULE_STATE=blocked + FM_RESGATE_SCHEDULE_REASON='authoritative clock unreadable; refusing to guess the schedule window' + return 0 + fi + dow=$FM_RESGATE_NOW_DOW + mod=$FM_RESGATE_NOW_MOD + weekday=0 + [ "$dow" -ge 1 ] && [ "$dow" -le 5 ] && weekday=1 + case "$role" in + work) + if [ "$weekday" -eq 1 ] \ + && [ "$mod" -ge "$FM_RESGATE_WORK_CAP_START_MIN" ] \ + && [ "$mod" -lt "$FM_RESGATE_WORK_CAP_END_MIN" ]; then + FM_RESGATE_SCHEDULE_STATE=capped + FM_RESGATE_SCHEDULE_REASON='Mo-Fr 10:00-19:30 on the work PC (captain working hours)' + else + FM_RESGATE_SCHEDULE_STATE=uncapped + FM_RESGATE_SCHEDULE_REASON='outside Mo-Fr 10:00-19:30 on the work PC (evening, night, or weekend)' + fi + ;; + home) + if [ "$weekday" -eq 1 ] \ + && [ "$mod" -ge "$FM_RESGATE_HOME_FREE_START_MIN" ] \ + && [ "$mod" -lt "$FM_RESGATE_HOME_FREE_END_MIN" ]; then + FM_RESGATE_SCHEDULE_STATE=uncapped + FM_RESGATE_SCHEDULE_REASON='Mo-Fr 04:00-19:00 on the home PC' + else + FM_RESGATE_SCHEDULE_STATE=capped + FM_RESGATE_SCHEDULE_REASON='outside Mo-Fr 04:00-19:00 on the home PC (evening, night, or weekend)' + fi + ;; + esac + return 0 +} + +# for the override marker files; overridable for tests exactly +# like every other script in this repo. +fm_resgate_override_path() { # + printf '%s/.resgate-cap-%s\n' "$1" "$2" +} + +fm_resgate_override_active() { # + local path + path=$(fm_resgate_override_path "$1" "$2") + [ -e "$path" ] +} + +# Arm the manual override for . Atomic write (mktemp + mv), matching +# state/.afk, so a concurrent reader never observes a half-written marker. +fm_resgate_override_set() { # [note] + local state=$1 role=$2 note=${3:-} path pending + fm_resgate_role_ok "$role" || return 1 + [ -d "$state" ] || mkdir -p "$state" || return 1 + path=$(fm_resgate_override_path "$state" "$role") + pending=$(mktemp "$path.pending.XXXXXX") || return 1 + { + printf 'armed_at=%s\n' "$(TZ=Europe/Berlin date +%Y-%m-%dT%H:%M:%S%z 2>/dev/null || printf unknown)" + printf 'note=%s\n' "${note:-Kappung}" + } > "$pending" && mv -f "$pending" "$path" && return 0 + rm -f "$pending" 2>/dev/null || true + return 1 +} + +# Release the manual override for . Returns 1 when the marker is still +# there afterwards (an unwritable state directory, a stale mount): reporting a +# clear that did not happen would leave the role pinned at the 50% cap while +# everyone believes "Kappung auf" took effect. +fm_resgate_override_clear() { # + local path + fm_resgate_role_ok "$2" || return 1 + path=$(fm_resgate_override_path "$1" "$2") + rm -f "$path" 2>/dev/null + [ ! -e "$path" ] +} + +# Effective state for : an active override forces "capped" immediately, +# regardless of which window a READABLE clock lands in. Sets FM_RESGATE_STATE +# and FM_RESGATE_REASON. +# +# The schedule is computed FIRST and "blocked" wins outright, because the +# override and the fail-closed rule collide when the authoritative clock is +# unreadable: the override loosens 0% to 50%, and the whole point of the +# override is to restrict, never to grant capacity a measurement could not +# justify. Fail-closed takes precedence, so arming the cap can only ever +# tighten the gate. +# shellcheck disable=SC2034 +fm_resgate_effective_state() { # + local state=$1 role=$2 + fm_resgate_schedule_state "$role" + if [ "$FM_RESGATE_SCHEDULE_STATE" = blocked ]; then + FM_RESGATE_STATE=$FM_RESGATE_SCHEDULE_STATE + FM_RESGATE_REASON=$FM_RESGATE_SCHEDULE_REASON + return 0 + fi + if fm_resgate_override_active "$state" "$role"; then + FM_RESGATE_STATE=capped + FM_RESGATE_REASON='manual override armed (Kappung); forced capped regardless of the clock window' + return 0 + fi + FM_RESGATE_STATE=$FM_RESGATE_SCHEDULE_STATE + FM_RESGATE_REASON=$FM_RESGATE_SCHEDULE_REASON + return 0 +} + +# Percentage of resources may use right now: 100 (uncapped), 50 +# (capped), or 0 (blocked - the clock or role was unreadable). Sets +# FM_RESGATE_PCT and FM_RESGATE_REASON. +# shellcheck disable=SC2034 +fm_resgate_capacity_pct() { # + fm_resgate_effective_state "$1" "$2" + case "$FM_RESGATE_STATE" in + uncapped) FM_RESGATE_PCT=$FM_RESGATE_UNCAPPED_PCT ;; + capped) FM_RESGATE_PCT=$FM_RESGATE_CAPPED_PCT ;; + *) FM_RESGATE_PCT=$FM_RESGATE_BLOCKED_PCT ;; + esac + return 0 +} + +# Apply a percentage to a raw resource/slot count. Integer floor division; +# fails closed to 0 on a non-numeric raw count or an out-of-range percentage +# rather than passing either through unchecked. +fm_resgate_apply_pct() { # + local raw=$1 pct=$2 + fm_resgate_is_uint "$raw" || { printf '0\n'; return 0; } + fm_resgate_is_uint "$pct" || { printf '0\n'; return 0; } + [ "$pct" -le 100 ] || { printf '0\n'; return 0; } + printf '%s\n' $(((10#$raw * 10#$pct) / 100)) +} + +fm_resgate_voice_port() { + printf '%s\n' "${FM_RESGATE_VOICE_PORT:-$FM_RESGATE_VOICE_PORT_DEFAULT}" +} + +# The validated SSH connect timeout in seconds, or exit 1 when the pin is +# unusable. Zero is rejected alongside the non-numeric and negative forms +# because bin/fm-timeout-lib.sh's header states a non-positive bound is not a +# bound at all - `timeout 0` and the perl fallback's `alarm 0` both DISABLE the +# deadline - so a pin of 0 or -5 would compute a bound of 5 or 0 seconds and +# hand an unreachable host an unbounded probe instead of a bounded one. +fm_resgate_ssh_timeout() { + local secs=${FM_RESGATE_SSH_TIMEOUT:-$FM_RESGATE_SSH_TIMEOUT_DEFAULT} + fm_resgate_is_uint "$secs" || return 1 + [ "$secs" -ge 1 ] || return 1 + printf '%s\n' "$secs" +} + +fm_resgate_ssh_raw() { # + local host=$1 cmd=$2 bound secs + secs=$(fm_resgate_ssh_timeout) || return 1 + bound=$((secs + 5)) + fm_run_timed "$bound" ssh \ + -o BatchMode=yes \ + -o ConnectTimeout="$secs" \ + -o ServerAliveInterval=2 \ + -o ServerAliveCountMax=2 \ + -o ForwardAgent=no \ + "$host" "$cmd" +} + +fm_resgate_gpu_busy_mb() { + printf '%s\n' "${FM_RESGATE_GPU_BUSY_MB:-$FM_RESGATE_GPU_BUSY_MB_DEFAULT}" +} + +# One PowerShell round trip: the JARVIS-voice gateway port's listen state, the +# Qwen-identifying named process, and aggregate GPU memory used (see the file +# header for why aggregate memory + named process, not per-process +# compute-apps attribution). Every line is emitted unconditionally, including +# on failure (=probe-failed), so absorption can tell "this specific +# reading failed" apart from "no probe output arrived at all". +# +# All three readings run under -ErrorAction Stop inside try/catch rather than +# -ErrorAction SilentlyContinue, because silencing the error makes an absent +# result indistinguishable from a failed measurement: a missing NetTCPIP +# module or an unhealthy CIM service would otherwise read as the fully +# permissive "voice_port=not-listening" while the gateway is in fact up. Only +# the one error identity that genuinely means "nothing matched" +# (CmdletizationQuery_NotFound* for the port, NoProcessFoundForGivenName* for +# the process) is reported as a negative reading; every other failure, +# including the cmdlet not existing at all, reports probe-failed, which +# absorption maps to unknown and the owner decision fails closed on. +fm_resgate_home_gpu_probe_cmd() { # + local port=$1 proc=$2 cmd + cmd=$(cat <<'CMD' +try { + $conn = Get-NetTCPConnection -LocalPort @PORT@ -State Listen -ErrorAction Stop + if ($conn) { Write-Output 'FM_RESGATE voice_port=listening' } else { Write-Output 'FM_RESGATE voice_port=not-listening' } +} catch { + if ($_.FullyQualifiedErrorId -like 'CmdletizationQuery_NotFound*') { Write-Output 'FM_RESGATE voice_port=not-listening' } + else { Write-Output 'FM_RESGATE voice_port=probe-failed' } +} +try { + $proc = Get-Process -Name '@PROCESS@' -ErrorAction Stop + if ($proc) { Write-Output 'FM_RESGATE gpu_process=running' } else { Write-Output 'FM_RESGATE gpu_process=not-running' } +} catch { + if ($_.FullyQualifiedErrorId -like 'NoProcessFoundForGivenName*') { Write-Output 'FM_RESGATE gpu_process=not-running' } + else { Write-Output 'FM_RESGATE gpu_process=probe-failed' } +} +try { + $used = & nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits 2>$null + if ($LASTEXITCODE -eq 0 -and $used) { Write-Output ('FM_RESGATE gpu_used_mb=' + ($used | Select-Object -First 1).Trim()) } + else { Write-Output 'FM_RESGATE gpu_used_mb=probe-failed' } +} catch { + Write-Output 'FM_RESGATE gpu_used_mb=probe-failed' +} +CMD + ) + cmd=${cmd//@PORT@/$port} + printf '%s\n' "${cmd//@PROCESS@/$proc}" +} + +# Parses fm_resgate_home_gpu_probe_cmd's output into FM_RESGATE_GPU_VOICE, +# FM_RESGATE_GPU_PROCESS, and FM_RESGATE_GPU_USED_MB ("yes"/"no", or a MiB +# integer; empty/unset when that line never arrived or read "probe-failed" - +# both mean "unknown", never a guessed permissive value). Returns 1 when NOT +# EVEN ONE line arrived at all (SSH itself failed). +# The remote host is Windows PowerShell, whose Write-Output terminates every +# line with CRLF; `read` only strips the trailing LF, so each line keeps a +# trailing CR that would otherwise make every exact-match case pattern below +# fail silently and fall through to "unknown" - live-verified against the +# real home host, not a hypothetical. +fm_resgate_absorb_gpu_probe() { # + local text=$1 line key val seen=0 + FM_RESGATE_GPU_VOICE= + FM_RESGATE_GPU_PROCESS= + FM_RESGATE_GPU_USED_MB= + while IFS= read -r line || [ -n "$line" ]; do + line=${line%$'\r'} + case "$line" in + FM_RESGATE\ *) + seen=1 + key=${line#FM_RESGATE } + val=${key#*=} + key=${key%%=*} + case "$key" in + voice_port) + case "$val" in + listening) FM_RESGATE_GPU_VOICE=yes ;; + not-listening) FM_RESGATE_GPU_VOICE=no ;; + esac + ;; + gpu_process) + case "$val" in + running) FM_RESGATE_GPU_PROCESS=yes ;; + not-running) FM_RESGATE_GPU_PROCESS=no ;; + esac + ;; + gpu_used_mb) + fm_resgate_is_uint "$val" && FM_RESGATE_GPU_USED_MB=$val + ;; + esac + ;; + esac + done < /dev/null || return 1 + host=$(fm_resgate_ssh_alias home) + out=$(fm_resgate_ssh_raw "$host" \ + "$(fm_resgate_home_gpu_probe_cmd "$port" "$FM_RESGATE_GPU_PROCESS_NAME")" \ + 2>/dev/null) || out= + fm_resgate_absorb_gpu_probe "$out" || return 1 + case "$FM_RESGATE_GPU_VOICE" in yes | no) ;; *) return 1 ;; esac + if [ "$FM_RESGATE_GPU_VOICE" = yes ]; then + FM_RESGATE_GPU_OWNER=voice + return 0 + fi + case "$FM_RESGATE_GPU_PROCESS" in yes | no) ;; *) return 1 ;; esac + fm_resgate_is_uint "${FM_RESGATE_GPU_USED_MB:-}" || return 1 + qwen_active=no + if [ "$FM_RESGATE_GPU_PROCESS" = yes ] \ + && [ "$FM_RESGATE_GPU_USED_MB" -ge "$busy" ]; then + qwen_active=yes + fi + if [ "$qwen_active" = yes ]; then + FM_RESGATE_GPU_OWNER=qwen + else + FM_RESGATE_GPU_OWNER=none + fi + return 0 +} + +# May (qwen|voice) start or keep running on the home PC's GPU right +# now? Sets FM_RESGATE_GPU_REASON. Fails closed (returns 1) on unknown: an +# unmeasurable reading must never be read as permission, and must never let one +# side quietly work around the other's reservation. +# shellcheck disable=SC2034 +fm_resgate_gpu_available_for() { # + local want=$1 + case "$want" in qwen | voice) ;; *) return 1 ;; esac + fm_resgate_home_gpu_owner + case "$FM_RESGATE_GPU_OWNER" in + none) + FM_RESGATE_GPU_REASON='GPU is free' + return 0 + ;; + qwen) + if [ "$want" = qwen ]; then + FM_RESGATE_GPU_REASON='Qwen already holds the GPU' + return 0 + fi + FM_RESGATE_GPU_REASON='GPU is reserved for Qwen; JARVIS voice must not start' + return 1 + ;; + voice) + if [ "$want" = voice ]; then + FM_RESGATE_GPU_REASON='JARVIS voice already holds the GPU' + return 0 + fi + FM_RESGATE_GPU_REASON='GPU is reserved for JARVIS voice; Qwen must not start' + return 1 + ;; + *) + FM_RESGATE_GPU_REASON='GPU ownership could not be measured; refusing rather than guessing' + return 1 + ;; + esac +} diff --git a/bin/fm-resgate.sh b/bin/fm-resgate.sh new file mode 100755 index 0000000000..fd87b99a55 --- /dev/null +++ b/bin/fm-resgate.sh @@ -0,0 +1,149 @@ +#!/usr/bin/env bash +# Fleet resource governance for the captain's work PC and home PC: weekly +# clock-window percentage caps, a manual override, and GPU exclusivity +# between Qwen and the JARVIS voice worker on the home PC. +# Usage: +# fm-resgate.sh schedule +# fm-resgate.sh cap +# fm-resgate.sh override set +# fm-resgate.sh override clear +# fm-resgate.sh override status +# fm-resgate.sh gpu status +# fm-resgate.sh gpu allow +# +# `schedule` prints the clock-window verdict alone (uncapped/capped/blocked), +# ignoring any manual override. `cap` prints the EFFECTIVE percentage (100, +# 50, or 0), folding in an active override. `override set/clear` arm or +# release the durable state/.resgate-cap- marker documented in +# docs/configuration.md "Fleet resource governance"; `both` touches both +# roles. All three override subcommands print one `=armed|clear` line +# per role they touched, re-read from disk after the operation, so a `both` +# run that only half applied still names each host's real state; the exit +# status is non-zero if any role failed. `gpu status` prints the freshly +# probed home-PC GPU owner; `gpu allow` exits 0 when may start or +# keep running on that GPU right now, 1 otherwise, printing the reason +# either way. The script header of bin/fm-resgate-lib.sh owns the schedule +# windows, the fail-closed rules, and the GPU-detection signals. +set -eu + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +FM_ROOT="${FM_ROOT_OVERRIDE:-$(cd "$SCRIPT_DIR/.." && pwd)}" +FM_HOME="${FM_HOME:-${FM_ROOT_OVERRIDE:-$FM_ROOT}}" +STATE="${FM_STATE_OVERRIDE:-$FM_HOME/state}" + +# shellcheck source=bin/fm-resgate-lib.sh +. "$SCRIPT_DIR/fm-resgate-lib.sh" + +usage() { + sed -n '2,26{s/^# \{0,1\}//;p;}' "$0" +} + +die() { + printf 'error: resgate: %s\n' "$1" >&2 + exit 1 +} + +roles_for() { # + case "$1" in + work | home) printf '%s\n' "$1" ;; + both) printf 'work\nhome\n' ;; + *) die "role must be work, home, or both: $1" ;; + esac +} + +cmd_schedule() { + local role=${1:-} + fm_resgate_role_ok "$role" || die "role must be work or home: $role" + fm_resgate_schedule_state "$role" + printf 'state=%s\n' "$FM_RESGATE_SCHEDULE_STATE" + printf 'reason=%s\n' "$FM_RESGATE_SCHEDULE_REASON" +} + +cmd_cap() { + local role=${1:-} + fm_resgate_role_ok "$role" || die "role must be work or home: $role" + fm_resgate_capacity_pct "$STATE" "$role" + printf 'pct=%s\n' "$FM_RESGATE_PCT" + printf 'state=%s\n' "$FM_RESGATE_STATE" + printf 'reason=%s\n' "$FM_RESGATE_REASON" +} + +# Print what the marker for ACTUALLY reads right now, re-read from +# disk rather than assumed from the operation's exit status. Every role a +# command touched reports its own line, so a `both` run that only half +# applied - `state/` writable for one operation and not the next, one marker +# stuck behind a stale mount - still names which host changed and which did +# not, instead of leaving the captain with one error line and no way to tell +# whether the other host was capped or released. +print_override_state() { # + if fm_resgate_override_active "$STATE" "$1"; then + printf '%s=armed\n' "$1" + else + printf '%s=clear\n' "$1" + fi +} + +cmd_override() { + local action=${1:-} role_arg=${2:-} role roles rc=0 + case "$action" in set | clear | status) ;; *) + die "override subcommand must be set, clear, or status: $action" ;; + esac + # Validated (and, on a bad role, exited) here, in this shell - never inside + # a process-substitution subshell, whose exit status the caller cannot see. + roles=$(roles_for "$role_arg") + for role in $roles; do + case "$action" in + set) + fm_resgate_override_set "$STATE" "$role" Kappung \ + || { printf 'error: resgate: could not arm override for %s\n' "$role" >&2; rc=1; } + ;; + clear) + # A failed role never aborts the run: the remaining roles are still + # attempted, because stopping early would silently leave a host in + # whichever state it was already in. + fm_resgate_override_clear "$STATE" "$role" \ + || { printf 'error: resgate: could not clear override for %s\n' "$role" >&2; rc=1; } + ;; + esac + print_override_state "$role" + done + return "$rc" +} + +cmd_gpu() { + local action=${1:-} want=${2:-} + case "$action" in + status) + fm_resgate_home_gpu_owner || true + printf 'owner=%s\n' "$FM_RESGATE_GPU_OWNER" + printf 'voice_port=%s\n' "${FM_RESGATE_GPU_VOICE:-unknown}" + printf 'process(%s)=%s\n' "$FM_RESGATE_GPU_PROCESS_NAME" "${FM_RESGATE_GPU_PROCESS:-unknown}" + printf 'gpu_used_mb=%s\n' "${FM_RESGATE_GPU_USED_MB:-unknown}" + printf 'gpu_busy_threshold_mb=%s\n' "$(fm_resgate_gpu_busy_mb)" + ;; + allow) + case "$want" in qwen | voice) ;; *) die "gpu allow needs qwen or voice: $want" ;; esac + if fm_resgate_gpu_available_for "$want"; then + printf 'allow=yes\n' + printf 'reason=%s\n' "$FM_RESGATE_GPU_REASON" + else + printf 'allow=no\n' + printf 'reason=%s\n' "$FM_RESGATE_GPU_REASON" + return 1 + fi + ;; + *) die "gpu subcommand must be status or allow: $action" ;; + esac +} + +[ "$#" -ge 1 ] || { usage >&2; exit 2; } +CMD=$1 +shift +case "$CMD" in + schedule) cmd_schedule "$@" ;; + cap) cmd_cap "$@" ;; + override) cmd_override "$@" ;; + gpu) cmd_gpu "$@" ;; + -h | --help) usage ;; + *) die "unknown command: $CMD" ;; +esac diff --git a/bin/fm-test-run.sh b/bin/fm-test-run.sh index a4ef94500f..a99b872e99 100755 --- a/bin/fm-test-run.sh +++ b/bin/fm-test-run.sh @@ -657,7 +657,7 @@ portable_serial_unhinted() { tmp=$(mktemp -d "${TMPDIR:-/tmp}/fm-test-unhinted.XXXXXX") || return 1 portable_serial_weight_hints | awk 'NF { print $1 }' | LC_ALL=C sort -u >"$tmp/hinted" list_portable_serial | LC_ALL=C sort -u >"$tmp/serial" - comm -23 "$tmp/serial" "$tmp/hinted" + LC_ALL=C comm -23 "$tmp/serial" "$tmp/hinted" rm -rf "$tmp" } @@ -805,8 +805,8 @@ run_coverage_guard() { return 1 fi cat "$tmp/s1" "$tmp/s2" | LC_ALL=C sort -u >"$tmp/shards_union" - missing=$(comm -23 "$tmp/proven" "$tmp/shards_union" || true) - extra=$(comm -13 "$tmp/proven" "$tmp/shards_union" || true) + missing=$(LC_ALL=C comm -23 "$tmp/proven" "$tmp/shards_union" || true) + extra=$(LC_ALL=C comm -13 "$tmp/proven" "$tmp/shards_union" || true) if [ -n "$missing" ] || [ -n "$extra" ]; then log "coverage guard: portable shards must equal the proven-isolated set" [ -z "$missing" ] || { log "missing from shards:"; printf '%s\n' "$missing" >&2; } @@ -850,8 +850,8 @@ run_coverage_guard() { return 1 fi LC_ALL=C sort -u "$tmp/serial_shards_raw" >"$tmp/serial_shards" - missing=$(comm -23 "$tmp/serial" "$tmp/serial_shards" || true) - extra=$(comm -13 "$tmp/serial" "$tmp/serial_shards" || true) + missing=$(LC_ALL=C comm -23 "$tmp/serial" "$tmp/serial_shards" || true) + extra=$(LC_ALL=C comm -13 "$tmp/serial" "$tmp/serial_shards" || true) if [ -n "$missing" ] || [ -n "$extra" ]; then log "coverage guard: portable serial shards must equal the portable serial lane" [ -z "$missing" ] || { log "missing from serial shards:"; printf '%s\n' "$missing" >&2; } @@ -863,7 +863,7 @@ run_coverage_guard() { for pair in "shards_union:serial" "shards_union:herdr" "serial:herdr"; do a=${pair%%:*} b=${pair#*:} - comm -12 "$tmp/$a" "$tmp/$b" >"$tmp/overlap" + LC_ALL=C comm -12 "$tmp/$a" "$tmp/$b" >"$tmp/overlap" if [ -s "$tmp/overlap" ]; then log "coverage guard: overlap between $a and $b:" cat "$tmp/overlap" >&2 @@ -881,8 +881,8 @@ run_coverage_guard() { return 1 fi LC_ALL=C sort -u "$tmp/union_raw" >"$tmp/union" - missing=$(comm -23 "$tmp/all" "$tmp/union" || true) - extra=$(comm -13 "$tmp/all" "$tmp/union" || true) + missing=$(LC_ALL=C comm -23 "$tmp/all" "$tmp/union" || true) + extra=$(LC_ALL=C comm -13 "$tmp/all" "$tmp/union" || true) if [ -n "$missing" ] || [ -n "$extra" ]; then log "coverage guard: union of portable shards + portable serial + Herdr must equal tests/*.test.sh" [ -z "$missing" ] || { log "missing from union:"; printf '%s\n' "$missing" >&2; } @@ -912,7 +912,7 @@ run_coverage_guard() { "$ROOT/bin/fm-test-isolation-proof.sh" --list | LC_ALL=C sort -u >"$tmp/proof_list" if ! cmp -s "$tmp/proven" "$tmp/proof_list"; then log "coverage guard: embedded proven-isolated set diverges from bin/fm-test-isolation-proof.sh --list" - comm -3 "$tmp/proven" "$tmp/proof_list" >&2 || true + LC_ALL=C comm -3 "$tmp/proven" "$tmp/proof_list" >&2 || true rm -rf "$tmp" return 1 fi diff --git a/docs/configuration.md b/docs/configuration.md index d89b583720..0f0f80cf3a 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -198,6 +198,40 @@ The bound is required rather than cosmetic because churn and pane staleness read The flag is a home-local supervision-noise preference and is not inherited by secondmate homes, which run their own crew mix. [`architecture.md`](architecture.md) owns the triage contract and `bin/fm-watch.sh`'s `signal_turnend_panes_churned` owns the exact evidence and fail-closed boundaries. +## Fleet resource governance (state/.resgate-cap-work, state/.resgate-cap-home) + +`bin/fm-resgate.sh` (CLI) and `bin/fm-resgate-lib.sh` (the primitives it wraps) protect the captain's own use of the work PC (`Valentino-Arbeit`) and home PC (`Valentino`, RTX 4080 Super) from the fleet, and keep Qwen and the JARVIS voice worker off the home PC's GPU at the same time. +The exact schedule windows, the percentage-cap arithmetic, the fail-closed rules, and the GPU-detection signals are owned by `bin/fm-resgate-lib.sh`'s header; this section covers only where the surface lives and how the captain controls it. + +Two fixed roles, `work` and `home`, evaluated once against THIS host's own clock forced into `Europe/Berlin` - never a remote host's clock, which could be wrong or drifted and must never be able to loosen or defeat the gate: + +```sh +bin/fm-resgate.sh schedule work # clock-window verdict alone (uncapped/capped/blocked) +bin/fm-resgate.sh cap home # effective percentage (100/50/0), folding in a manual override +bin/fm-resgate.sh gpu status # freshly probed home-PC GPU owner: none/qwen/voice/unknown +bin/fm-resgate.sh gpu allow qwen # exit 0/1: may Qwen start or keep running on the GPU right now +``` + +Manual override: `state/.resgate-cap-` is a plain presence-based marker, written atomically like `state/.afk`. +While it exists, that role reads capped (50%) immediately, regardless of which window the clock lands in - but the marker can only ever tighten the gate, never loosen it: an unreadable clock stays `blocked` at 0% even with the marker armed, because a control whose purpose is to restrict must not hand out capacity no measurement supports. +Firstmate matches the captain's chat wording case-insensitively, and checks the release form FIRST: "Kappung auf" always means release, never an arming instruction, because bare `Kappung` is a substring of it and would otherwise be misread as "arm both". +Bare "Kappung auf" releases both markers, and "Kappung auf " releases only that one, resolved by the same hostname matching as the arming form below. +Only when the message is not the release form does firstmate arm: bare `Kappung` arms both roles, `Kappung ` arms only that one, matching `Valentino-Arbeit`/`work`/`Arbeits-PC` to `work` and `Valentino`/`home`/`Heim-PC` to `home` (`auf` is not a hostname and never resolves to a role). +The hostname tokens are matched longest-first for the same substring reason: `Valentino` is a substring of `Valentino-Arbeit`, so `Valentino-Arbeit` and `Arbeits-PC` must both be ruled out before bare `Valentino` is considered, or "Kappung Valentino-Arbeit" caps the home PC and leaves the work PC - the machine the captain actually asked to protect - uncapped through his working hours. +This is a plain state file firstmate touches directly (`bin/fm-resgate.sh override set|clear `, or an equivalent direct write following `fm_resgate_override_set`/`fm_resgate_override_clear`'s header contract in `bin/fm-resgate-lib.sh`), not a skill, and not wired into spawn or dispatch plumbing beyond this marker. +Arming and releasing can both fail (an unwritable `state/`), and both report it: the CLI prints `could not arm/clear override for ` and exits non-zero, so a release that did not actually happen is never mistaken for a lifted cap. +`set`, `clear`, and `status` all print one `=armed|clear` line per role they touched, re-read from the marker after the operation rather than assumed from its exit status, and a failing role never aborts the remaining ones: when a `both` run only half applies, the output still names which host is capped and which is released, instead of leaving a single error line and no way to tell what happened to the other host. + +Fail-closed discipline: every measurement this surface cannot read - the authoritative clock, an SSH probe, the voice port, the GPU reading - yields the most restrictive answer, never a guess. +"Cannot read" includes a clock that answers confidently with the wrong zone: `date` does not fail when `Europe/Berlin` is unresolvable (missing tzdata on a slim image), it silently falls back to UTC, so the read is accepted only when the zone abbreviation and UTC offset it returns are a matching Europe/Berlin pair, and is otherwise treated exactly like an unreadable clock. +A schedule read that cannot happen at all reports 0% (`blocked`), stricter than the ordinary 50% cap and stricter than an armed override, and a GPU reading that cannot be completed reports `unknown`, which refuses both Qwen and JARVIS voice rather than picking a side. +An out-of-range `FM_RESGATE_VOICE_PORT`, a non-numeric `FM_RESGATE_GPU_BUSY_MB`, or a `FM_RESGATE_SSH_TIMEOUT` that is not a positive integer is treated the same way: the GPU reading reports `unknown` instead of probing the wrong port, comparing against an unusable threshold, or running the probe with a bound of zero seconds, which disables the deadline rather than bounding it. + +GPU exclusivity: JARVIS voice is detected by its gateway port (currently `7414`, see `data/learnings.md`), never by process name, because process-name detection has broken this fleet's integration before. +That port reading is authoritative and is decided first: a listening gateway means `owner=voice` and the Qwen signals are never consulted, because aggregate card memory cannot say whose memory it is - an idle-but-resident Qwen model plus the voice worker's own VRAM would otherwise misread as contention and refuse the genuinely-running voice worker the card it already holds. +Qwen is detected by a named-process check (`ollama`, the currently live identity) corroborated by aggregate GPU memory clearing a threshold, not by `nvidia-smi --query-compute-apps` per-process attribution: live-tested against the real home host, that query lists every ordinary desktop GPU context (window compositor, open browsers) with no per-process memory field left to filter the noise by, so it cannot isolate a genuine workload there. +Both signals for the process-plus-memory check, and the port check, come from one bounded SSH round trip to the home host. + ## Gate defaults (.no-mistakes.yaml) The tracked `.no-mistakes.yaml` sets `test.evidence.store_in_repo: true` and pins `commands.lint` to `bin/fm-lint.sh` so local lint matches CI. diff --git a/docs/scripts.md b/docs/scripts.md index d23aafca7c..7f83e7ba40 100644 --- a/docs/scripts.md +++ b/docs/scripts.md @@ -55,6 +55,8 @@ The shared no-mistakes gate refusal for fleet lifecycle entrypoints is summarize | `fm-remote-readiness-lib.sh` | Shared remote second-mate readiness gate: check and, when needed, repair then re-check through `fm-remote-doctor.sh` | | [`fm-project-origin-lib.sh`](../bin/fm-project-origin-lib.sh) | Accepted origin-form owner shared by both remote provisioning boundaries | | `fm-spawn.sh` | Spawn crewmates, scouts, `id=repo` batches, and secondmates on the resolved harness and runtime backend | +| `fm-resgate.sh` | CLI for the weekly clock-window resource cap, manual override marker, and home-PC GPU exclusivity | +| `fm-resgate-lib.sh` | Schedule-window, percentage-cap, override-marker, and GPU-exclusivity primitives fm-resgate.sh wraps | | `fm-backend.sh` | Runtime-backend selection, meta helpers, selector resolution, and operation dispatch | | `fm-backend-hometag-lib.sh` | Shared per-installation home-tag derivation for zellij tab and cmux workspace titles | | `fm-composer-lib.sh` | Single fleet-wide owner of composer shapes, capability-aware screen classification, and verdicts | diff --git a/tests/fm-resgate.test.sh b/tests/fm-resgate.test.sh new file mode 100755 index 0000000000..6b120f46f9 --- /dev/null +++ b/tests/fm-resgate.test.sh @@ -0,0 +1,801 @@ +#!/usr/bin/env bash +# fm-resgate-lib.sh / fm-resgate.sh - weekly clock-window resource governance, +# the manual override marker, and home-PC GPU exclusivity between Qwen and +# JARVIS voice. +# +# Covers: the captain's exact schedule windows and their boundaries (including +# the Friday-evening-through-Monday-morning free span on the work PC and the +# always-capped weekend on the home PC, both of which fall out of the same +# small per-weekday window rather than separate weekend-boundary code); the +# fail-closed clock, role, and GPU-probe paths; the override marker's atomic +# set/clear/status roundtrip and its precedence over the clock; and the GPU +# owner/availability decision, including the real CRLF line-ending bug found +# while live-testing this suite against the actual home host (Windows +# PowerShell terminates every line with CRLF; a naive `read -r` loop leaves +# the trailing CR on the last field and silently breaks every exact-match +# case pattern). +set -u + +# shellcheck source=tests/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +# shellcheck source=bin/fm-resgate-lib.sh +. "$ROOT/bin/fm-resgate-lib.sh" + +CLI="$ROOT/bin/fm-resgate.sh" + +# --- role validation --------------------------------------------------------- + +test_role_ok() { + fm_resgate_role_ok work || fail "work must be a valid role" + fm_resgate_role_ok home || fail "home must be a valid role" + fm_resgate_role_ok both && fail "both must not be a valid single role" + fm_resgate_role_ok '' && fail "empty must not be a valid role" + pass "fm_resgate_role_ok accepts only work and home" +} + +# --- clock parsing ------------------------------------------------------------ + +test_now_fields_from_override() { + FM_RESGATE_NOW_OVERRIDE="3 14 05" fm_resgate_now_fields \ + || fail "a well-formed override must parse" + # Re-run in this shell so the globals are visible to the assertion below. + # shellcheck disable=SC2030,SC2031 + ( FM_RESGATE_NOW_OVERRIDE="3 14 05"; fm_resgate_now_fields + [ "$FM_RESGATE_NOW_DOW" = 3 ] || exit 1 + [ "$FM_RESGATE_NOW_MOD" = "$((14 * 60 + 5))" ] || exit 1 + ) || fail "override dow/mod must parse to Wednesday 14:05" + pass "fm_resgate_now_fields parses a well-formed override" +} + +test_now_fields_rejects_malformed_override() { + FM_RESGATE_NOW_OVERRIDE="not a clock reading" fm_resgate_now_fields \ + && fail "a malformed override must not parse" + FM_RESGATE_NOW_OVERRIDE="8 14 05" fm_resgate_now_fields \ + && fail "a day-of-week of 8 must not parse" + FM_RESGATE_NOW_OVERRIDE="3 24 05" fm_resgate_now_fields \ + && fail "an hour of 24 must not parse" + FM_RESGATE_NOW_OVERRIDE="3 14 60" fm_resgate_now_fields \ + && fail "a minute of 60 must not parse" + pass "fm_resgate_now_fields rejects malformed clock readings" +} + +test_now_fields_handles_leading_zero_hours() { + # 08 and 09 are invalid octal literals; a naive $((hh*60+mm)) would abort. + # shellcheck disable=SC2030,SC2031 + ( FM_RESGATE_NOW_OVERRIDE="1 08 09"; fm_resgate_now_fields + [ "$FM_RESGATE_NOW_MOD" = "$((8 * 60 + 9))" ] + ) || fail "leading-zero hour/minute fields must not be read as octal" + pass "fm_resgate_now_fields treats leading-zero HH/MM as decimal" +} + +# --- schedule state: work PC -------------------------------------------------- + +test_work_capped_within_window() { + FM_RESGATE_NOW_OVERRIDE="3 14 00" fm_resgate_schedule_state work + [ "$FM_RESGATE_SCHEDULE_STATE" = capped ] \ + || fail "work PC must be capped Wednesday 14:00 (inside 10:00-19:30)" + pass "work PC is capped mid-window on a weekday" +} + +test_work_capped_at_start_boundary() { + FM_RESGATE_NOW_OVERRIDE="1 10 00" fm_resgate_schedule_state work + [ "$FM_RESGATE_SCHEDULE_STATE" = capped ] \ + || fail "work PC must be capped starting exactly at 10:00" + FM_RESGATE_NOW_OVERRIDE="1 09 59" fm_resgate_schedule_state work + [ "$FM_RESGATE_SCHEDULE_STATE" = uncapped ] \ + || fail "work PC must still be uncapped at 09:59" + pass "work PC's capped window starts exactly at 10:00, inclusive" +} + +test_work_uncapped_at_end_boundary() { + FM_RESGATE_NOW_OVERRIDE="1 19 30" fm_resgate_schedule_state work + [ "$FM_RESGATE_SCHEDULE_STATE" = uncapped ] \ + || fail "work PC must be uncapped again exactly at 19:30" + FM_RESGATE_NOW_OVERRIDE="1 19 29" fm_resgate_schedule_state work + [ "$FM_RESGATE_SCHEDULE_STATE" = capped ] \ + || fail "work PC must still be capped at 19:29" + pass "work PC's capped window ends exactly at 19:30, exclusive" +} + +test_work_free_through_weekend_span() { + local case + for case in "5 19 30" "5 23 59" "6 00 00" "6 12 00" "7 23 59" "1 00 00" "1 09 59"; do + FM_RESGATE_NOW_OVERRIDE="$case" fm_resgate_schedule_state work + [ "$FM_RESGATE_SCHEDULE_STATE" = uncapped ] \ + || fail "work PC must be uncapped throughout Fri 19:30 - Mon 10:00 (failed at '$case')" + done + pass "work PC stays uncapped continuously from Friday 19:30 through Monday 10:00" +} + +# --- schedule state: home PC --------------------------------------------------- + +test_home_free_within_window() { + FM_RESGATE_NOW_OVERRIDE="3 10 00" fm_resgate_schedule_state home + [ "$FM_RESGATE_SCHEDULE_STATE" = uncapped ] \ + || fail "home PC must be free Wednesday 10:00 (inside 04:00-19:00)" + pass "home PC is free mid-window on a weekday" +} + +test_home_free_starts_at_boundary() { + FM_RESGATE_NOW_OVERRIDE="1 04 00" fm_resgate_schedule_state home + [ "$FM_RESGATE_SCHEDULE_STATE" = uncapped ] \ + || fail "home PC must be free starting exactly at 04:00" + FM_RESGATE_NOW_OVERRIDE="1 03 59" fm_resgate_schedule_state home + [ "$FM_RESGATE_SCHEDULE_STATE" = capped ] \ + || fail "home PC must still be capped at 03:59" + pass "home PC's free window starts exactly at 04:00, inclusive" +} + +test_home_free_ends_at_boundary() { + FM_RESGATE_NOW_OVERRIDE="1 19 00" fm_resgate_schedule_state home + [ "$FM_RESGATE_SCHEDULE_STATE" = capped ] \ + || fail "home PC must be capped again exactly at 19:00" + FM_RESGATE_NOW_OVERRIDE="1 18 59" fm_resgate_schedule_state home + [ "$FM_RESGATE_SCHEDULE_STATE" = uncapped ] \ + || fail "home PC must still be free at 18:59" + pass "home PC's free window ends exactly at 19:00, exclusive" +} + +test_home_capped_all_weekend() { + local case + for case in "5 19 00" "5 23 59" "6 00 00" "6 12 00" "7 23 59" "1 00 00" "1 03 59"; do + FM_RESGATE_NOW_OVERRIDE="$case" fm_resgate_schedule_state home + [ "$FM_RESGATE_SCHEDULE_STATE" = capped ] \ + || fail "home PC must be capped throughout Fri 19:00 - Mon 04:00 (failed at '$case')" + done + pass "home PC stays capped continuously across the whole weekend" +} + +# --- fail-closed clock --------------------------------------------------------- + +test_schedule_blocked_on_unreadable_clock() { + FM_RESGATE_NOW_OVERRIDE="garbage" fm_resgate_schedule_state work + [ "$FM_RESGATE_SCHEDULE_STATE" = blocked ] \ + || fail "an unreadable clock must yield 'blocked', not a guessed state" + pass "schedule state fails closed to 'blocked' on an unreadable clock" +} + +test_capacity_pct_zero_on_unreadable_clock() { + local state + state=$(fm_test_tmproot resgate-pct-blocked) + FM_RESGATE_NOW_OVERRIDE="garbage" fm_resgate_capacity_pct "$state" work + [ "$FM_RESGATE_PCT" = 0 ] \ + || fail "an unreadable clock must yield 0%, stricter than the ordinary 50% cap" + pass "capacity_pct fails closed to 0% on an unreadable clock" +} + +# --- fail-closed clock: the zone must actually have resolved --------------------- +# +# `date` exits 0 on a zone it cannot resolve and silently reports UTC, so these +# shadow `date` on PATH to hand fm_resgate_now_fields each reading a real host +# could produce and assert the resulting schedule verdict. + +fake_date_at() { # + local fakebin=$1 + # Substitutes the requested specifiers rather than echoing a fixed line, so + # the stub answers whatever format the library asks for - the reading it + # returns is what varies here, never the shape of the call. + cat > "$fakebin/date" <&1) || rc=$? + [ "$rc" -ne 0 ] \ + || fail "the CLI must exit non-zero when the cap could not actually be released" + case "$out" in + *"could not clear override for work"*) ;; + *) fail "the CLI must say the release failed, got: $out" ;; + esac + + chmod 755 "$state" + fm_resgate_override_clear "$state" work \ + || fail "a clear that really removes the marker must report success" + fm_resgate_override_active "$state" work \ + && fail "the marker must be gone after a successful clear" + pass "a failed override clear is reported, never mistaken for a lifted cap" +} + +# --- percentage arithmetic ------------------------------------------------------ + +test_apply_pct_arithmetic() { + [ "$(fm_resgate_apply_pct 10 100)" = 10 ] || fail "100% of 10 must be 10" + [ "$(fm_resgate_apply_pct 10 50)" = 5 ] || fail "50% of 10 must be 5" + [ "$(fm_resgate_apply_pct 3 50)" = 1 ] || fail "50% of 3 must floor to 1" + [ "$(fm_resgate_apply_pct 10 0)" = 0 ] || fail "0% of anything must be 0" + pass "fm_resgate_apply_pct halves (and floors) correctly" +} + +test_apply_pct_fails_closed_on_bad_input() { + [ "$(fm_resgate_apply_pct abc 50)" = 0 ] || fail "a non-numeric raw count must yield 0" + [ "$(fm_resgate_apply_pct 10 150)" = 0 ] || fail "a percentage over 100 must yield 0, not overshoot" + [ "$(fm_resgate_apply_pct '' 50)" = 0 ] || fail "an empty raw count must yield 0" + pass "fm_resgate_apply_pct fails closed on non-numeric or out-of-range input" +} + +# --- GPU exclusivity: mocked SSH ----------------------------------------------- +# +# The fake `ssh` prints exactly the FM_RESGATE lines a real probe would, with +# real Windows CRLF line endings, so absorption is exercised the same way the +# live host exercises it. + +fake_ssh_returning() { # + local fakebin=$1 output=$2 + cat > "$fakebin/ssh" < + cat > "$1/ssh" <<'SH' +#!/usr/bin/env bash +exit 255 +SH + chmod +x "$1/ssh" +} + +test_gpu_owner_qwen_when_process_and_memory_both_active() { + local tmp fakebin out + tmp=$(fm_test_tmproot resgate-gpu-qwen) + fakebin=$(fm_fakebin "$tmp") + out='FM_RESGATE voice_port=not-listening\r\nFM_RESGATE gpu_process=running\r\nFM_RESGATE gpu_used_mb=9046\r\n' + fake_ssh_returning "$fakebin" "$out" + # shellcheck disable=SC2030,SC2031 + ( PATH="$fakebin:$PATH" + . "$ROOT/bin/fm-resgate-lib.sh" + fm_resgate_home_gpu_owner + [ "$FM_RESGATE_GPU_OWNER" = qwen ] + ) || fail "process running + memory above threshold must read owner=qwen" + pass "GPU owner is qwen when both the named process and aggregate memory are active" +} + +test_gpu_owner_none_when_process_running_but_memory_idle() { + local tmp fakebin out + tmp=$(fm_test_tmproot resgate-gpu-idle-process) + fakebin=$(fm_fakebin "$tmp") + # A background service can be installed and running with no model loaded; + # the process check alone must not be read as "Qwen is active". + out='FM_RESGATE voice_port=not-listening\r\nFM_RESGATE gpu_process=running\r\nFM_RESGATE gpu_used_mb=512\r\n' + fake_ssh_returning "$fakebin" "$out" + # shellcheck disable=SC2030,SC2031 + ( PATH="$fakebin:$PATH" + . "$ROOT/bin/fm-resgate-lib.sh" + fm_resgate_home_gpu_owner + [ "$FM_RESGATE_GPU_OWNER" = none ] + ) || fail "an idle-but-running process below the memory threshold must read owner=none" + pass "GPU owner is none when the process is running but aggregate memory stays below threshold" +} + +test_gpu_owner_voice_when_port_listening() { + local tmp fakebin out + tmp=$(fm_test_tmproot resgate-gpu-voice) + fakebin=$(fm_fakebin "$tmp") + out='FM_RESGATE voice_port=listening\r\nFM_RESGATE gpu_process=not-running\r\nFM_RESGATE gpu_used_mb=1800\r\n' + fake_ssh_returning "$fakebin" "$out" + # shellcheck disable=SC2030,SC2031 + ( PATH="$fakebin:$PATH" + . "$ROOT/bin/fm-resgate-lib.sh" + fm_resgate_home_gpu_owner + [ "$FM_RESGATE_GPU_OWNER" = voice ] + ) || fail "a listening voice port must read owner=voice" + pass "GPU owner is voice when the gateway port is listening" +} + +test_gpu_owner_voice_wins_over_resident_qwen_signal() { + local tmp fakebin out + tmp=$(fm_test_tmproot resgate-gpu-voice-wins) + fakebin=$(fm_fakebin "$tmp") + # The home PC's likely normal state: the voice worker holds the card (port + # listening, its own VRAM on the meter) while the ollama service sits started + # but idle. Aggregate memory cannot say whose memory it is, so crediting it to + # Qwen here would refuse the voice worker the card it already holds - JARVIS + # blocking itself. The port is authoritative and decides alone. + out='FM_RESGATE voice_port=listening\r\nFM_RESGATE gpu_process=running\r\nFM_RESGATE gpu_used_mb=9046\r\n' + fake_ssh_returning "$fakebin" "$out" + # shellcheck disable=SC2030,SC2031 + ( PATH="$fakebin:$PATH" + . "$ROOT/bin/fm-resgate-lib.sh" + fm_resgate_home_gpu_owner || exit 1 + [ "$FM_RESGATE_GPU_OWNER" = voice ] || exit 1 + fm_resgate_gpu_available_for voice || exit 1 + fm_resgate_gpu_available_for qwen && exit 1 + exit 0 + ) || fail "a listening voice port must read owner=voice even with the Qwen process and card memory active, allowing voice and refusing qwen" + pass "a listening voice port owns the GPU outright; a resident Qwen signal never blocks the voice worker" +} + +test_gpu_available_for_blocks_the_other_side() { + local tmp fakebin out + tmp=$(fm_test_tmproot resgate-gpu-exclusive) + fakebin=$(fm_fakebin "$tmp") + out='FM_RESGATE voice_port=not-listening\r\nFM_RESGATE gpu_process=running\r\nFM_RESGATE gpu_used_mb=9046\r\n' + fake_ssh_returning "$fakebin" "$out" + # shellcheck disable=SC2030,SC2031 + ( PATH="$fakebin:$PATH" + . "$ROOT/bin/fm-resgate-lib.sh" + fm_resgate_gpu_available_for qwen || exit 1 + fm_resgate_gpu_available_for voice && exit 1 + exit 0 + ) || fail "GPU reserved for Qwen must allow qwen and refuse voice" + pass "gpu_available_for enforces exclusivity: the reserved side is allowed, the other refused" +} + +test_gpu_owner_unknown_on_crlf_lines_still_parses_correctly() { + # Regression for the exact bug found live-testing against the real host: + # Windows CRLF line endings left a trailing \r on the last field of each + # line, breaking every exact-match case pattern silently. + local tmp fakebin out + tmp=$(fm_test_tmproot resgate-gpu-crlf) + fakebin=$(fm_fakebin "$tmp") + out='FM_RESGATE voice_port=not-listening\r\nFM_RESGATE gpu_process=not-running\r\nFM_RESGATE gpu_used_mb=1200\r\n' + fake_ssh_returning "$fakebin" "$out" + # shellcheck disable=SC2030,SC2031 + ( PATH="$fakebin:$PATH" + . "$ROOT/bin/fm-resgate-lib.sh" + fm_resgate_home_gpu_owner + [ "$FM_RESGATE_GPU_VOICE" = no ] || exit 1 + [ "$FM_RESGATE_GPU_PROCESS" = no ] || exit 1 + [ "$FM_RESGATE_GPU_USED_MB" = 1200 ] || exit 1 + [ "$FM_RESGATE_GPU_OWNER" = none ] || exit 1 + ) || fail "CRLF-terminated probe lines must still parse to their exact values, not fall through to unknown" + pass "GPU probe parsing survives real Windows CRLF line endings" +} + +test_gpu_owner_unknown_on_probe_failure_field() { + local tmp fakebin out + tmp=$(fm_test_tmproot resgate-gpu-probefail) + fakebin=$(fm_fakebin "$tmp") + out='FM_RESGATE voice_port=not-listening\r\nFM_RESGATE gpu_process=running\r\nFM_RESGATE gpu_used_mb=probe-failed\r\n' + fake_ssh_returning "$fakebin" "$out" + # shellcheck disable=SC2030,SC2031 + ( PATH="$fakebin:$PATH" + . "$ROOT/bin/fm-resgate-lib.sh" + fm_resgate_home_gpu_owner && exit 1 + [ "$FM_RESGATE_GPU_OWNER" = unknown ] || exit 1 + fm_resgate_gpu_available_for qwen && exit 1 + fm_resgate_gpu_available_for voice && exit 1 + exit 0 + ) || fail "a failed individual reading (nvidia-smi failure) must fail closed to unknown for BOTH workloads" + pass "a single failed probe field fails the whole GPU reading closed, never a partial guess" +} + +test_gpu_owner_unknown_on_voice_port_probe_failure() { + local tmp fakebin out + tmp=$(fm_test_tmproot resgate-gpu-portfail) + fakebin=$(fm_fakebin "$tmp") + # The port measurement itself failed (missing NetTCPIP module, unhealthy CIM + # service). That must never be indistinguishable from "not-listening": the + # voice gateway may well be up, and reading it as free would let Qwen start + # on the card beside it - the exact simultaneity this gate exists to prevent. + out='FM_RESGATE voice_port=probe-failed\r\nFM_RESGATE gpu_process=not-running\r\nFM_RESGATE gpu_used_mb=1200\r\n' + fake_ssh_returning "$fakebin" "$out" + # shellcheck disable=SC2030,SC2031 + ( PATH="$fakebin:$PATH" + . "$ROOT/bin/fm-resgate-lib.sh" + fm_resgate_home_gpu_owner && exit 1 + [ "$FM_RESGATE_GPU_OWNER" = unknown ] || exit 1 + [ -z "$FM_RESGATE_GPU_VOICE" ] || exit 1 + fm_resgate_gpu_available_for qwen && exit 1 + fm_resgate_gpu_available_for voice && exit 1 + exit 0 + ) || fail "a failed voice-port reading must fail closed to unknown, never read as not-listening" + pass "a failed voice-port reading fails the GPU decision closed for both workloads" +} + +test_gpu_owner_unknown_on_process_probe_failure() { + local tmp fakebin out + tmp=$(fm_test_tmproot resgate-gpu-procfail) + fakebin=$(fm_fakebin "$tmp") + out='FM_RESGATE voice_port=not-listening\r\nFM_RESGATE gpu_process=probe-failed\r\nFM_RESGATE gpu_used_mb=9046\r\n' + fake_ssh_returning "$fakebin" "$out" + # shellcheck disable=SC2030,SC2031 + ( PATH="$fakebin:$PATH" + . "$ROOT/bin/fm-resgate-lib.sh" + fm_resgate_home_gpu_owner && exit 1 + [ "$FM_RESGATE_GPU_OWNER" = unknown ] || exit 1 + exit 0 + ) || fail "a failed process reading must fail closed to unknown, never read as not-running" + pass "a failed process reading fails the GPU decision closed" +} + +test_gpu_owner_unknown_on_unusable_ssh_timeout() { + local tmp fakebin out + tmp=$(fm_test_tmproot resgate-gpu-sshtimeout) + fakebin=$(fm_fakebin "$tmp") + # A healthy probe reachable through a fake ssh: only the timeout pin is bad. + # bin/fm-timeout-lib.sh's header states a non-positive bound is not a bound - + # it disables the deadline - so 0 and a negative pin (which the +5 arithmetic + # turns into exactly 0) must refuse to probe rather than probe unbounded. + out='FM_RESGATE voice_port=not-listening\r\nFM_RESGATE gpu_process=running\r\nFM_RESGATE gpu_used_mb=9046\r\n' + fake_ssh_returning "$fakebin" "$out" + # shellcheck disable=SC2030,SC2031 + ( PATH="$fakebin:$PATH" + . "$ROOT/bin/fm-resgate-lib.sh" + FM_RESGATE_SSH_TIMEOUT=-5 fm_resgate_home_gpu_owner 2>/dev/null && exit 1 + [ "$FM_RESGATE_GPU_OWNER" = unknown ] || exit 1 + FM_RESGATE_SSH_TIMEOUT=0 fm_resgate_home_gpu_owner 2>/dev/null && exit 1 + [ "$FM_RESGATE_GPU_OWNER" = unknown ] || exit 1 + FM_RESGATE_SSH_TIMEOUT=soon fm_resgate_home_gpu_owner 2>/dev/null && exit 1 + [ "$FM_RESGATE_GPU_OWNER" = unknown ] || exit 1 + fm_resgate_home_gpu_owner || exit 1 + [ "$FM_RESGATE_GPU_OWNER" = qwen ] || exit 1 + exit 0 + ) || fail "a zero, negative, or non-numeric SSH timeout must fail the GPU reading closed instead of probing with a disabled deadline" + pass "an unusable FM_RESGATE_SSH_TIMEOUT fails the GPU reading closed to unknown" +} + +test_gpu_owner_unknown_on_unusable_tunables() { + local tmp fakebin out + tmp=$(fm_test_tmproot resgate-gpu-tunables) + fakebin=$(fm_fakebin "$tmp") + # Fully healthy probe output: only the mistyped session pin is wrong. A bad + # threshold makes the -ge comparison error out (reading the card as free) and + # a bad port probes something that is not the gateway; both must fail closed. + out='FM_RESGATE voice_port=not-listening\r\nFM_RESGATE gpu_process=running\r\nFM_RESGATE gpu_used_mb=9046\r\n' + fake_ssh_returning "$fakebin" "$out" + # shellcheck disable=SC2030,SC2031 + ( PATH="$fakebin:$PATH" + . "$ROOT/bin/fm-resgate-lib.sh" + FM_RESGATE_GPU_BUSY_MB=lots fm_resgate_home_gpu_owner 2>/dev/null && exit 1 + [ "$FM_RESGATE_GPU_OWNER" = unknown ] || exit 1 + FM_RESGATE_VOICE_PORT=99999 fm_resgate_home_gpu_owner 2>/dev/null && exit 1 + [ "$FM_RESGATE_GPU_OWNER" = unknown ] || exit 1 + FM_RESGATE_VOICE_PORT=notaport fm_resgate_home_gpu_owner 2>/dev/null && exit 1 + [ "$FM_RESGATE_GPU_OWNER" = unknown ] || exit 1 + fm_resgate_home_gpu_owner || exit 1 + [ "$FM_RESGATE_GPU_OWNER" = qwen ] || exit 1 + exit 0 + ) || fail "an unusable threshold or port must fail the GPU reading closed, not read the card as free" + pass "an unusable GPU threshold or voice port fails the reading closed to unknown" +} + +# --- GPU probe script: executed by its real consumer (PowerShell) -------------- +# +# fm_resgate_home_gpu_probe_cmd emits a PowerShell program that runs on the home +# host; its error discrimination (which failures mean "nothing matched" and which +# mean "the measurement itself failed") is only meaningful when PowerShell itself +# evaluates it, so these run the generated program under pwsh with the failure +# modes stubbed in, rather than inspecting its text. + +run_probe_under_pwsh() { # -> probe stdout + local prelude=$1 tmp script + tmp=$(fm_test_tmproot resgate-probe-pwsh) + script="$tmp/probe.ps1" + { + printf '%s\n' "$prelude" + fm_resgate_home_gpu_probe_cmd 7414 definitely-no-such-proc-xyz + } > "$script" + pwsh -NoProfile -NonInteractive -File "$script" 2>/dev/null +} + +test_probe_script_reports_port_probe_failure_distinctly() { + local out + command -v pwsh > /dev/null 2>&1 || { echo "skip - pwsh not installed"; return 0; } + + # No Get-NetTCPConnection at all: the NetTCPIP module is unavailable, the + # real-world failure this must not report as a free port. + out=$(run_probe_under_pwsh '') + case "$out" in + *voice_port=probe-failed*) ;; + *) fail "a missing Get-NetTCPConnection must report probe-failed, got: $out" ;; + esac + + # The CIM/WMI path itself errors while the gateway may well be listening. + out=$(run_probe_under_pwsh \ + "function Get-NetTCPConnection { [CmdletBinding()] param([int]\$LocalPort,[string]\$State) throw 'CIM server unavailable' }") + case "$out" in + *voice_port=probe-failed*) ;; + *) fail "a failing CIM query must report probe-failed, got: $out" ;; + esac + + # Genuinely nothing bound to the port: the one error identity that really + # does mean "no match" must still read as a negative, not a failure. + out=$(run_probe_under_pwsh \ + "function Get-NetTCPConnection { [CmdletBinding()] param([int]\$LocalPort,[string]\$State) Write-Error -Message 'no match' -ErrorId 'CmdletizationQuery_NotFound_LocalPort' -Category ObjectNotFound }") + case "$out" in + *voice_port=not-listening*) ;; + *) fail "a genuine no-match must still read as not-listening, got: $out" ;; + esac + + out=$(run_probe_under_pwsh \ + "function Get-NetTCPConnection { [CmdletBinding()] param([int]\$LocalPort,[string]\$State) [pscustomobject]@{ LocalPort = \$LocalPort } }") + case "$out" in + *voice_port=listening*) ;; + *) fail "a returned connection must read as listening, got: $out" ;; + esac + pass "the generated probe separates a failed port measurement from a genuinely free port" +} + +test_probe_script_reports_process_reading_from_real_cmdlet() { + local out + command -v pwsh > /dev/null 2>&1 || { echo "skip - pwsh not installed"; return 0; } + + # Real Get-Process, real absent process: must be the negative reading, not a + # failure - otherwise every ordinary idle host would read unknown forever. + out=$(run_probe_under_pwsh '') + case "$out" in + *gpu_process=not-running*) ;; + *) fail "a genuinely absent process must read not-running, got: $out" ;; + esac + + out=$(run_probe_under_pwsh \ + "function Get-Process { [CmdletBinding()] param([string]\$Name) throw 'process enumeration failed' }") + case "$out" in + *gpu_process=probe-failed*) ;; + *) fail "a failing process enumeration must report probe-failed, got: $out" ;; + esac + pass "the generated probe separates a failed process measurement from a genuinely absent process" +} + +test_gpu_owner_unknown_when_ssh_unreachable() { + local tmp fakebin + tmp=$(fm_test_tmproot resgate-gpu-unreachable) + fakebin=$(fm_fakebin "$tmp") + fake_ssh_unreachable "$fakebin" + # shellcheck disable=SC2030,SC2031 + ( PATH="$fakebin:$PATH" + . "$ROOT/bin/fm-resgate-lib.sh" + fm_resgate_home_gpu_owner && exit 1 + [ "$FM_RESGATE_GPU_OWNER" = unknown ] || exit 1 + exit 0 + ) || fail "an unreachable home host must fail closed to unknown, never permissive" + pass "GPU owner fails closed to unknown when the home host is unreachable" +} + +test_gpu_skip_remote_never_probes() { + # shellcheck disable=SC2030,SC2031 + ( FM_RESGATE_SKIP_REMOTE=1 + fm_resgate_home_gpu_owner && exit 1 + [ "$FM_RESGATE_GPU_OWNER" = unknown ] || exit 1 + exit 0 + ) || fail "FM_RESGATE_SKIP_REMOTE=1 must fail closed without attempting a probe" + pass "FM_RESGATE_SKIP_REMOTE=1 fails closed without probing" +} + +# --- CLI (public interface) ----------------------------------------------------- + +test_cli_schedule_and_cap() { + local out + out=$(FM_RESGATE_NOW_OVERRIDE="3 14 00" "$CLI" schedule work) \ + || fail "CLI schedule must exit 0" + case "$out" in *state=capped*) ;; *) fail "CLI schedule must print state=capped: $out" ;; esac + out=$(FM_STATE_OVERRIDE="$(fm_test_tmproot resgate-cli-cap)" \ + FM_RESGATE_NOW_OVERRIDE="3 14 00" "$CLI" cap work) \ + || fail "CLI cap must exit 0" + case "$out" in *pct=50*) ;; *) fail "CLI cap must print pct=50: $out" ;; esac + pass "CLI schedule/cap commands print the expected verdict" +} + +test_cli_override_both_arms_and_clears_two_files() { + local state + state=$(fm_test_tmproot resgate-cli-override-both) + FM_STATE_OVERRIDE="$state" "$CLI" override set both > /dev/null \ + || fail "CLI override set both must exit 0" + [ -e "$state/.resgate-cap-work" ] || fail "override set both must arm the work marker" + [ -e "$state/.resgate-cap-home" ] || fail "override set both must arm the home marker" + FM_STATE_OVERRIDE="$state" "$CLI" override clear both > /dev/null \ + || fail "CLI override clear both must exit 0" + [ -e "$state/.resgate-cap-work" ] && fail "override clear both must remove the work marker" + [ -e "$state/.resgate-cap-home" ] && fail "override clear both must remove the home marker" + pass "CLI override set/clear both arms and releases the markers for both roles" +} + +test_cli_override_both_reports_each_role_when_only_one_applies() { + local state out rc=0 + state=$(fm_test_tmproot resgate-cli-override-partial) + FM_STATE_OVERRIDE="$state" "$CLI" override set both > /dev/null \ + || fail "arming both roles is this test's precondition" + # Stage a marker that cannot be released: `rm -f` refuses a directory, so + # home's clear fails while work's succeeds - the half-applied `both` run + # the captain has to be able to read off the output. + rm -f "$state/.resgate-cap-home" + mkdir "$state/.resgate-cap-home" || fail "could not stage an unremovable home marker" + + out=$(FM_STATE_OVERRIDE="$state" "$CLI" override clear both 2>&1) || rc=$? + [ "$rc" -ne 0 ] || fail "a half-applied clear must exit non-zero, got 0: $out" + case "$out" in + *work=clear*) ;; + *) fail "the released host must be reported as clear, got: $out" ;; + esac + case "$out" in + *home=armed*) ;; + *) fail "the host that stayed capped must be reported as armed, got: $out" ;; + esac + fm_resgate_override_active "$state" work \ + && fail "work's marker must really be gone after the reported release" + pass "a half-applied override both reports each host's real state, not only the failure" +} + +test_cli_gpu_allow_exit_codes() { + local tmp fakebin out + tmp=$(fm_test_tmproot resgate-cli-gpu) + fakebin=$(fm_fakebin "$tmp") + out='FM_RESGATE voice_port=not-listening\r\nFM_RESGATE gpu_process=running\r\nFM_RESGATE gpu_used_mb=9046\r\n' + fake_ssh_returning "$fakebin" "$out" + # shellcheck disable=SC2031 + PATH="$fakebin:$PATH" "$CLI" gpu allow qwen \ + || fail "CLI gpu allow qwen must exit 0 when Qwen holds the GPU" + # shellcheck disable=SC2031 + PATH="$fakebin:$PATH" "$CLI" gpu allow voice \ + && fail "CLI gpu allow voice must exit non-zero when Qwen holds the GPU" + pass "CLI gpu allow exit codes reflect the exclusivity decision" +} + +test_role_ok +test_now_fields_from_override +test_now_fields_rejects_malformed_override +test_now_fields_handles_leading_zero_hours +test_work_capped_within_window +test_work_capped_at_start_boundary +test_work_uncapped_at_end_boundary +test_work_free_through_weekend_span +test_home_free_within_window +test_home_free_starts_at_boundary +test_home_free_ends_at_boundary +test_home_capped_all_weekend +test_schedule_blocked_on_unreadable_clock +test_capacity_pct_zero_on_unreadable_clock +test_schedule_blocked_when_berlin_zone_did_not_resolve +test_schedule_blocked_when_zone_and_offset_disagree +test_schedule_reads_a_genuinely_resolved_berlin_clock +test_override_set_active_clear_roundtrip +test_override_forces_capped_regardless_of_schedule +test_blocked_clock_beats_armed_override +test_override_clear_reports_failure_when_the_marker_survives +test_apply_pct_arithmetic +test_apply_pct_fails_closed_on_bad_input +test_gpu_owner_qwen_when_process_and_memory_both_active +test_gpu_owner_none_when_process_running_but_memory_idle +test_gpu_owner_voice_when_port_listening +test_gpu_owner_voice_wins_over_resident_qwen_signal +test_gpu_available_for_blocks_the_other_side +test_gpu_owner_unknown_on_crlf_lines_still_parses_correctly +test_gpu_owner_unknown_on_probe_failure_field +test_gpu_owner_unknown_on_voice_port_probe_failure +test_gpu_owner_unknown_on_process_probe_failure +test_gpu_owner_unknown_on_unusable_tunables +test_gpu_owner_unknown_on_unusable_ssh_timeout +test_probe_script_reports_port_probe_failure_distinctly +test_probe_script_reports_process_reading_from_real_cmdlet +test_gpu_owner_unknown_when_ssh_unreachable +test_gpu_skip_remote_never_probes +test_cli_schedule_and_cap +test_cli_override_both_arms_and_clears_two_files +test_cli_override_both_reports_each_role_when_only_one_applies +test_cli_gpu_allow_exit_codes diff --git a/tests/fm-test-run.test.sh b/tests/fm-test-run.test.sh index 79a847c372..ef9e27a4af 100755 --- a/tests/fm-test-run.test.sh +++ b/tests/fm-test-run.test.sh @@ -25,8 +25,8 @@ test_list_all_exact_suite_coverage() { done | LC_ALL=C sort ) [ -n "$listed" ] || fail "--list --all printed nothing" - missing=$(comm -23 <(printf '%s\n' "$expected") <(printf '%s\n' "$listed") || true) - extra=$(comm -13 <(printf '%s\n' "$expected") <(printf '%s\n' "$listed") || true) + missing=$(LC_ALL=C comm -23 <(printf '%s\n' "$expected") <(printf '%s\n' "$listed") || true) + extra=$(LC_ALL=C comm -13 <(printf '%s\n' "$expected") <(printf '%s\n' "$listed") || true) [ -z "$missing" ] || fail "--list --all missing scripts: $missing" [ -z "$extra" ] || fail "--list --all unexpected scripts: $extra" # No duplicates. @@ -639,7 +639,7 @@ test_portable_shard_union_and_coverage_guard() { herdr=$("$RUNNER" --list --family real-herdr-gated) [ -n "$s1" ] && [ -n "$s2" ] || fail "portable parallel shards must be non-empty" # Shards disjoint. - overlap=$(comm -12 <(printf '%s\n' "$s1" | LC_ALL=C sort) <(printf '%s\n' "$s2" | LC_ALL=C sort) || true) + overlap=$(LC_ALL=C comm -12 <(printf '%s\n' "$s1" | LC_ALL=C sort) <(printf '%s\n' "$s2" | LC_ALL=C sort) || true) [ -z "$overlap" ] || fail "portable parallel shards overlap: $overlap" # Union of shards equals proven-isolated. [ "$(printf '%s\n' "$s1" "$s2" | LC_ALL=C sort -u)" = \