diff --git a/.gitignore b/.gitignore index 90a7b4b27..e1f449841 100644 --- a/.gitignore +++ b/.gitignore @@ -60,10 +60,12 @@ htmlcov/ # Skill run results (large, regenerated by agent) .cursor/skills/*/results/ -# PR description scratchpad. Tools (gh CLI, codex) sometimes drop a -# pr.md at the repo root when drafting a pull request; the -# description belongs in the PR body, not in the tree. +# PR description scratchpads and local review notes. pr.md +pr_*.md +Specialist.discuss.md +specialist.discuss*.md +specialist.info.md # F0/F1 integration: keep test baselines, dry-run merge maps, and # archived regression sweeps tracked. diff --git a/src/hyperloom/agents/kernel/scripts/install.sh b/src/hyperloom/agents/kernel/scripts/install.sh index d527d5e07..4e9ccc94c 100644 --- a/src/hyperloom/agents/kernel/scripts/install.sh +++ b/src/hyperloom/agents/kernel/scripts/install.sh @@ -223,6 +223,19 @@ TRACELENS_MIRROR_DIR="${TRACELENS_MIRROR_DIR:-${_open_source_root}/TraceLens-int # missing from env, source $REPO_ROOT/.env but protect already-set values. REPO_ROOT="${REPO_ROOT:-$(pwd)}" DOTENV="${REPO_ROOT}/.env" + +# Vars whose already-resolved value must survive the `. $REPO_ROOT/.env` below. +# The dotenv is a *credentials* fallback, but it is a full env file: sourcing it +# under `set -a` re-imports every path var it happens to carry. A stale entry — +# or one written by a DIFFERENT checkout / workspace sharing this .env — then +# silently overwrites what the caller resolved, and the run installs against one +# workspace while writing its env file into another (#Hyperloom-2 / atom mixup). +# Only non-empty pre-source values are restored, so the dotenv still fills gaps. +_DOTENV_PROTECTED_VARS='REPO_ROOT KERNEL_AGENT_ROOT HYPERLOOM_KERNEL_AGENT_ROOT +USER_DATA_PATH HYPERLOOM_RUNTIME_DIR KERNEL_AGENT_ENV HYPERLOOM_ROOT +MAGPIE_PATH INFERENCEX_PATH TRACELENS_ROOT TRACELENS_INTERNAL_ROOT +GEAK_ROOT GEAK_E2E_RUNNER PYTHONPATH' + if [ -z "${ANTHROPIC_BASE_URL:-}" ] || [ -z "${ANTHROPIC_API_KEY:-}" ] \ || [ -z "${ANTHROPIC_AUTH_TOKEN:-}" ] || [ -z "${DEEPSEEK_API_KEY:-}" ] \ || [ -z "${DEEPSEEK_BASE_URL:-}" ]; then @@ -232,6 +245,9 @@ if [ -z "${ANTHROPIC_BASE_URL:-}" ] || [ -z "${ANTHROPIC_API_KEY:-}" ] \ _snap_anthropic_token="${ANTHROPIC_AUTH_TOKEN-}" _snap_deepseek_key="${DEEPSEEK_API_KEY-}" _snap_deepseek_url="${DEEPSEEK_BASE_URL-}" + for _v in $_DOTENV_PROTECTED_VARS; do + eval "_snap_prot_${_v}=\"\${${_v}-}\"" + done set -a # shellcheck disable=SC1091 . "$REPO_ROOT/.env" @@ -241,6 +257,18 @@ if [ -z "${ANTHROPIC_BASE_URL:-}" ] || [ -z "${ANTHROPIC_API_KEY:-}" ] \ [ -n "$_snap_anthropic_token" ] && export ANTHROPIC_AUTH_TOKEN="$_snap_anthropic_token" [ -n "$_snap_deepseek_key" ] && export DEEPSEEK_API_KEY="$_snap_deepseek_key" [ -n "$_snap_deepseek_url" ] && export DEEPSEEK_BASE_URL="$_snap_deepseek_url" + for _v in $_DOTENV_PROTECTED_VARS; do + eval "_snap_val=\"\${_snap_prot_${_v}-}\"" + if [ -n "${_snap_val}" ]; then + eval "_cur_val=\"\${${_v}-}\"" + if [ "${_cur_val}" != "${_snap_val}" ]; then + echo "[kernel-agent] .env would have overridden ${_v}=${_snap_val} with ${_cur_val}; keeping the caller's value" >&2 + fi + eval "export ${_v}=\"\${_snap_prot_${_v}}\"" + fi + unset "_snap_prot_${_v}" + done + unset _v _snap_val _cur_val unset _snap_anthropic_url _snap_anthropic_key _snap_anthropic_token unset _snap_deepseek_key _snap_deepseek_url echo "[kernel-agent] loaded credentials fallback from $REPO_ROOT/.env (env wins)" diff --git a/src/hyperloom/agents/robustness/role/envelope.py b/src/hyperloom/agents/robustness/role/envelope.py index 8799cc66d..36cc5aa81 100644 --- a/src/hyperloom/agents/robustness/role/envelope.py +++ b/src/hyperloom/agents/robustness/role/envelope.py @@ -38,6 +38,8 @@ class IntentType(str, Enum): RESPONSE = "response" REVIEW_VERDICT = "review_verdict" KILL_TASK = "kill_task" + # Robustness never emits this; kept in the mirror for the contract test. + EXTEND_LEASE = "extend_lease" PRUNE_BRANCH = "prune_branch" ESCALATE_STRATEGY_CHANGE = "escalate_strategy_change" # Robustness never emits this; kept in the mirror for the contract test. @@ -739,14 +741,15 @@ class IntentSpec: # Required-field map for intents robustness never emits but the upstream # contract test still diffs against. No builder/validator: they are here only # to keep :data:`PAYLOAD_REQUIRED` byte-equal with upstream ``_PAYLOAD_REQUIRED``. -# ``SPECIALIST_DONE`` is the specialist exit envelope (PolicyGate R3 validates -# it); ``REVIEW_VERDICT`` enforces only the structural ``target_proposal_msg_id`` -# here (verdict/verdict_map mutual exclusion lives in upstream policy). +# ``SPECIALIST_DONE`` is the specialist exit envelope; ``REVIEW_VERDICT`` +# enforces only the structural ``target_proposal_msg_id`` here +# (verdict/verdict_map mutual exclusion lives in upstream policy). _REQUIRED_ONLY: Mapping[IntentType, tuple[str, ...]] = { IntentType.PROPOSE_ACTION: ("action_name", "predicted_gain_pct"), IntentType.REQUEST: ("target_agent", "kind"), IntentType.RESPONSE: ("in_reply_to", "kind"), IntentType.REVIEW_VERDICT: ("target_proposal_msg_id",), + IntentType.EXTEND_LEASE: ("task_id", "extra_sec"), IntentType.SPECIALIST_DONE: ( "gap_canonical_id", "domain", diff --git a/src/hyperloom/inference_optimizer/SKILL.md b/src/hyperloom/inference_optimizer/SKILL.md index c92c354da..efabae787 100644 --- a/src/hyperloom/inference_optimizer/SKILL.md +++ b/src/hyperloom/inference_optimizer/SKILL.md @@ -400,13 +400,18 @@ Steps for the launching agent: lightweight classify from the model's local `config.json` (decoder type, attention variant, expert counts, MTP, SWA window) and set `"source": "config_classify"`. -3. **Write the convention file** — once `session_dir` exists, write the - profile to exactly `/model_arch.json`: the file named - `model_arch.json` in the current session directory. Do not write it to - `$USER_DATA_PATH/model_arch.json`; that path is shared by concurrent - sessions and can be overwritten by another launch. Include `model_name` - (required for the stale-file guard). Set it to the **clean model name** - (e.g. `Qwen2.5-7B-Instruct`); +3. **Write the profile BEFORE launch and point `$HYPERLOOM_MODEL_ARCH_FILE` + at it.** The CLI creates `session_dir` and reads the profile in the *same + process*, so a file written to `/model_arch.json` after the + session dir appears always loses the race — the run seeds + `state.model_arch={}` and the profile never reaches any prompt. Write the + JSON to a launcher-owned path instead and export + `HYPERLOOM_MODEL_ARCH_FILE=` in the shell that spawns + `optimize`; the CLI copies it into `/model_arch.json` for + provenance. Use a **session-unique** filename — `$USER_DATA_PATH` is shared + by concurrent sessions on WekaFS, so a fixed name races other launches. + Include `model_name` (required for the stale-file guard). Set it to the + **clean model name** (e.g. `Qwen2.5-7B-Instruct`); the guard normalizes launch forms — flat dirs, HF repo ids, and HF hub cache `models--org--repo/snapshots/` paths — so do NOT use the snapshot commit hash. All other fields are optional; renderers drop diff --git a/src/hyperloom/inference_optimizer/cli/model_gate.py b/src/hyperloom/inference_optimizer/cli/model_gate.py index 83296137d..56429058d 100644 --- a/src/hyperloom/inference_optimizer/cli/model_gate.py +++ b/src/hyperloom/inference_optimizer/cli/model_gate.py @@ -311,6 +311,37 @@ def _resolve_amd_gpu_type(explicit: str | None = None) -> str | None: _MLX_QUANT_MODES = frozenset({"affine", "mlx"}) +def _read_preseeded_model_arch(arch_path: Path) -> str | None: + """Return the pre-seeded ``$HYPERLOOM_MODEL_ARCH_FILE`` text, or ``None``. + + Copies the source into ``arch_path`` so the session keeps its own copy of + the profile the run was seeded with. A failed copy is not fatal — the text + is still returned and the profile still reaches the prompts. + + Args: + arch_path (Path): The in-session ``model_arch.json`` destination. + + Returns: + str | None: The profile JSON text, or ``None`` when the env is unset or + the file is unreadable. + """ + src = (os.environ.get("HYPERLOOM_MODEL_ARCH_FILE") or "").strip() + if not src: + return None + try: + text = Path(src).expanduser().read_text(encoding="utf-8") + except OSError as exc: + logging.warning("model_arch_preseed_unreadable: %s (%s)", src, exc) + return None + try: + arch_path.parent.mkdir(parents=True, exist_ok=True) + arch_path.write_text(text, encoding="utf-8") + except OSError as exc: + logging.warning("model_arch_preseed_copy_failed: %s -> %s (%s)", src, arch_path, exc) + logging.info("model_arch_preseeded_from: %s", src) + return text + + def _load_model_arch( workspace_root: Path, model_name: str, @@ -325,6 +356,13 @@ def _load_model_arch( ``models--org--repo/snapshots/`` paths so a declared clean name still matches a commit-hash launch basename. + The session dir is created and seeded in the same CLI process, so a + launcher following SKILL Step 1.5 ("write it once ``session_dir`` exists") + can never win that race. ``$HYPERLOOM_MODEL_ARCH_FILE`` is the pre-launch + escape hatch: point it at a profile written *before* launch and it is used + when the in-session file is absent, then copied into the session dir so the + run keeps its own provenance copy. + Args: workspace_root (Path): Directory containing ``model_arch.json``. model_name (str): The resolved model identity (display name / basename). @@ -341,7 +379,9 @@ def _load_model_arch( try: raw = arch_path.read_text(encoding="utf-8") except FileNotFoundError: - return {} + raw = _read_preseeded_model_arch(arch_path) + if raw is None: + return {} except OSError as exc: logging.warning("model_arch_unreadable: %s (%s)", arch_path, exc) return {} diff --git a/src/hyperloom/inference_optimizer/protocol/intent.py b/src/hyperloom/inference_optimizer/protocol/intent.py index 7d68aa899..3911a5d7b 100644 --- a/src/hyperloom/inference_optimizer/protocol/intent.py +++ b/src/hyperloom/inference_optimizer/protocol/intent.py @@ -33,7 +33,8 @@ class IntentType(str, Enum): REQUEST = "request" RESPONSE = "response" REVIEW_VERDICT = "review_verdict" # Critic-only - KILL_TASK = "kill_task" # Robustness-only; payload.scope must be "task" + KILL_TASK = "kill_task" # payload.scope must be "task" + EXTEND_LEASE = "extend_lease" # refresh a live task's lease TTL # Robustness-only scheduling police. PRUNE_BRANCH = "prune_branch" ESCALATE_STRATEGY_CHANGE = "escalate_strategy_change" @@ -61,9 +62,10 @@ class Intent: # verdict/verdict_map mutual exclusion enforced by _validate_review_verdict_payload. IntentType.REVIEW_VERDICT: ("target_proposal_msg_id",), IntentType.KILL_TASK: ("task_id", "reason"), + IntentType.EXTEND_LEASE: ("task_id", "extra_sec"), IntentType.PRUNE_BRANCH: ("family", "reason"), IntentType.ESCALATE_STRATEGY_CHANGE: ("reason", "next_action_hint"), - # specialist exit envelope; per-variant schema enforced by PolicyGate R3. + # specialist exit envelope; the runner re-stamps and defaults the payload. IntentType.SPECIALIST_DONE: ("gap_canonical_id", "domain", "proposal_set", "empty", "summary"), } diff --git a/src/hyperloom/inference_optimizer/tests/test_agent_roles_and_policy.py b/src/hyperloom/inference_optimizer/tests/test_agent_roles_and_policy.py index dcb587574..45b4e0c15 100644 --- a/src/hyperloom/inference_optimizer/tests/test_agent_roles_and_policy.py +++ b/src/hyperloom/inference_optimizer/tests/test_agent_roles_and_policy.py @@ -59,7 +59,7 @@ def test_orchestration_permissions(): assert IntentType.PRUNE_BRANCH in role.allowed_intents assert IntentType.ESCALATE_STRATEGY_CHANGE in role.allowed_intents assert IntentType.REVIEW_VERDICT not in role.allowed_intents - assert IntentType.KILL_TASK not in role.allowed_intents + assert IntentType.KILL_TASK in role.allowed_intents assert IntentType.RESPONSE not in role.allowed_intents @@ -123,7 +123,7 @@ def test_review_verdict_critic_only(): def test_kill_and_robustness_only_renamed(): - assert KILL_TASK_SOURCE_ALLOWLIST == frozenset({"robustness"}) + assert KILL_TASK_SOURCE_ALLOWLIST == frozenset({"robustness", "orchestration"}) assert ROBUSTNESS_ONLY_SOURCE_ALLOWLIST == frozenset({"robustness"}) assert ROBUSTNESS_ONLY_INTENTS == frozenset( { @@ -527,16 +527,26 @@ def test_gate_robustness_kill_task_ok(gate): ) -def test_gate_orchestration_kill_task_rejected(gate): +def test_gate_orchestration_kill_task_allowed(gate): + gate.validate_intent( + "orchestration", + Intent( + type=IntentType.KILL_TASK, + payload={"task_id": "t1", "reason": "stalled"}, + ), + ) + + +def test_gate_orchestration_kill_task_process_scope_rejected(gate): with pytest.raises(PolicyDenied) as exc: gate.validate_intent( "orchestration", Intent( type=IntentType.KILL_TASK, - payload={"task_id": "t1", "reason": "stalled"}, + payload={"task_id": "t1", "reason": "stalled", "scope": "process"}, ), ) - assert exc.value.rule == "role" + assert exc.value.rule == "kill_scope" def test_gate_robustness_kill_task_process_scope_rejected(gate): diff --git a/src/hyperloom/inference_optimizer/tests/test_common_utils.py b/src/hyperloom/inference_optimizer/tests/test_common_utils.py index 4dfbdbb5b..4d8d26358 100644 --- a/src/hyperloom/inference_optimizer/tests/test_common_utils.py +++ b/src/hyperloom/inference_optimizer/tests/test_common_utils.py @@ -2429,26 +2429,6 @@ def test_llm_prompt_parse_response_edges() -> None: assert parse_llm_response("[]") == {"executive_summary": "", "section_narratives": {}} -# --------------------------------------------------------------------------- -# orchestrator.state.orchestration_memory.deterministic_memory_fallback -# --------------------------------------------------------------------------- - - -def test_deterministic_memory_fallback_bad_gain() -> None: - from hyperloom.orchestrator.state.orchestration_memory import deterministic_memory_fallback - - state = SimpleNamespace( - current_best={"tput": 123.0}, - optimization_stack=[], - cumulative_gain_validated="not-a-number", - phase="EXPLORE", - macro_cycle=2, - ) - record = deterministic_memory_fallback(state) - assert "current_plan" in record - assert "phase=EXPLORE" in record["current_plan"] - - # --------------------------------------------------------------------------- # orchestrator.specialists.profile # --------------------------------------------------------------------------- diff --git a/src/hyperloom/inference_optimizer/tests/test_coordinator_async_batch2_unit.py b/src/hyperloom/inference_optimizer/tests/test_coordinator_async_batch2_unit.py index 49440398f..ec9337046 100644 --- a/src/hyperloom/inference_optimizer/tests/test_coordinator_async_batch2_unit.py +++ b/src/hyperloom/inference_optimizer/tests/test_coordinator_async_batch2_unit.py @@ -800,9 +800,6 @@ class _Policy: def should_checkpoint(self, **kw): return False - def is_hard_compaction(self, n): - return False - coord._checkpoint_policy = _Policy() assert await coord._maybe_checkpoint_orchestration(tick=5) is False @@ -820,9 +817,6 @@ class _Policy: def should_checkpoint(self, **kw): return True - def is_hard_compaction(self, n): - return False - coord._checkpoint_policy = _Policy() took = await coord._maybe_checkpoint_orchestration(tick=12, phase_changed=True) assert took is True @@ -863,7 +857,6 @@ async def _run(**kw): def test_checkpoint_policy_context_fraction_env(tmp_path, monkeypatch) -> None: monkeypatch.setenv("USER_DATA_PATH", str(tmp_path)) monkeypatch.setenv("INFERENCE_OPTIMIZER_CTX_SOFT_FRACTION", "0.5") - monkeypatch.setenv("INFERENCE_OPTIMIZER_CTX_HARD_FRACTION", "0.75") sd = make_session_dir() from .conftest import seed_target_analysis_marker @@ -872,7 +865,6 @@ def test_checkpoint_policy_context_fraction_env(tmp_path, monkeypatch) -> None: backends["orchestration"].model = "claude-opus-4-8" # type: ignore[attr-defined] c = Coordinator(sd, backends=backends) assert c._checkpoint_policy.context_token_soft == 100_000 - assert c._checkpoint_policy.context_token_hard == 150_000 def test_orchestration_memory_rollback_env(tmp_path, monkeypatch) -> None: @@ -901,9 +893,6 @@ class _Policy: def should_checkpoint(self, **kw): return True - def is_hard_compaction(self, n): # default: not hard - return False - coord._checkpoint_policy = _Policy() @@ -947,30 +936,6 @@ async def test_checkpoint_degenerate_three_times_emits_medium_observation(coord: assert any(p.get("severity") == "medium" and p.get("consecutive") == 3 for p in degraded) -@pytest.mark.asyncio -async def test_checkpoint_degenerate_hard_uses_fallback(coord: Coordinator) -> None: - import time - - _make_conversational(coord, raw_text="still no JSON") - coord._checkpoint_enabled = True - coord._orchestration_seeded = True - coord._run_started_monotonic = time.monotonic() - coord.shared_state.phase = "EXPLORE" - - class _Policy: - def should_checkpoint(self, **kw): - return True - - def is_hard_compaction(self, n): # near the window → force - return True - - coord._checkpoint_policy = _Policy() - took = await coord._maybe_checkpoint_orchestration(tick=20) - assert took is True - assert coord._orchestration_seeded is False - assert coord.shared_state.orchestration_memory.get("current_plan", "").startswith("[auto]") - - @pytest.mark.asyncio async def test_checkpoint_failure_resets_tracker(coord: Coordinator) -> None: import time @@ -1109,35 +1074,10 @@ def _boom(*a, **k): raise RuntimeError("telemetry failed") monkeypatch.setattr(coord.shared_state, "to_phase_budget_telemetry", _boom) - - async def _scan_boom(): - raise RuntimeError("scan failed") - - monkeypatch.setattr(coord.phase_explore, "_scan_stale_specialists", _scan_boom) monkeypatch.setattr(coord.conversation, "_conversation_progress_signal", _boom) out = await coord._compose_prompt("robustness") - assert "Specialist health" in out - - -@pytest.mark.asyncio -async def test_compose_prompt_robustness_lists_stale_specialists( - coord: Coordinator, - monkeypatch, -) -> None: - async def _stale(): - return [{"task_id": "spec-stale", "running_seconds": 999}] - - monkeypatch.setattr(coord.phase_explore, "_scan_stale_specialists", _stale) - out = await coord._compose_prompt("robustness") - assert "stale specialists" in out - assert "spec-stale" in out - - -# -- _promote_to_shared_state additional branches --------------------------- -def _ptask(tid: str, kind: str): - from hyperloom.orchestrator.state.task_registry import Task - - return Task(task_id=tid, kind=kind, state="running", params={}, idempotency_key=f"{tid}-k") + # Every advisory failure is swallowed; the prompt still renders. + assert "SESSION_DIR=" in out @pytest.mark.asyncio @@ -1355,26 +1295,47 @@ async def test_budget_limited_conc_sweep_skip_records_done(coord: Coordinator) - assert evidence["conc_sweep_status"] == "skipped" -# -- _scan_stale_specialists (running rows) --------------------------------- +# -- _promote_to_shared_state additional branches --------------------------- +def _ptask(tid: str, kind: str): + from hyperloom.orchestrator.state.task_registry import Task + + return Task(task_id=tid, kind=kind, state="running", params={}, idempotency_key=f"{tid}-k") + + +# -- specialist visibility contract ----------------------------------------- @pytest.mark.asyncio -async def test_scan_stale_specialists_flags_running(coord: Coordinator) -> None: - coord._specialist_stale_sec = 0 # any running specialist is "stale" +async def test_compose_prompt_has_no_specialist_status_block(coord: Coordinator) -> None: + """No periodic specialist block: it can never observe a live specialist. + + The prompt renders only between blocking actions, so a running specialist + is structurally absent from it. Reporting "none running" there would + manufacture a false belief; in-flight work reaches the agent via + ``specialist_progress`` observations and ``get_running_tasks`` instead. + """ spec = await coord.tasks.create( kind="specialist", - params={}, - idempotency_key="stale-spec", + params={"domain": "serving_specialist"}, + idempotency_key="visible-spec", ) await coord.tasks.transition(spec.task_id, "running") - # A non-specialist running task is skipped by the kind guard. - other = await coord.tasks.create( - kind="explore", - params={}, - idempotency_key="stale-other", + for agent in ("orchestration", "robustness"): + out = await coord._compose_prompt(agent) + assert "Specialist health" not in out + assert "stale" not in out.lower() + + +@pytest.mark.asyncio +async def test_running_tasks_reader_sees_live_specialist(coord: Coordinator) -> None: + """``get_running_tasks`` is the on-demand path and is not turn-bound.""" + spec = await coord.tasks.create( + kind="specialist", + params={"domain": "serving_specialist", "gap_canonical_id": "gap.xyz"}, + idempotency_key="queryable-spec", ) - await coord.tasks.transition(other.task_id, "running") - stale = await coord._scan_stale_specialists() - assert any(r["task_id"] == spec.task_id for r in stale) - assert all(r["task_id"] != other.task_id for r in stale) + await coord.tasks.transition(spec.task_id, "running") + out = coord.conversation._context_running_tasks_reader() + assert spec.task_id in out + assert "serving_specialist" in out # -- _fan_out_specialist_wave (valid entries) ------------------------------- @@ -1712,7 +1673,6 @@ async def test_handle_intent_routes_rare_types(coord: Coordinator, monkeypatch) IntentType.PRUNE_BRANCH: "_handle_prune_branch", IntentType.ALERT: "_handle_alert", IntentType.UPDATE_STATE: "_handle_update_state", - IntentType.SPECIALIST_DONE: "_handle_specialist_done", } for it, attr in routes.items(): diff --git a/src/hyperloom/inference_optimizer/tests/test_coordinator_async_methods_coverage_unit.py b/src/hyperloom/inference_optimizer/tests/test_coordinator_async_methods_coverage_unit.py index 763cb8d22..203560e7f 100644 --- a/src/hyperloom/inference_optimizer/tests/test_coordinator_async_methods_coverage_unit.py +++ b/src/hyperloom/inference_optimizer/tests/test_coordinator_async_methods_coverage_unit.py @@ -445,12 +445,6 @@ async def test_escalate_skip_to_close_allowed_after_enablement(coord: Coordinato assert coord.shared_state.pending_escalate_hint == "skip_to_close" -# -- _scan_stale_specialists ----------------------------------------------- -@pytest.mark.asyncio -async def test_scan_stale_specialists_empty(coord: Coordinator) -> None: - assert await coord._scan_stale_specialists() == [] - - # -- _maybe_autosubmit_specialist_patches ---------------------------------- @pytest.mark.asyncio async def test_autosubmit_skipped_when_no_patches(coord: Coordinator) -> None: diff --git a/src/hyperloom/inference_optimizer/tests/test_coordinator_runtime.py b/src/hyperloom/inference_optimizer/tests/test_coordinator_runtime.py index 436b35303..f29e4c9b7 100644 --- a/src/hyperloom/inference_optimizer/tests/test_coordinator_runtime.py +++ b/src/hyperloom/inference_optimizer/tests/test_coordinator_runtime.py @@ -1611,9 +1611,9 @@ async def runner(ctx): assert ran["called"] is True assert res.state == "succeeded" assert res.result == {"tput": 1.0} - assert any( - "vanished" in rec.message.lower() and "_transition_resilient" in rec.message for rec in caplog.records - ), "expected the disappearing-row warning to fire" + assert any("vanished" in rec.message.lower() for rec in caplog.records), ( + "expected the disappearing-row warning to fire" + ) db.close() diff --git a/src/hyperloom/inference_optimizer/tests/test_coverage_boost2_unit.py b/src/hyperloom/inference_optimizer/tests/test_coverage_boost2_unit.py index 5796b3b95..33d021e70 100644 --- a/src/hyperloom/inference_optimizer/tests/test_coverage_boost2_unit.py +++ b/src/hyperloom/inference_optimizer/tests/test_coverage_boost2_unit.py @@ -9,24 +9,6 @@ from types import SimpleNamespace -# --------------------------------------------------------------------------- # -# orchestrator.orchestration_memory.deterministic_memory_fallback # -# --------------------------------------------------------------------------- # -def test_deterministic_memory_fallback_bad_gain() -> None: - from hyperloom.orchestrator.state.orchestration_memory import deterministic_memory_fallback - - state = SimpleNamespace( - current_best={"tput": 123.0}, - optimization_stack=[], - cumulative_gain_validated="not-a-number", # triggers except - phase="EXPLORE", - macro_cycle=2, - ) - record = deterministic_memory_fallback(state) - assert "current_plan" in record - assert "phase=EXPLORE" in record["current_plan"] - - # --------------------------------------------------------------------------- # # orchestrator.specialist_profile # # --------------------------------------------------------------------------- # diff --git a/src/hyperloom/inference_optimizer/tests/test_critic_verdict_map.py b/src/hyperloom/inference_optimizer/tests/test_critic_verdict_map.py index 828ca5c68..8126a4340 100644 --- a/src/hyperloom/inference_optimizer/tests/test_critic_verdict_map.py +++ b/src/hyperloom/inference_optimizer/tests/test_critic_verdict_map.py @@ -688,8 +688,8 @@ async def create_or_return_existing(self, **kwargs: Any): # noqa: ANN401 assert create_calls[0]["params"]["grid"] == grid -# 7. Specialist prompt — max_proposals self-curation contract (Section 1 + 8) -def _build_specialist_prompt_text(max_proposals: int) -> str: +# 7. Specialist prompt — proposal self-curation contract (Section 1 + 8) +def _build_specialist_prompt_text() -> str: from hyperloom.orchestrator.specialists.domains import get_domain from hyperloom.orchestrator.prompts.specialist_prompt_builder import ( SpecialistPromptInputs, @@ -700,42 +700,19 @@ def _build_specialist_prompt_text(max_proposals: int) -> str: task_id="t-test", domain=get_domain("serving_specialist"), max_turns=12, - max_proposals=max_proposals, gap_canonical_id="gap-x", ) system_prompt, user_prompt = build_specialist_prompts(inputs) return system_prompt + "\n" + user_prompt -def test_default_specialist_max_proposals_is_twelve(): - """Single-source-of-truth: policy.py owns the self-curation target (=12) and the builder re-exports it.""" - from hyperloom.orchestrator.policy.gate import ( - DEFAULT_SPECIALIST_MAX_PROPOSALS, - ) - from hyperloom.orchestrator.prompts.specialist_prompt_builder import ( - DEFAULT_SPECIALIST_MAX_PROPOSALS as PROMPT_DEFAULT, - ) - - assert DEFAULT_SPECIALIST_MAX_PROPOSALS == 12 - assert PROMPT_DEFAULT == 12 - - -def test_specialist_prompt_renders_max_proposals_5(): - """Caller can shrink the prompt-side self-curation target.""" - text = _build_specialist_prompt_text(max_proposals=5) - assert "AT MOST **5** entries" in text - assert "top-5" in text - # Critic-feedback warning present so the specialist knows marginal candidates cost. +def test_specialist_prompt_renders_top_6_target(): + text = _build_specialist_prompt_text() + assert "AT MOST **6** entries" in text + assert "top-6" in text assert "reviews each surviving variant" in text -def test_specialist_prompt_renders_default_top_12_target(): - text = _build_specialist_prompt_text(max_proposals=12) - assert "AT MOST **12** entries" in text - assert "top-12" in text - assert "AT MOST **5** entries" not in text - - # critic_robustness breakdown renderer (formerly test_critic_robustness_renderer_units.py) class TestCriticRobustnessRenderer: """Exercises the four observable shapes of the collector input.""" diff --git a/src/hyperloom/inference_optimizer/tests/test_dispatched_task_policy.py b/src/hyperloom/inference_optimizer/tests/test_dispatched_task_policy.py index 776739aa6..cb03d176e 100644 --- a/src/hyperloom/inference_optimizer/tests/test_dispatched_task_policy.py +++ b/src/hyperloom/inference_optimizer/tests/test_dispatched_task_policy.py @@ -82,7 +82,6 @@ async def _stub(_ctx) -> dict: assert executed["ran"] is False updated = await sub.tasks.get(task.task_id) assert updated.state == "cancelled" - assert updated.attempts == 0 @pytest.mark.asyncio @@ -319,7 +318,6 @@ async def _stub(_ctx) -> dict: assert executed["ran"] is True updated = await tasks2.get(task_id) assert updated.state == "succeeded" - assert updated.attempts == 1 class _ReconcileCoordStub: @@ -498,3 +496,27 @@ async def _stub(_ctx) -> dict: res = await sub.run_task(task) assert res.state == "succeeded" assert executed["ran"] is True + + +@pytest.mark.asyncio +async def test_killed_running_task_keeps_its_result(tmp_path, monkeypatch): + """A task cancelled mid-flight still returns its executor result.""" + sub = _runner_with_policy(tmp_path, monkeypatch) + task = await sub.tasks.create( + kind="report", + params={}, + idempotency_key="k-kill-midflight", + ) + + async def _stub(ctx): + # Simulate a kill landing while the executor is still running. + await sub.tasks.transition(ctx.task.task_id, "cancelled", evidence={"reason": "killed"}) + return {"produced": "work"} + + sub.register_executor("report", _stub) + + res = await sub.run_task(task) + assert res.state == "succeeded" + assert res.result == {"produced": "work"} + updated = await sub.tasks.get(task.task_id) + assert updated.state == "cancelled" diff --git a/src/hyperloom/inference_optimizer/tests/test_model_arch_advisory.py b/src/hyperloom/inference_optimizer/tests/test_model_arch_advisory.py index 84be831eb..04bfbce6d 100644 --- a/src/hyperloom/inference_optimizer/tests/test_model_arch_advisory.py +++ b/src/hyperloom/inference_optimizer/tests/test_model_arch_advisory.py @@ -86,6 +86,50 @@ def test_load_model_arch_stale_mismatch_returns_empty(tmp_path: Path): assert _load_model_arch(tmp_path, "DeepSeek-R1-0528") == {} +# 1a. $HYPERLOOM_MODEL_ARCH_FILE pre-seed: the session dir is created and read +# in the same CLI process, so a launcher can only get a profile in by seeding +# it before launch. +def test_load_model_arch_preseed_env_loads_and_copies(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + src = tmp_path / "preseed.json" + src.write_text(json.dumps(_VALID_ARCH), encoding="utf-8") + session = tmp_path / "session" + session.mkdir() + monkeypatch.setenv("HYPERLOOM_MODEL_ARCH_FILE", str(src)) + + out = _load_model_arch(session, "DeepSeek-R1-0528") + + assert out == _VALID_ARCH + # Copied into the session for provenance. + assert json.loads((session / "model_arch.json").read_text(encoding="utf-8")) == _VALID_ARCH + + +def test_load_model_arch_in_session_file_wins_over_preseed(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + src = tmp_path / "preseed.json" + src.write_text(json.dumps({**_VALID_ARCH, "source": "preseed"}), encoding="utf-8") + session = tmp_path / "session" + session.mkdir() + _write(session, {**_VALID_ARCH, "source": "in_session"}) + monkeypatch.setenv("HYPERLOOM_MODEL_ARCH_FILE", str(src)) + + assert _load_model_arch(session, "DeepSeek-R1-0528")["source"] == "in_session" + + +def test_load_model_arch_preseed_unreadable_returns_empty(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("HYPERLOOM_MODEL_ARCH_FILE", str(tmp_path / "missing.json")) + assert _load_model_arch(tmp_path / "session", "DeepSeek-R1-0528") == {} + + +def test_load_model_arch_preseed_still_stale_guarded(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + """Pre-seeding does not bypass the stale-model guard.""" + src = tmp_path / "preseed.json" + src.write_text(json.dumps({**_VALID_ARCH, "model_name": "Llama-3.1-8B"}), encoding="utf-8") + session = tmp_path / "session" + session.mkdir() + monkeypatch.setenv("HYPERLOOM_MODEL_ARCH_FILE", str(src)) + + assert _load_model_arch(session, "DeepSeek-R1-0528") == {} + + # 1b. HF hub cache path: launched --model is a snapshots/ dir whose # basename is a commit hash, but the declared clean model_name must still match. _HF_SNAPSHOT = ( diff --git a/src/hyperloom/inference_optimizer/tests/test_orchestration_memory_unit.py b/src/hyperloom/inference_optimizer/tests/test_orchestration_memory_unit.py index df71e862d..065983bfa 100644 --- a/src/hyperloom/inference_optimizer/tests/test_orchestration_memory_unit.py +++ b/src/hyperloom/inference_optimizer/tests/test_orchestration_memory_unit.py @@ -10,7 +10,6 @@ CheckpointTracker, build_memory_record, context_window_for_model, - deterministic_memory_fallback, is_degenerate_checkpoint, parse_checkpoint_reply, render_memory_for_seed, @@ -246,14 +245,6 @@ def test_policy_context_token_soft_disabled_by_default(): ) -def test_policy_is_hard_compaction(): - p = CheckpointPolicy(context_token_hard=170_000) - assert p.is_hard_compaction(170_000) - assert not p.is_hard_compaction(169_999) - # Disabled (0) never fires. - assert not CheckpointPolicy().is_hard_compaction(10**9) - - def test_tracker_set_context_tokens_and_reset_preserves_level(): t = CheckpointTracker() t.set_context_tokens(123_456) @@ -322,33 +313,6 @@ def test_build_memory_record_new_value_wins(): assert rec["learnings"] == ["L1", "L2"] -class _FakeState: - phase = "EXPLORE" - macro_cycle = 2 - current_best = {"tput": 1234.5} - optimization_stack = [{"a": 1}, {"b": 2}] - cumulative_gain_validated = 12.34 - - -def test_deterministic_memory_fallback_is_non_degenerate(): - fb = deterministic_memory_fallback(_FakeState()) - assert "EXPLORE" in fb["current_plan"] - assert "cycle=2" in fb["current_plan"] - assert "best_tput=1234.5" in fb["current_plan"] - assert "validated_gain=12.34%" in fb["current_plan"] - assert fb["tried_and_why"] == ["stack has 2 accepted change(s)"] - assert not is_degenerate_checkpoint(fb) - - -def test_deterministic_memory_fallback_missing_attrs(): - class _Empty: - pass - - fb = deterministic_memory_fallback(_Empty()) - # Never raises; yields a usable record. - assert fb["current_plan"].startswith("[auto]") - - # ---- next_cycle_directive: parse, sanitize, carry-forward ---- diff --git a/src/hyperloom/inference_optimizer/tests/test_per_domain_prompts.py b/src/hyperloom/inference_optimizer/tests/test_per_domain_prompts.py index b6d1bcfca..564add5cc 100644 --- a/src/hyperloom/inference_optimizer/tests/test_per_domain_prompts.py +++ b/src/hyperloom/inference_optimizer/tests/test_per_domain_prompts.py @@ -571,23 +571,21 @@ def test_R2_robustness_cannot_dispatch_specialist(gate): assert "Orchestration" in (exc.value.hint or "") -def test_R2_unknown_domain_denied(gate): - with pytest.raises(PolicyDenied) as exc: - gate.validate_intent( - "orchestration", - Intent( - type=IntentType.DELEGATE, - payload={ - "action_name": "specialist", - "params": { - "domain": "fake_specialist", - "gap_canonical_id": "gap.x", - }, +def test_R2_unknown_domain_allowed(gate): + """An unknown domain tag is observed, not denied; SpecialistRunner synthesizes an empty result.""" + gate.validate_intent( + "orchestration", + Intent( + type=IntentType.DELEGATE, + payload={ + "action_name": "specialist", + "params": { + "domain": "fake_specialist", + "gap_canonical_id": "gap.x", }, - ), - ) - assert exc.value.rule == "specialist_unknown_domain" - assert "tag" in (exc.value.hint or "") + }, + ), + ) def test_R2_missing_gap_denied(gate): @@ -644,23 +642,22 @@ def test_R2_max_turns_zero_allowed_unbounded(gate): ) -def test_R2_max_turns_negative_denied(gate): - with pytest.raises(PolicyDenied) as exc: - gate.validate_intent( - "orchestration", - Intent( - type=IntentType.DELEGATE, - payload={ - "action_name": "specialist", - "params": { - "domain": "serving_specialist", - "gap_canonical_id": "gap.x", - "max_turns": -1, - }, +def test_R2_max_turns_negative_allowed(gate): + """A negative max_turns is self-limiting (empty turn range), so it is not denied.""" + gate.validate_intent( + "orchestration", + Intent( + type=IntentType.DELEGATE, + payload={ + "action_name": "specialist", + "params": { + "domain": "serving_specialist", + "gap_canonical_id": "gap.x", + "max_turns": -1, }, - ), - ) - assert exc.value.rule == "specialist_dispatch_source" + }, + ), + ) def test_R2_specialist_action_skips_unknown_action_registry_path(gate): @@ -687,122 +684,6 @@ def test_R2_specialist_action_skips_unknown_action_registry_path(gate): ) -# 4. PolicyGate R3 — specialist_done_source -def test_R3_specialist_done_from_specialist_agent_ok(gate): - gate.validate_intent( - f"{SPECIALIST_FROM_AGENT_PREFIX}task-abc", - Intent(type=IntentType.SPECIALIST_DONE, payload=_valid_done_payload()), - ) - - -def test_R3_specialist_done_from_orchestration_denied(gate): - """Non-``specialist:*`` agents cannot emit specialist_done (the role-matrix gate fires).""" - with pytest.raises(PolicyDenied) as exc: - gate.validate_intent( - "orchestration", - Intent( - type=IntentType.SPECIALIST_DONE, - payload=_valid_done_payload(), - ), - ) - assert exc.value.rule == "role" - - -def test_R3_specialist_done_missing_gap_denied(gate): - payload = _valid_done_payload() - payload.pop("gap_canonical_id") - payload["gap_canonical_id"] = "" - with pytest.raises(PolicyDenied) as exc: - gate.validate_intent( - f"{SPECIALIST_FROM_AGENT_PREFIX}task-abc", - Intent(type=IntentType.SPECIALIST_DONE, payload=payload), - ) - assert exc.value.rule == "specialist_done_source" - - -def test_R3_specialist_done_unknown_domain_denied(gate): - payload = _valid_done_payload(domain="nonsense_specialist") - with pytest.raises(PolicyDenied) as exc: - gate.validate_intent( - f"{SPECIALIST_FROM_AGENT_PREFIX}task-abc", - Intent(type=IntentType.SPECIALIST_DONE, payload=payload), - ) - assert exc.value.rule == "specialist_done_source" - - -def test_R3_specialist_done_empty_true_requires_reason(gate): - payload = _valid_done_payload(empty=True, proposals=[]) - payload["summary"] = "" - with pytest.raises(PolicyDenied) as exc: - gate.validate_intent( - f"{SPECIALIST_FROM_AGENT_PREFIX}task-abc", - Intent(type=IntentType.SPECIALIST_DONE, payload=payload), - ) - assert exc.value.rule == "specialist_done_source" - - -def test_R3_specialist_done_empty_with_proposals_denied(gate): - payload = _valid_done_payload( - empty=True, - proposals=[{"name": "should_not_be_here"}], - ) - with pytest.raises(PolicyDenied) as exc: - gate.validate_intent( - f"{SPECIALIST_FROM_AGENT_PREFIX}task-abc", - Intent(type=IntentType.SPECIALIST_DONE, payload=payload), - ) - assert exc.value.rule == "specialist_done_source" - - -def test_R3_specialist_done_variant_missing_name_denied(gate): - payload = _valid_done_payload( - empty=False, - proposals=[{"extra_args": "--no-name"}], - ) - with pytest.raises(PolicyDenied) as exc: - gate.validate_intent( - f"{SPECIALIST_FROM_AGENT_PREFIX}task-abc", - Intent(type=IntentType.SPECIALIST_DONE, payload=payload), - ) - assert exc.value.rule == "specialist_done_source" - - -def test_R3_specialist_can_emit_send_message_heartbeat(gate): - gate.validate_intent( - f"{SPECIALIST_FROM_AGENT_PREFIX}task-abc", - Intent(type=IntentType.SEND_MESSAGE, payload={"topic": "heartbeat", "body_md": "still thinking"}), - ) - - -def test_R3_specialist_cannot_emit_propose_action(gate): - with pytest.raises(PolicyDenied) as exc: - gate.validate_intent( - f"{SPECIALIST_FROM_AGENT_PREFIX}task-abc", - Intent(type=IntentType.PROPOSE_ACTION, payload={"action_name": "explore", "predicted_gain_pct": 0.0}), - ) - assert exc.value.rule == "specialist_done_source" - - -def test_R3_specialist_missing_task_id_suffix_denied(gate): - with pytest.raises(PolicyDenied) as exc: - gate.validate_intent( - SPECIALIST_FROM_AGENT_PREFIX, # no suffix - Intent(type=IntentType.SPECIALIST_DONE, payload=_valid_done_payload()), - ) - assert exc.value.rule == "specialist_done_source" - - -def test_R3_specialist_done_confidence_out_of_range_denied(gate): - payload = _valid_done_payload() - payload["confidence"] = 1.5 - with pytest.raises(PolicyDenied) as exc: - gate.validate_intent( - f"{SPECIALIST_FROM_AGENT_PREFIX}task-abc", - Intent(type=IntentType.SPECIALIST_DONE, payload=payload), - ) - assert exc.value.rule == "specialist_done_source" - - # research_lane lane registration. def test_research_lane_in_known_lanes(): assert "research_lane" in KNOWN_LANES @@ -946,12 +827,6 @@ async def test_specialist_runner_synthesises_empty_done_on_max_turns(tmp_path): assert result.specialist_done["domain"] == "serving_specialist" assert "max_turns_exhausted" in result.specialist_done.get("reason", "") assert result.turns_used == 2 # max_turns reached - # The synthesised payload is still valid against PolicyGate R3. - gate = PolicyGate(role_registry=default_role_registry()) - gate.validate_intent( - f"{SPECIALIST_FROM_AGENT_PREFIX}task-stale", - Intent(type=IntentType.SPECIALIST_DONE, payload=result.specialist_done), - ) @pytest.mark.asyncio @@ -1012,18 +887,18 @@ def test_specialist_tool_denylist_is_empty(): assert write_tool in DEFAULT_SPECIALIST_TOOLS -def test_build_empty_specialist_done_is_R3_valid(): - """The failure-path helper must always produce a payload PolicyGate R3 accepts.""" +def test_build_empty_specialist_done_shape(): + """The failure-path helper must always produce a well-formed done payload.""" done = build_empty_specialist_done( gap_canonical_id="gap.x", domain="serving_specialist", reason="example reason", ) - gate = PolicyGate(role_registry=default_role_registry()) - gate.validate_intent( - f"{SPECIALIST_FROM_AGENT_PREFIX}task-test", - Intent(type=IntentType.SPECIALIST_DONE, payload=done), - ) + assert done["gap_canonical_id"] == "gap.x" + assert done["domain"] == "serving_specialist" + assert done["proposal_set"] == [] + assert done["empty"] is True + assert done["summary"] # 8. SharedState specialist round bookkeeping diff --git a/src/hyperloom/inference_optimizer/tests/test_policy_gate.py b/src/hyperloom/inference_optimizer/tests/test_policy_gate.py index e35167d64..631b6b67f 100644 --- a/src/hyperloom/inference_optimizer/tests/test_policy_gate.py +++ b/src/hyperloom/inference_optimizer/tests/test_policy_gate.py @@ -178,22 +178,8 @@ def test_freeform_description_valid() -> None: ) -@pytest.mark.parametrize( - "desc", - [ - "ps aux | grep sglang | xargs kill -9", - "pgrep -f bench_serving | xargs kill", - "killall -9 python", - ], -) -def test_freeform_description_kill_redline(desc: str) -> None: - with pytest.raises(PolicyDenied) as exc: - PolicyGate._check_freeform_task_description(desc, where="task[0]") - assert exc.value.rule == "specialist_freeform_redline" - - -def test_freeform_description_kill_not_overmatched() -> None: +def test_freeform_description_destructive_text_allowed() -> None: PolicyGate._check_freeform_task_description( - "Measure the kernel launch latency; do not kill the serving process.", + "killall -9 python", where="task[0]", ) diff --git a/src/hyperloom/inference_optimizer/tests/test_policy_helpers_coverage_unit.py b/src/hyperloom/inference_optimizer/tests/test_policy_helpers_coverage_unit.py index e35167d64..631b6b67f 100644 --- a/src/hyperloom/inference_optimizer/tests/test_policy_helpers_coverage_unit.py +++ b/src/hyperloom/inference_optimizer/tests/test_policy_helpers_coverage_unit.py @@ -178,22 +178,8 @@ def test_freeform_description_valid() -> None: ) -@pytest.mark.parametrize( - "desc", - [ - "ps aux | grep sglang | xargs kill -9", - "pgrep -f bench_serving | xargs kill", - "killall -9 python", - ], -) -def test_freeform_description_kill_redline(desc: str) -> None: - with pytest.raises(PolicyDenied) as exc: - PolicyGate._check_freeform_task_description(desc, where="task[0]") - assert exc.value.rule == "specialist_freeform_redline" - - -def test_freeform_description_kill_not_overmatched() -> None: +def test_freeform_description_destructive_text_allowed() -> None: PolicyGate._check_freeform_task_description( - "Measure the kernel launch latency; do not kill the serving process.", + "killall -9 python", where="task[0]", ) diff --git a/src/hyperloom/inference_optimizer/tests/test_protocol_layer.py b/src/hyperloom/inference_optimizer/tests/test_protocol_layer.py index 4790146a9..b9612c9aa 100644 --- a/src/hyperloom/inference_optimizer/tests/test_protocol_layer.py +++ b/src/hyperloom/inference_optimizer/tests/test_protocol_layer.py @@ -333,7 +333,6 @@ async def test_task_registry_legal_transitions(db): t = await tr.create(kind="bench_runner", params={}, idempotency_key="kT1") t2 = await tr.transition(t.task_id, "running") assert t2.state == "running" - assert t2.attempts == 1 t3 = await tr.transition(t.task_id, "succeeded", evidence={"tput": 1840}) assert t3.state == "succeeded" diff --git a/src/hyperloom/inference_optimizer/tests/test_role_realignment.py b/src/hyperloom/inference_optimizer/tests/test_role_realignment.py index edc0344ce..3542ec874 100644 --- a/src/hyperloom/inference_optimizer/tests/test_role_realignment.py +++ b/src/hyperloom/inference_optimizer/tests/test_role_realignment.py @@ -5,18 +5,21 @@ from __future__ import annotations +import time +from datetime import datetime, timezone from pathlib import Path import pytest from hyperloom.orchestrator.actions.registry import ActionRegistry +from hyperloom.orchestrator.loop.coordinator_helpers import _parse_iso_unix from hyperloom.inference_optimizer.protocol.intent import Intent, IntentType from hyperloom.orchestrator.state.shared_state import SharedState from hyperloom.orchestrator.prompts.prompt_builder import ( build_orchestration_prompt, default_enabled_actions, ) -from hyperloom.inference_optimizer.session.paths import make_session_dir +from hyperloom.inference_optimizer.session.paths import asset_system_prompts_dir, make_session_dir # fixtures @@ -48,6 +51,21 @@ def test_orchestration_prompt_includes_phase_contract(registry): assert "policy_denied" in text.lower() +def test_orchestration_prompt_explains_kill_task_resource_lifetime(registry): + text = build_orchestration_prompt( + action_registry=registry, + enabled_actions=default_enabled_actions(no_kernel=False), + framework="sglang", + objective_kind="gain_pct", + objective_value=10.0, + max_minutes=120, + rules_fragment_path=asset_system_prompts_dir() / "orchestration.md", + ) + + assert "does **not** terminate an already-running specialist process" in text + assert "Do not use this to promptly free capacity" in text + + def test_orchestration_prompt_no_kernel_marks_kernel_skipped(registry): text = build_orchestration_prompt( action_registry=registry, @@ -321,15 +339,17 @@ async def test_compose_prompt_orchestration_omits_warm_start_when_empty( @pytest.mark.asyncio -async def test_compose_prompt_robustness_renders_specialist_health( +@pytest.mark.parametrize("agent_name", ["robustness", "orchestration"]) +async def test_compose_prompt_omits_specialist_health_block( coordinator_with_mocks, + agent_name, ): + """The periodic specialist block is intentionally gone (see conversation.py).""" c = coordinator_with_mocks try: - prompt = await c._compose_prompt("robustness") - assert "=== Specialist health ===" in prompt - assert "running=0 stale=0" in prompt - assert "stale_threshold_sec=600" in prompt + prompt = await c._compose_prompt(agent_name) + assert "Specialist health" not in prompt + assert "stale" not in prompt.lower() finally: await c.stop() @@ -355,19 +375,645 @@ async def test_compose_prompt_robustness_includes_budget_telemetry( @pytest.mark.asyncio -async def test_scan_stale_specialists_returns_empty_when_no_specialists( +async def test_running_tasks_reader_reports_held_resources(coordinator_with_mocks): + """Lease expiry, lanes and GPU ids reach the planner. + + These four fields are the whole point of the on-demand path: the prompt + tells the planner to weigh "what is queued behind the lane or GPUs it + holds" and to extend a lease that is near expiry, so each has to survive + the join from ``leases`` / ``gpu_leases`` into the rendered line. + """ + c = coordinator_with_mocks + try: + task = await c.tasks.create( + kind="specialist", + params={}, + idempotency_key="k-resources", + lease_ttl_sec=1800, + ) + await c.tasks.transition(task.task_id, "running") + # Two lanes, deliberately out of sorted order and with different + # expiries: the renderer must sort the lanes and report the SOONEST + # expiry, because that is when reclaim starts. + for lane, holder, expires in ( + ("research_lane", "h-late", "2099-12-31T23:59:59+00:00"), + ("gpu_research_lane", "h-soon", "2099-01-01T00:00:00+00:00"), + ): + c.bus.db.raw.execute( + "INSERT INTO leases(lane, holder_id, task_id, action, pid, " + "acquired_at, expires_at, heartbeat_at) VALUES (?,?,?,?,?,?,?,?)", + ( + lane, + holder, + task.task_id, + "specialist", + 1, + "2026-05-19T18:00:00+00:00", + expires, + "2026-05-19T18:00:00+00:00", + ), + ) + for gpu_id in (3, 1): + c.bus.db.raw.execute( + "INSERT INTO gpu_leases(gpu_id, holder_id, task_id, acquired_at, " + "expires_at, heartbeat_at) VALUES (?,?,?,?,?,?)", + ( + gpu_id, + f"h-gpu{gpu_id}", + task.task_id, + "2026-05-19T18:00:00+00:00", + "2099-12-31T23:59:59+00:00", + "2026-05-19T18:00:00+00:00", + ), + ) + c.bus.db.raw.commit() + + out = c._context_running_tasks_reader() + assert "lanes=['gpu_research_lane', 'research_lane']" in out + assert "gpu_ids=[1, 3]" in out + # Soonest expiry wins: reclaim starts at the FIRST lane to lapse, so + # reporting the latest would overstate the remaining window by a year. + reported = int(out.split("lease_expires_in_sec=")[1].split()[0]) + now = datetime.now(timezone.utc) + soonest = int((datetime.fromisoformat("2099-01-01T00:00:00+00:00") - now).total_seconds()) + latest = int((datetime.fromisoformat("2099-12-31T23:59:59+00:00") - now).total_seconds()) + assert abs(reported - soonest) <= 5, f"expected soonest {soonest}, got {reported}" + assert abs(reported - latest) > 1000 + finally: + await c.stop() + + +@pytest.mark.asyncio +async def test_running_tasks_reader_reports_heartbeat_age( coordinator_with_mocks, + session_dir, ): + """Heartbeat age is read from the same files the reap loop polls. + + ``process.log`` counts as proof of life alongside ``heartbeat.json`` — a + specialist mid-benchmark can go minutes without restamping the heartbeat + while its log grows, and treating that as silence would invite a spurious + kill. + """ + from hyperloom.inference_optimizer.session.session_paths import runs_dir + c = coordinator_with_mocks try: - stale = await c._scan_stale_specialists() - assert stale == [] + task = await c.tasks.create( + kind="specialist", + params={}, + idempotency_key="k-heartbeat", + ) + await c.tasks.transition(task.task_id, "running") + # No workspace yet: the field is omitted rather than reported as zero. + assert "heartbeat_age_sec=" not in c._context_running_tasks_reader() + + ws = runs_dir(c.session_dir, "specialist", task.task_id) + ws.mkdir(parents=True, exist_ok=True) + (ws / "process.log").write_text("benchmarking\n", encoding="utf-8") + out = c._context_running_tasks_reader() + assert "heartbeat_age_sec=" in out finally: await c.stop() -# Phase budget telemetry math (independent of coordinator) -def test_to_phase_budget_telemetry_handles_empty_history(): - s = SharedState() - out = s.to_phase_budget_telemetry() - assert out == "(no phase history yet)" +@pytest.mark.asyncio +async def test_running_tasks_reader_skips_heartbeat_for_non_specialist( + coordinator_with_mocks, + session_dir, +): + """The kind guard holds even when a same-named workspace exists. + + ``runs_dir`` is keyed on task_id, so a non-specialist whose id collides + with a specialist workspace would otherwise inherit a heartbeat that + describes someone else's process. + """ + from hyperloom.inference_optimizer.session.session_paths import runs_dir + + c = coordinator_with_mocks + try: + task = await c.tasks.create( + kind="explore", + params={}, + idempotency_key="k-explore-hb", + ) + await c.tasks.transition(task.task_id, "running") + # Plant the liveness file the specialist path would have read. + ws = runs_dir(c.session_dir, "specialist", task.task_id) + ws.mkdir(parents=True, exist_ok=True) + (ws / "heartbeat.json").write_text("{}", encoding="utf-8") + + out = c._context_running_tasks_reader() + assert task.task_id in out + assert "kind='explore'" in out + assert "heartbeat_age_sec=" not in out + finally: + await c.stop() + + +@pytest.mark.asyncio +async def test_running_tasks_reader_survives_db_failure(coordinator_with_mocks): + """A read failure degrades to a message, never an exception. + + This reader backs a context tool the planner calls on its own turn; an + exception here would surface as an SDK stream failure and discard every + intent already collected in that turn. + """ + c = coordinator_with_mocks + try: + + def _boom(*_a, **_k): + raise RuntimeError("db gone") + + c.bus.db.fetchall_sync = _boom + out = c._context_running_tasks_reader() + assert "running tasks unavailable" in out + assert "db gone" in out + finally: + await c.stop() + + +@pytest.mark.asyncio +async def test_running_tasks_reader_reports_in_flight_task(coordinator_with_mocks): + """A running task is visible with its elapsed time and idempotency key.""" + c = coordinator_with_mocks + try: + assert "no tasks in flight" in c._context_running_tasks_reader() + task = await c.tasks.create( + kind="specialist", + params={"domain": "serving_specialist", "gap_canonical_id": "gap.x"}, + idempotency_key="k-running-1", + lease_ttl_sec=1800, + ) + await c.tasks.transition(task.task_id, "running") + out = c._context_running_tasks_reader() + assert "=== Tasks in flight ===" in out + assert task.task_id in out + assert "kind='specialist'" in out + assert "domain='serving_specialist'" in out + assert "gap='gap.x'" in out + assert "idempotency_key='k-running-1'" in out + assert "lease_ttl_sec=1800" in out + assert "running_sec=" in out + finally: + await c.stop() + + +@pytest.mark.asyncio +async def test_extend_lease_grows_ttl_and_lane_rows(coordinator_with_mocks): + """extend_lease pushes out both the task TTL and every lane row it holds.""" + from hyperloom.inference_optimizer.protocol.intent import Intent, IntentType + + c = coordinator_with_mocks + try: + task = await c.tasks.create( + kind="specialist", + params={}, + idempotency_key="k-extend-1", + requires_lanes=["research_lane"], + lease_ttl_sec=1800, + ) + await c.tasks.transition(task.task_id, "running") + lease = await c.locks.acquire_many( + ["research_lane"], + holder_id=f"h-{task.task_id}", + task_id=task.task_id, + action="specialist", + ttl_sec=1800, + ) + assert lease is not None + before = await c.tasks.get(task.task_id) + + await c._handle_intent( + "orchestration", + Intent( + type=IntentType.EXTEND_LEASE, + payload={"task_id": task.task_id, "extra_sec": 600, "reason": "close to done"}, + ), + ) + + updated = await c.tasks.get(task.task_id) + assert updated.lease_ttl_sec == 2400 + # updated_at marks when the task started running; extending must not move it. + assert updated.updated_at == before.updated_at + rows = await c.db.fetchall("SELECT lane, expires_at FROM leases WHERE task_id=?", (task.task_id,)) + assert [r["lane"] for r in rows] == ["research_lane"] + # The lane must expire at the REMAINING budget (cumulative TTL minus the + # elapsed run time), not at now + the full cumulative TTL. + expires_in = _parse_iso_unix(str(rows[0]["expires_at"])) - time.time() + assert expires_in <= 2400 + started = _parse_iso_unix(updated.updated_at) + remaining_budget = 2400 - (time.time() - started) + assert abs(expires_in - remaining_budget) < 5 + finally: + await c.stop() + + +@pytest.mark.asyncio +async def test_extend_lease_does_not_regrant_elapsed_time(coordinator_with_mocks): + """A task that already burned most of its TTL only gets the remainder back.""" + from hyperloom.inference_optimizer.protocol.intent import Intent, IntentType + + c = coordinator_with_mocks + try: + task = await c.tasks.create( + kind="specialist", + params={}, + idempotency_key="k-extend-elapsed", + requires_lanes=["research_lane"], + lease_ttl_sec=1800, + ) + await c.tasks.transition(task.task_id, "running") + await c.locks.acquire_many( + ["research_lane"], + holder_id=f"h-{task.task_id}", + task_id=task.task_id, + action="specialist", + ttl_sec=1800, + ) + # Backdate the running mark so the task looks 1000s old. + started_iso = datetime.fromtimestamp(time.time() - 1000, tz=timezone.utc).isoformat() + await c.db.execute( + "UPDATE tasks SET updated_at=? WHERE task_id=?", + (started_iso, task.task_id), + ) + + await c._handle_intent( + "orchestration", + Intent( + type=IntentType.EXTEND_LEASE, + payload={"task_id": task.task_id, "extra_sec": 600}, + ), + ) + + rows = await c.db.fetchall("SELECT expires_at FROM leases WHERE task_id=?", (task.task_id,)) + expires_in = _parse_iso_unix(str(rows[0]["expires_at"])) - time.time() + # 1800 + 600 cumulative, 1000 already spent -> ~1400s left, not 2400. + assert 1300 < expires_in < 1450 + finally: + await c.stop() + + +@pytest.mark.asyncio +async def test_extend_lease_late_grant_keeps_new_increment_for_lanes_and_gpus(coordinator_with_mocks): + """A late extension must not reduce newly granted resource time to one second.""" + c = coordinator_with_mocks + try: + task = await c.tasks.create( + kind="specialist", + params={"needs_gpu": True}, + idempotency_key="k-extend-late", + requires_lanes=["research_lane"], + lease_ttl_sec=1800, + ) + await c.tasks.transition(task.task_id, "running") + await c.locks.acquire_many( + ["research_lane"], + holder_id=f"h-{task.task_id}", + task_id=task.task_id, + action="specialist", + ttl_sec=1800, + ) + await c.db.execute( + "INSERT INTO gpu_leases(gpu_id, holder_id, task_id, acquired_at, expires_at, heartbeat_at) " + "VALUES (?,?,?,?,?,?)", + (0, f"h-gpu-{task.task_id}", task.task_id, "2026-01-01T00:00:00+00:00", "2026-01-01T00:30:00+00:00", "2026-01-01T00:00:00+00:00"), + ) + started_iso = datetime.fromtimestamp(time.time() - 3000, tz=timezone.utc).isoformat() + await c.db.execute("UPDATE tasks SET updated_at=? WHERE task_id=?", (started_iso, task.task_id)) + + await c._handle_intent( + "orchestration", + Intent( + type=IntentType.EXTEND_LEASE, + payload={"task_id": task.task_id, "extra_sec": 600}, + ), + ) + + lane_rows = await c.db.fetchall("SELECT expires_at FROM leases WHERE task_id=?", (task.task_id,)) + gpu_rows = await c.db.fetchall("SELECT expires_at FROM gpu_leases WHERE task_id=?", (task.task_id,)) + for row in [*lane_rows, *gpu_rows]: + expires_in = _parse_iso_unix(str(row["expires_at"])) - time.time() + assert 550 < expires_in < 650 + finally: + await c.stop() + + +@pytest.mark.asyncio +async def test_extend_lease_reports_degraded_when_gpu_refresh_fails(coordinator_with_mocks): + """A swallowed GPU-refresh failure must not read as a clean extension.""" + from hyperloom.inference_optimizer.protocol.intent import Intent, IntentType + + c = coordinator_with_mocks + try: + task = await c.tasks.create( + kind="specialist", + params={"needs_gpu": True}, + idempotency_key="k-extend-gpu-fail", + requires_lanes=["research_lane"], + lease_ttl_sec=1800, + ) + await c.tasks.transition(task.task_id, "running") + + async def _boom(*_a, **_kw): + raise RuntimeError("gpu pool down") + + c.gpu_specialist_pool.extend = _boom # type: ignore[method-assign] + + recorded: list[dict] = [] + original = c._record_observation + + async def _capture(agent, topic, payload): + recorded.append(dict(payload)) + return await original(agent, topic, payload) + + c._record_observation = _capture # type: ignore[method-assign] + + await c._handle_intent( + "orchestration", + Intent( + type=IntentType.EXTEND_LEASE, + payload={"task_id": task.task_id, "extra_sec": 600}, + ), + ) + + kinds = [r.get("kind") for r in recorded] + assert "extend_lease_degraded" in kinds + assert "extend_lease" not in kinds + degraded = next(r for r in recorded if r.get("kind") == "extend_lease_degraded") + assert "gpu pool down" in degraded["gpu_refresh_error"] + finally: + await c.stop() + + +@pytest.mark.asyncio +async def test_extend_lease_grants_live_subprocess_extension(coordinator_with_mocks): + """The handler must hand the grant to the reaper, not just move DB rows. + + The reap-loop side of this (that the deadline actually moves) is covered in + test_specialist_subprocess.py; here we pin the wiring. + """ + from hyperloom.inference_optimizer.protocol.intent import Intent, IntentType + from hyperloom.orchestrator.specialists import subprocess_ as _sub + + c = coordinator_with_mocks + try: + task = await c.tasks.create( + kind="specialist", + params={}, + idempotency_key="k-extend-live", + lease_ttl_sec=1800, + ) + await c.tasks.transition(task.task_id, "running") + _sub.clear_wall_budget_extension(task.task_id) + + await c._handle_intent( + "orchestration", + Intent( + type=IntentType.EXTEND_LEASE, + payload={"task_id": task.task_id, "extra_sec": 600}, + ), + ) + assert _sub.wall_budget_extension(task.task_id) == 600.0 + + # Repeated extensions accumulate on the live deadline. + await c._handle_intent( + "orchestration", + Intent( + type=IntentType.EXTEND_LEASE, + payload={"task_id": task.task_id, "extra_sec": 300}, + ), + ) + assert _sub.wall_budget_extension(task.task_id) == 900.0 + _sub.clear_wall_budget_extension(task.task_id) + finally: + await c.stop() + + +@pytest.mark.asyncio +async def test_extend_lease_survives_wall_budget_grant_failure(coordinator_with_mocks): + """A wall-budget failure retains DB changes but reports a degraded extension.""" + from hyperloom.inference_optimizer.protocol.intent import Intent, IntentType + from hyperloom.orchestrator.specialists import subprocess_ as _sub + + c = coordinator_with_mocks + try: + task = await c.tasks.create( + kind="specialist", + params={}, + idempotency_key="k-extend-grant-fail", + lease_ttl_sec=1800, + ) + await c.tasks.transition(task.task_id, "running") + + def _boom(*_a, **_kw): + raise RuntimeError("registry unavailable") + + original = _sub.grant_wall_budget_extension + _sub.grant_wall_budget_extension = _boom # type: ignore[assignment] + recorded: list[dict] = [] + original_record = c._record_observation + + async def _capture(agent, topic, payload): + recorded.append(dict(payload)) + return await original_record(agent, topic, payload) + + c._record_observation = _capture # type: ignore[method-assign] + try: + await c._handle_intent( + "orchestration", + Intent( + type=IntentType.EXTEND_LEASE, + payload={"task_id": task.task_id, "extra_sec": 600}, + ), + ) + finally: + _sub.grant_wall_budget_extension = original # type: ignore[assignment] + + # The DB-side extension still stands. + updated = await c.tasks.get(task.task_id) + assert updated.lease_ttl_sec == 2400 + degraded = next(r for r in recorded if r.get("kind") == "extend_lease_degraded") + assert "registry unavailable" in degraded["wall_budget_extension_error"] + finally: + await c.stop() + + +@pytest.mark.asyncio +async def test_extend_lease_survives_unreadable_running_age(coordinator_with_mocks): + """A failed age lookup must degrade to the full TTL, not abort the extension.""" + from hyperloom.inference_optimizer.protocol.intent import Intent, IntentType + + c = coordinator_with_mocks + try: + task = await c.tasks.create( + kind="specialist", + params={}, + idempotency_key="k-extend-age-fail", + requires_lanes=["research_lane"], + lease_ttl_sec=1800, + ) + await c.tasks.transition(task.task_id, "running") + await c.locks.acquire_many( + ["research_lane"], + holder_id=f"h-{task.task_id}", + task_id=task.task_id, + action="specialist", + ttl_sec=1800, + ) + + # extend_lease() itself still works; only the follow-up age read fails. + real_get = c.tasks.get + calls = {"n": 0} + + async def _get(task_id): + calls["n"] += 1 + if calls["n"] > 0 and task_id == task.task_id: + raise RuntimeError("row vanished") + return await real_get(task_id) + + c.tasks.get = _get # type: ignore[method-assign] + + await c._handle_intent( + "orchestration", + Intent( + type=IntentType.EXTEND_LEASE, + payload={"task_id": task.task_id, "extra_sec": 600}, + ), + ) + + c.tasks.get = real_get # type: ignore[method-assign] + # Lane still moved — falling back to the full TTL is the safe direction + # (a lease that outlives the task beats one reaped mid-run). + rows = await c.db.fetchall("SELECT expires_at FROM leases WHERE task_id=?", (task.task_id,)) + assert _parse_iso_unix(str(rows[0]["expires_at"])) > time.time() + updated = await c.tasks.get(task.task_id) + assert updated.lease_ttl_sec == 2400 + finally: + await c.stop() + + +@pytest.mark.asyncio +async def test_extend_lease_rejects_non_running_task(coordinator_with_mocks): + """A queued task cannot be extended; the rejection is recorded, not raised.""" + from hyperloom.inference_optimizer.protocol.intent import Intent, IntentType + + c = coordinator_with_mocks + try: + task = await c.tasks.create( + kind="specialist", + params={}, + idempotency_key="k-extend-2", + lease_ttl_sec=1800, + ) + await c._handle_intent( + "orchestration", + Intent( + type=IntentType.EXTEND_LEASE, + payload={"task_id": task.task_id, "extra_sec": 600}, + ), + ) + updated = await c.tasks.get(task.task_id) + assert updated.lease_ttl_sec == 1800 + finally: + await c.stop() + + +@pytest.mark.asyncio +async def test_send_message_to_specialist_writes_inbox(coordinator_with_mocks): + """A message addressed to a running specialist lands in its workspace inbox.""" + import json as _json + + from hyperloom.inference_optimizer.protocol.intent import Intent, IntentType + from hyperloom.inference_optimizer.session.session_paths import runs_dir + + c = coordinator_with_mocks + try: + await c._handle_intent( + "orchestration", + Intent( + type=IntentType.SEND_MESSAGE, + payload={ + "to": "specialist:task-steer", + "topic": "observation", + "body_md": "the mandate changed; measure prefill instead", + }, + ), + ) + inbox = runs_dir(c.session_dir, "specialist", "task-steer") / "inbox.json" + assert inbox.exists() + entries = _json.loads(inbox.read_text(encoding="utf-8")) + assert len(entries) == 1 + assert entries[0]["from"] == "orchestration" + assert "prefill" in entries[0]["body"]["body_md"] + finally: + await c.stop() + + +@pytest.mark.asyncio +async def test_send_message_to_specialist_prefers_worktree_inbox(coordinator_with_mocks): + """The production specialist has a worktree; the prompt advertises that path.""" + import json as _json + + from hyperloom.inference_optimizer.protocol.intent import Intent, IntentType + from hyperloom.inference_optimizer.session.session_paths import runs_dir + + c = coordinator_with_mocks + try: + workspace = runs_dir(c.session_dir, "specialist", "task-wt") + (workspace / "worktree").mkdir(parents=True, exist_ok=True) + + await c._handle_intent( + "orchestration", + Intent( + type=IntentType.SEND_MESSAGE, + payload={ + "to": "specialist:task-wt", + "topic": "observation", + "body_md": "steer toward decode", + }, + ), + ) + + worktree_inbox = workspace / "worktree" / "inbox.json" + assert worktree_inbox.exists() + assert not (workspace / "inbox.json").exists() + entries = _json.loads(worktree_inbox.read_text(encoding="utf-8")) + assert entries[0]["body"]["body_md"] == "steer toward decode" + finally: + await c.stop() + + +@pytest.mark.asyncio +async def test_extend_lease_also_pushes_gpu_rows(coordinator_with_mocks): + """The GPU lease is extended with the lane rows so the TTL ordering holds.""" + from hyperloom.inference_optimizer.protocol.intent import Intent, IntentType + + c = coordinator_with_mocks + try: + task = await c.tasks.create( + kind="specialist", + params={"needs_gpu": True}, + idempotency_key="k-extend-gpu", + requires_lanes=["research_lane"], + lease_ttl_sec=1800, + ) + await c.tasks.transition(task.task_id, "running") + await c.db.execute( + "INSERT INTO gpu_leases(gpu_id, holder_id, task_id, acquired_at, expires_at, heartbeat_at) " + "VALUES (?,?,?,?,?,?)", + (0, "h-gpu", task.task_id, "2026-01-01T00:00:00+00:00", "2026-01-01T00:30:00+00:00", "2026-01-01T00:00:00+00:00"), + ) + + await c._handle_intent( + "orchestration", + Intent( + type=IntentType.EXTEND_LEASE, + payload={"task_id": task.task_id, "extra_sec": 600}, + ), + ) + + rows = await c.db.fetchall("SELECT expires_at FROM gpu_leases WHERE task_id=?", (task.task_id,)) + assert rows and rows[0]["expires_at"] > "2026-01-01T00:30:00+00:00" + finally: + await c.stop() diff --git a/src/hyperloom/inference_optimizer/tests/test_specialist_dispatch_params.py b/src/hyperloom/inference_optimizer/tests/test_specialist_dispatch_params.py index db0631613..80e245f00 100644 --- a/src/hyperloom/inference_optimizer/tests/test_specialist_dispatch_params.py +++ b/src/hyperloom/inference_optimizer/tests/test_specialist_dispatch_params.py @@ -221,29 +221,25 @@ def test_freeform_description_too_long_rejected(gate, orchestration_role): @pytest.mark.parametrize( - "redline", + "desc", [ "clean up with rm -rf / now", "run mkfs.ext4 on the scratch disk", "please shutdown the host afterwards", ], ) -def test_freeform_redline_rejected(gate, orchestration_role, redline): - with pytest.raises(PolicyDenied) as exc: - gate._validate_specialist_dispatch( - orchestration_role, - _dispatch({"scope": "freeform", "task_description": redline}), - ) - assert exc.value.rule == "specialist_freeform_redline" +def test_freeform_destructive_text_allowed(gate, orchestration_role, desc): + gate._validate_specialist_dispatch( + orchestration_role, + _dispatch({"scope": "freeform", "task_description": desc}), + ) -def test_freeform_empty_wave_rejected(gate, orchestration_role): - with pytest.raises(PolicyDenied) as exc: - gate._validate_specialist_dispatch( - orchestration_role, - _dispatch({"scope": "freeform", "tasks": []}), - ) - assert exc.value.rule == "specialist_freeform_wave_invalid" +def test_freeform_empty_wave_falls_through_to_single_task(gate, orchestration_role): + gate._validate_specialist_dispatch( + orchestration_role, + _dispatch({"scope": "freeform", "tasks": [], "task_description": "probe"}), + ) def test_freeform_wave_too_large_rejected(gate, orchestration_role): @@ -256,22 +252,18 @@ def test_freeform_wave_too_large_rejected(gate, orchestration_role): assert exc.value.rule == "specialist_freeform_wave_too_large" -def test_freeform_wave_non_dict_task_rejected(gate, orchestration_role): - with pytest.raises(PolicyDenied) as exc: - gate._validate_specialist_dispatch( - orchestration_role, - _dispatch({"scope": "freeform", "tasks": ["not a dict"]}), - ) - assert exc.value.rule == "specialist_freeform_task_invalid" +def test_freeform_wave_non_dict_task_skipped(gate, orchestration_role): + gate._validate_specialist_dispatch( + orchestration_role, + _dispatch({"scope": "freeform", "tasks": ["not a dict"]}), + ) -def test_freeform_wave_empty_task_description_rejected(gate, orchestration_role): - with pytest.raises(PolicyDenied) as exc: - gate._validate_specialist_dispatch( - orchestration_role, - _dispatch({"scope": "freeform", "tasks": [{"task_description": ""}]}), - ) - assert exc.value.rule == "specialist_freeform_empty_description" +def test_freeform_wave_empty_task_description_skipped(gate, orchestration_role): + gate._validate_specialist_dispatch( + orchestration_role, + _dispatch({"scope": "freeform", "tasks": [{"task_description": ""}]}), + ) # --------------------------------------------------------------------------- # @@ -281,19 +273,17 @@ def test_freeform_wave_empty_task_description_rejected(gate, orchestration_role) @pytest.mark.skipif(len(_REAL_TAGS) < 1, reason="no knowledge-domain tags") -def test_domains_scope_requires_multiple_tags(gate, orchestration_role): - with pytest.raises(PolicyDenied) as exc: - gate._validate_specialist_dispatch( - orchestration_role, - _dispatch( - { - "scope": "domains", - "tags": [_REAL_TAGS[0]], - "gap_canonical_id": "gap.x.session-1", - } - ), - ) - assert exc.value.rule == "specialist_scope_too_narrow" +def test_domains_scope_with_one_tag_allowed(gate, orchestration_role): + gate._validate_specialist_dispatch( + orchestration_role, + _dispatch( + { + "scope": "domains", + "tags": [_REAL_TAGS[0]], + "gap_canonical_id": "gap.x.session-1", + } + ), + ) # --------------------------------------------------------------------------- # @@ -358,19 +348,17 @@ def test_gap_backfill_noop_when_no_anchor_match_still_rejects(orchestration_role @pytest.mark.skipif(len(_REAL_TAGS) < 2, reason="need >=2 knowledge-domain tags") -def test_single_domain_scope_rejects_multiple_tags(gate, orchestration_role): - with pytest.raises(PolicyDenied) as exc: - gate._validate_specialist_dispatch( - orchestration_role, - _dispatch( - { - "scope": "domain", - "tags": _REAL_TAGS[:2], - "gap_canonical_id": "gap.x.session-1", - } - ), - ) - assert exc.value.rule == "specialist_scope_mismatch" +def test_single_domain_scope_with_multiple_tags_allowed(gate, orchestration_role): + gate._validate_specialist_dispatch( + orchestration_role, + _dispatch( + { + "scope": "domain", + "tags": _REAL_TAGS[:2], + "gap_canonical_id": "gap.x.session-1", + } + ), + ) # --------------------------------------------------------------------------- # @@ -619,11 +607,11 @@ def test_normalize_dispatch_tags_keeps_valid_anchor_and_dedups(): assert normalize_dispatch_tags({"tags": ["serving_specialist", "framework"]}) == ["framework"] -def test_normalize_dispatch_tags_passes_garbage_through_for_rejection(): +def test_normalize_dispatch_tags_passes_garbage_through(): from hyperloom.orchestrator.specialists.domains import normalize_dispatch_tags # Genuinely unknown tags are NOT invented into an anchor — they pass - # through verbatim so PolicyGate's specialist_unknown_domain still fires. + # through verbatim so the runner can synthesize an empty result. assert normalize_dispatch_tags({"tags": ["totally_bogus"]}) == ["totally_bogus"] @@ -648,19 +636,18 @@ def test_dispatch_with_domain_key_tag_no_longer_rejected(gate, orchestration_rol ) -def test_dispatch_with_genuine_garbage_tag_still_rejected(gate, orchestration_role): - with pytest.raises(PolicyDenied) as exc: - gate._validate_specialist_dispatch( - orchestration_role, - _dispatch( - { - "scope": "domain", - "tags": ["totally_bogus"], - "gap_canonical_id": "gap.x.session-1", - } - ), - ) - assert exc.value.rule == "specialist_unknown_domain" +def test_dispatch_with_garbage_tag_allowed(gate, orchestration_role): + """An out-of-vocabulary tag is observed, not denied; the runner synthesizes an empty result.""" + gate._validate_specialist_dispatch( + orchestration_role, + _dispatch( + { + "scope": "domain", + "tags": ["totally_bogus"], + "gap_canonical_id": "gap.x.session-1", + } + ), + ) def test_specialist_emit_hint_lists_all_eight_llm_domains(): diff --git a/src/hyperloom/inference_optimizer/tests/test_specialist_lifecycle.py b/src/hyperloom/inference_optimizer/tests/test_specialist_lifecycle.py index 2159f7e97..af474d57b 100644 --- a/src/hyperloom/inference_optimizer/tests/test_specialist_lifecycle.py +++ b/src/hyperloom/inference_optimizer/tests/test_specialist_lifecycle.py @@ -226,57 +226,6 @@ async def test_record_specialist_result_idempotent_on_round_id(coord): assert state.specialist_rounds[0]["round_id"] == "round-7" -# 2. _handle_specialist_done — intent routing path -@pytest.mark.asyncio -async def test_handle_specialist_done_routes_known_task(coord): - """A known task_id triggers the full bookkeeping pass.""" - task = _StubTask(task_id="task-route", params={}) - coord.tasks.register(task) - - intent = Intent( - type=IntentType.SPECIALIST_DONE, - payload=_done_payload(domain="serving_specialist"), - ) - await coord._handle_specialist_done( - source=f"{SPECIALIST_FROM_AGENT_PREFIX}task-route", - intent=intent, - ) - - state: _StubSharedState = coord.shared_state - assert len(state.specialist_rounds) == 1 - assert state.last_specialist["task_id"] == "task-route" - - -@pytest.mark.asyncio -async def test_handle_specialist_done_unknown_task_logs_and_skips(coord, caplog): - """An unknown task_id logs a warning and skips bookkeeping (defense-in-depth).""" - intent = Intent( - type=IntentType.SPECIALIST_DONE, - payload=_done_payload(domain="serving_specialist"), - ) - await coord._handle_specialist_done( - source=f"{SPECIALIST_FROM_AGENT_PREFIX}unknown-id", - intent=intent, - ) - state: _StubSharedState = coord.shared_state - assert state.specialist_rounds == [] - - -@pytest.mark.asyncio -async def test_handle_specialist_done_bad_source_prefix(coord): - """A source not prefixed with ``specialist:`` → empty task_id → no bookkeeping.""" - intent = Intent( - type=IntentType.SPECIALIST_DONE, - payload=_done_payload(), - ) - await coord._handle_specialist_done( - source="orchestration", - intent=intent, - ) - state: _StubSharedState = coord.shared_state - assert state.specialist_rounds == [] - - # 3. _task_id_from_specialist_source helper def test_task_id_from_specialist_source_extracts_prefix(): from hyperloom.orchestrator.loop.coordinator import Coordinator @@ -574,14 +523,3 @@ async def test_force_stalled_domain_noop_outside_explore(force_coord): ) await force_coord._maybe_force_stalled_domain_specialist() force_coord._handle_intent.assert_not_awaited() - - -# 6. Intent routing branch wired into the dispatch table -def test_handle_intent_dispatch_table_has_specialist_done_branch(): - """The dispatch table routes SPECIALIST_DONE to ``_handle_specialist_done``.""" - from hyperloom.inference_optimizer.protocol.intent import IntentType - from hyperloom.orchestrator.loop.intent_router import _INTENT_DISPATCH - - assert _INTENT_DISPATCH.get(IntentType.SPECIALIST_DONE) == "_handle_specialist_done", ( - "_INTENT_DISPATCH must route SPECIALIST_DONE to _handle_specialist_done (KB_gaps/Gap-03)" - ) diff --git a/src/hyperloom/inference_optimizer/tests/test_specialist_prompt_builder_coverage_unit.py b/src/hyperloom/inference_optimizer/tests/test_specialist_prompt_builder_coverage_unit.py index bc72bcc10..8e0016e03 100644 --- a/src/hyperloom/inference_optimizer/tests/test_specialist_prompt_builder_coverage_unit.py +++ b/src/hyperloom/inference_optimizer/tests/test_specialist_prompt_builder_coverage_unit.py @@ -215,6 +215,13 @@ def test_gpu_autonomy_block_and_auto_retry(): assert "On-GPU autonomy" in sys_p assert "specialists.rebench" in sys_p assert "Auto-retry notice" in sys_p + assert "port 8888" not in sys_p + + +def test_no_gpu_iron_rule_retains_serving_port_boundary(): + inp = _rich_inputs(allocated_gpu_ids=()) + sys_p, _ = build_specialist_prompts(inp) + assert "port 8888" in sys_p def test_render_measured_impact_variants(): diff --git a/src/hyperloom/inference_optimizer/tests/test_specialist_rebench_unit.py b/src/hyperloom/inference_optimizer/tests/test_specialist_rebench_unit.py index 41c25ddd7..c041f3f0a 100644 --- a/src/hyperloom/inference_optimizer/tests/test_specialist_rebench_unit.py +++ b/src/hyperloom/inference_optimizer/tests/test_specialist_rebench_unit.py @@ -3,10 +3,8 @@ """Unit coverage for the optional GPU-specialist rebench helper. -``run_grid`` / Magpie are mocked, so these exercise the pure logic: port -resolution + 8888 refusal, leased-card reporting, env-pair parsing, the -success / failed-status / no-result / exception result shapes, and the CLI -``main``. +``run_grid`` / Magpie are mocked, so these exercise port resolution, +leased-card reporting, env-pair parsing, result shapes, and the CLI ``main``. """ from __future__ import annotations @@ -19,18 +17,11 @@ from hyperloom.orchestrator.specialists import rebench as sr -def test_pick_free_port_never_production() -> None: - port = sr._pick_free_port() - assert isinstance(port, int) - assert port != sr.PRODUCTION_SERVING_PORT - - -def test_resolve_port_auto_explicit_and_reject_8888() -> None: - assert sr._resolve_port(None) != sr.PRODUCTION_SERVING_PORT - assert sr._resolve_port(0) != sr.PRODUCTION_SERVING_PORT +def test_resolve_port_auto_and_explicit() -> None: + assert 0 < sr._resolve_port(None) <= 65535 + assert 0 < sr._resolve_port(0) <= 65535 assert sr._resolve_port(12345) == 12345 - with pytest.raises(ValueError): - sr._resolve_port(sr.PRODUCTION_SERVING_PORT) + assert sr._resolve_port(8888) == 8888 def test_current_leased_cards_precedence(monkeypatch) -> None: @@ -85,6 +76,7 @@ async def _fake_run_grid(**kwargs): assert "w1" in res["warnings"] assert seen["base_extra_args"] == "--kv-cache-dtype fp8_e4m3" assert seen["grid"][0].extra_server_args == "" + assert seen["preclean_before_run"] is False @pytest.mark.asyncio @@ -161,10 +153,3 @@ async def _fake(**kwargs): monkeypatch.setattr(sr, "run_specialist_rebench", _fake) rc = sr.main(["--output", str(tmp_path / "o"), "--env", "A=1", "--extra-args=--foo bar"]) assert rc == 1 - - -def test_main_rejects_explicit_8888(tmp_path, capsys) -> None: - rc = sr.main(["--output", str(tmp_path / "o"), "--port", "8888"]) - assert rc == 1 - out = json.loads(capsys.readouterr().out) - assert "8888" in out["error"] diff --git a/src/hyperloom/inference_optimizer/tests/test_specialist_subprocess.py b/src/hyperloom/inference_optimizer/tests/test_specialist_subprocess.py index 4af369b91..9b465896a 100644 --- a/src/hyperloom/inference_optimizer/tests/test_specialist_subprocess.py +++ b/src/hyperloom/inference_optimizer/tests/test_specialist_subprocess.py @@ -28,6 +28,7 @@ SPECIALIST_TOOL_DENYLIST, SpecialistRunner, ) +from hyperloom.orchestrator.specialists import subprocess_ from hyperloom.orchestrator.specialists.subprocess_ import ( SpecialistSubprocessConfig, SpecialistSubprocessDispatcher, @@ -209,6 +210,18 @@ def _make_fake_claude( {payload_json} EOF exit 3 +""" + elif behavior == "partial_then_done": + # Checkpoint first, wait for the reaper to see it, then exit normally. + body += f""" +cat > "$WORKSPACE/specialist_done.partial.json" <<'EOF' +{payload_json} +EOF +sleep 1 +cat > "$WORKSPACE/specialist_done.json" <<'EOF' +{payload_json} +EOF +exit 0 """ elif behavior == "hang": # Sleep past any wall budget without writing done.json. @@ -611,7 +624,9 @@ async def test_subprocess_recovers_partial_when_no_final( ctx = _make_runner_ctx("t-spec-partial") result = await runner.run(ctx) - assert result.status == "succeeded" + # Salvaged work keeps the findings but must not read as a clean run. + assert result.status == "partial" + assert "recovered_from_partial" in result.notes assert result.specialist_done["empty"] is False assert result.specialist_done.get("_recovered_from_partial") is True assert result.specialist_done["proposal_set"] @@ -755,6 +770,145 @@ def _fake_kill(p: Any) -> None: assert killed["v"] is True +# ── extend_lease moves the live wall-clock deadline ────────────────────────── +@pytest.fixture +def _live_reaper(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + """A reaper whose only stop condition is the hard wall-clock cap. + + Keeps process.log fresh so the staleness path never fires, and stubs + ``_kill`` so the timeout never signals a real process group (the fake + proc reuses this pytest process's pid). + """ + workspace = tmp_path / "ws" + workspace.mkdir() + (workspace / "process.log").write_text("alive\n", encoding="utf-8") + + cfg = SpecialistSubprocessConfig( + # Far above the run so only the wall-clock cap can end the loop. + heartbeat_stale_seconds=3600.0, + poll_interval_seconds=0.05, + ) + disp = SpecialistSubprocessDispatcher(config=cfg) + proc = _FakeProc() + monkeypatch.setattr( + SpecialistSubprocessDispatcher, + "_kill", + staticmethod(lambda p: setattr(p, "alive", False)), + ) + return disp, proc, workspace + + +@pytest.mark.asyncio +async def test_reap_loop_times_out_at_base_budget_without_extension(_live_reaper): + """Baseline: with no extension the run dies at its original cap.""" + disp, proc, workspace = _live_reaper + subprocess_.clear_wall_budget_extension("task-base") + + started = time.monotonic() + outcome = await disp._reap_loop( + proc=proc, + workspace=workspace, + done_files=(), + heartbeat_file=workspace / "heartbeat.json", + max_seconds=0.3, + started=time.monotonic(), + task_id="task-base", + ) + elapsed = time.monotonic() - started + + assert outcome["timed_out"] is True, outcome + # Killed at ~0.3s. The bound is loose because only the direction matters: + # a loaded CI box can stretch this, but it can never finish early. + assert elapsed < 5.0, elapsed + + +@pytest.mark.asyncio +async def test_reap_loop_deadline_moves_when_extension_granted_mid_run(_live_reaper): + """The regression this fix exists for. + + ``extend_lease`` used to push the task / lane / GPU leases out while the + subprocess kept the ``max_seconds`` deadline computed once at spawn, so the + specialist still died on schedule. The reaper must re-read the extension + every poll. + """ + disp, proc, workspace = _live_reaper + subprocess_.clear_wall_budget_extension("task-live") + + started = time.monotonic() + loop = asyncio.create_task( + disp._reap_loop( + proc=proc, + workspace=workspace, + done_files=(), + heartbeat_file=workspace / "heartbeat.json", + max_seconds=0.3, + started=time.monotonic(), + task_id="task-live", + ) + ) + # Grant the extension while the run is still in flight, before the + # original 0.3s cap would have fired. + await asyncio.sleep(0.15) + subprocess_.grant_wall_budget_extension("task-live", 0.6) + # The reaper recomputes `max_seconds + wall_budget_extension(task_id)` + # every poll, so this is the deadline it now enforces. + assert subprocess_.wall_budget_extension("task-live") == 0.6 + outcome = await loop + elapsed = time.monotonic() - started + + assert outcome["timed_out"] is True, outcome + # Survived past the base cap — the load-independent half of the proof + # (a slow box only ever pushes this later, never earlier). + assert elapsed > 0.7, elapsed + subprocess_.clear_wall_budget_extension("task-live") + + +@pytest.mark.asyncio +async def test_reap_loop_ignores_extension_for_a_different_task(_live_reaper): + """Extensions are per-task; another task's grant must not leak across.""" + disp, proc, workspace = _live_reaper + subprocess_.clear_wall_budget_extension("task-mine") + subprocess_.grant_wall_budget_extension("task-other", 600) + + started = time.monotonic() + outcome = await disp._reap_loop( + proc=proc, + workspace=workspace, + done_files=(), + heartbeat_file=workspace / "heartbeat.json", + max_seconds=0.3, + started=time.monotonic(), + task_id="task-mine", + ) + elapsed = time.monotonic() - started + + assert outcome["timed_out"] is True, outcome + # The other task's 600s grant would have kept this alive far past any + # plausible scheduling delay, so a bound this loose still proves isolation. + assert elapsed < 30.0, elapsed + subprocess_.clear_wall_budget_extension("task-other") + + +def test_wall_budget_extension_registry_guards(): + """Blank ids and non-positive grants are no-ops, not stored entries.""" + subprocess_.clear_wall_budget_extension("guard-task") + # Non-positive extra_sec must not create an entry. + assert subprocess_.grant_wall_budget_extension("guard-task", 0) == 0.0 + assert subprocess_.grant_wall_budget_extension("guard-task", -30) == 0.0 + assert subprocess_.wall_budget_extension("guard-task") == 0.0 + # Blank / whitespace task ids are ignored rather than keyed on "". + assert subprocess_.grant_wall_budget_extension("", 600) == 0.0 + assert subprocess_.grant_wall_budget_extension(" ", 600) == 0.0 + assert subprocess_.wall_budget_extension("") == 0.0 + # Lookups are whitespace-insensitive so a padded id still finds its grant. + subprocess_.grant_wall_budget_extension(" guard-task ", 120) + assert subprocess_.wall_budget_extension("guard-task") == 120.0 + # Clearing an unknown id is a no-op, not a KeyError. + subprocess_.clear_wall_budget_extension("never-seen") + subprocess_.clear_wall_budget_extension("guard-task") + assert subprocess_.wall_budget_extension("guard-task") == 0.0 + + # ── P2/T4: needs_gpu specialist runs inside a GpuSpecialistLease actor ──────── class _FakeGpuSpecialistLease: """Fake GpuSpecialistLease: start_async() writes done.json + log, then 'exits'.""" @@ -845,6 +999,48 @@ def _boom(*_a, **_k): assert result.exit_code == 0 +@pytest.mark.asyncio +async def test_run_clears_stale_wall_budget_extension( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + """A reused task_id must not inherit a previous run's granted extension. + + The registry is keyed by task_id and lives for the process, so a grant left + behind by an earlier dispatch would silently widen the next run's deadline. + """ + workspace = tmp_path / "ws" + lease = _FakeGpuSpecialistLease(workspace) + monkeypatch.setattr( + subprocess_.subprocess, + "Popen", + lambda *_a, **_k: pytest.fail("gpu_lease path must not spawn locally"), + ) + + # A grant left over from a prior dispatch of the same task id. + subprocess_.grant_wall_budget_extension("t-reused", 9999) + assert subprocess_.wall_budget_extension("t-reused") == 9999.0 + + cfg = SpecialistSubprocessConfig(poll_interval_seconds=0.05) + disp = SpecialistSubprocessDispatcher(config=cfg) + result = await disp.run( + task_id="t-reused", + workspace=workspace, + worktree=None, + worktree_base=None, + system_prompt="sys", + user_prompt="usr", + allowed_tools=(), + max_turns=1, + wall_budget_sec=60.0, + gpu_lease=lease, + ) + + assert result.exit_code == 0 + # Cleared on entry and again when the run finished. + assert subprocess_.wall_budget_extension("t-reused") == 0.0 + + def test_kill_on_ray_lease_process_delegates_to_actor(): """_kill on a _RayLeaseProcess reaps via the actor, not killpg.""" from hyperloom.orchestrator.specialists.subprocess_ import _RayLeaseProcess @@ -857,3 +1053,141 @@ def test_kill_on_ray_lease_process_delegates_to_actor(): assert lease.stopped is True # After reap the actor reports not-alive; poll latches the exit code. assert handle.poll() == 0 + + +@pytest.mark.parametrize("raw", ["not-a-number", ""]) +def test_ray_specialist_pending_deadline_invalid_env_falls_back( + monkeypatch: pytest.MonkeyPatch, + raw: str, +): + """A malformed scheduling-timeout override cannot make dispatch crash.""" + monkeypatch.setenv("INFERENCE_OPTIMIZER_RAY_SPECIALIST_SCHED_TIMEOUT_SEC", raw) + assert subprocess_._ray_specialist_pending_deadline_sec() == 300.0 + + +def test_ray_lease_process_dead_actor_without_exit_code_is_latched(): + """An unreachable Ray actor is a terminal failure, not an endless poll.""" + from hyperloom.orchestrator.actions.executors._ray_serving import _RAY_ACTOR_DIED_RC + from hyperloom.orchestrator.specialists.subprocess_ import _RayLeaseProcess + + class _DeadLease: + def is_alive(self): + return False + + def exit_code(self): + return None + + handle = _RayLeaseProcess(_DeadLease(), pid=1234) + assert handle.poll() == _RAY_ACTOR_DIED_RC + # The terminal value is latched; a second poll does not query the lease. + handle._lease = None + assert handle.poll() == _RAY_ACTOR_DIED_RC + + +def test_build_claude_cmd_includes_optional_flags_and_filters_emit_intent(tmp_path: Path): + """Optional CLI wiring is composed once and must survive as valid argv.""" + workspace = tmp_path / "workspace" + worktree = workspace / "worktree" + framework = tmp_path / "framework" + for path in (workspace, worktree, framework): + path.mkdir(parents=True, exist_ok=True) + prompt = workspace / "prompt.md" + prompt.write_text("prompt", encoding="utf-8") + + cfg = SpecialistSubprocessConfig( + model="claude-test", + mcp_config_path="/tmp/mcp.json", + framework_source_roots=(str(framework), str(framework)), + extra_claude_args=("--debug",), + leaf_agents_json='{"researcher": {"description": "test"}}', + ) + cmd = SpecialistSubprocessDispatcher(cfg)._build_claude_cmd( + prompt_file=prompt, + workspace=workspace, + worktree=worktree, + allowed_tools=("Read", "Task", "emit_intent"), + ) + + assert cmd[cmd.index("--model") + 1] == "claude-test" + assert cmd[cmd.index("--allowedTools") + 1] == "Read,Task" + assert "emit_intent" not in cmd + assert cmd[cmd.index("--agents") + 1] == cfg.leaf_agents_json + assert cmd[cmd.index("--mcp-config") + 1] == "/tmp/mcp.json" + assert cmd[-1] == "--debug" + add_dirs = [cmd[i + 1] for i, value in enumerate(cmd[:-1]) if value == "--add-dir"] + # Worktree first, workspace second, then each distinct framework root. + assert add_dirs == [str(worktree), str(workspace), str(framework)] + + +@pytest.mark.asyncio +async def test_partial_progress_skips_invalid_payload_and_swallow_callback_error(tmp_path: Path): + """Malformed checkpoints and telemetry failures never terminate a run.""" + disp = SpecialistSubprocessDispatcher(SpecialistSubprocessConfig()) + invalid = tmp_path / "invalid.partial.json" + invalid.write_text("not json", encoding="utf-8") + + async def _must_not_run(*_args): + raise AssertionError("invalid payload must not reach the callback") + + assert ( + await disp._publish_partial_progress( + partial_files=(invalid,), + since_mtime=0.0, + elapsed=1.0, + progress_cb=_must_not_run, + ) + == 0.0 + ) + + valid = tmp_path / "valid.partial.json" + valid.write_text('{"summary": "still working"}', encoding="utf-8") + + async def _callback_fails(*_args): + raise RuntimeError("telemetry sink unavailable") + + newest = await disp._publish_partial_progress( + partial_files=(valid,), + since_mtime=0.0, + elapsed=2.0, + progress_cb=_callback_fails, + ) + assert newest == valid.stat().st_mtime + + +@pytest.mark.asyncio +async def test_partial_checkpoint_published_while_alive( + tmp_path: Path, + fake_framework_repo: Path, +): + """A checkpoint written mid-run reaches the progress callback before exit.""" + bin_dir = tmp_path / "bin" + fake_claude = _make_fake_claude(bin_dir, behavior="partial_then_done") + session_dir = tmp_path / "session" + session_dir.mkdir() + + config = SpecialistSubprocessConfig( + claude_executable=str(fake_claude), + model="", + framework_source_roots=(str(fake_framework_repo),), + per_turn_max_seconds=15.0, + poll_interval_seconds=0.2, + ) + runner = SpecialistRunner( + subprocess_config=config, + session_dir=session_dir, + default_max_turns=2, + ) + seen: list[tuple[dict, float]] = [] + + async def _progress(payload, elapsed): + seen.append((payload, elapsed)) + + ctx = _make_runner_ctx("t-spec-progress") + ctx.extra["specialist_progress_cb"] = _progress + + result = await runner.run(ctx) + assert result.status == "succeeded" + assert seen, "no progress checkpoint was published while the run was alive" + payload, elapsed = seen[0] + assert payload["summary"] == "fake claude subprocess output" + assert elapsed >= 0.0 diff --git a/src/hyperloom/orchestrator/bus/db_maintenance.py b/src/hyperloom/orchestrator/bus/db_maintenance.py index c8fb91700..d594e4577 100644 --- a/src/hyperloom/orchestrator/bus/db_maintenance.py +++ b/src/hyperloom/orchestrator/bus/db_maintenance.py @@ -15,8 +15,8 @@ ``min(min_processed_seq, max_seq - keep_recent)`` — strictly below the resume anchor. * **In-flight safety** — ``tasks`` pruning never touches ``queued`` / ``running`` - / ``failed`` / ``needs_manual_review`` rows; only truly-done - (``succeeded`` / ``cancelled``) rows beyond a keep-recent count are removed. + / ``failed`` rows; only truly-done (``succeeded`` / ``cancelled``) rows beyond + a keep-recent count are removed. """ from __future__ import annotations @@ -153,8 +153,8 @@ async def prune_tasks( ) -> int: """Delete old done (succeeded/cancelled) tasks beyond ``keep_done``. - Never touches queued/running/failed/needs_manual_review rows. Returns the - number of rows deleted. + Never touches queued/running/failed rows. Returns the number of rows + deleted. Args: db: Open SQLite connection to prune the ``tasks`` table on. diff --git a/src/hyperloom/orchestrator/bus/gpu_pool.py b/src/hyperloom/orchestrator/bus/gpu_pool.py index 5cf65c360..ee1ce5d5a 100644 --- a/src/hyperloom/orchestrator/bus/gpu_pool.py +++ b/src/hyperloom/orchestrator/bus/gpu_pool.py @@ -382,6 +382,28 @@ async def release(self, lease: GpuLease | None) -> None: params, ) + async def extend(self, task_id: str, ttl_sec: int) -> int: + """Push a task's GPU rows out to ``ttl_sec`` from now. + + Keeps the ``kill <= gpu_lease TTL <= gpu_research_lane TTL`` ordering + when a lane lease is extended. + + Args: + task_id: The task whose GPU rows should be refreshed. + ttl_sec: New lifetime in seconds from now. + + Returns: + The number of GPU rows refreshed. + """ + expires_iso = datetime.fromtimestamp(time.time() + max(0, int(ttl_sec)), tz=timezone.utc).isoformat() + now_iso = _now_iso() + async with self.db.transaction() as cur: + cur.execute( + "UPDATE gpu_leases SET expires_at=?, heartbeat_at=? WHERE task_id=?", + (expires_iso, now_iso, task_id), + ) + return int(cur.rowcount or 0) + async def reap_expired(self) -> int: """Actively delete TTL-expired GPU leases; returns rows reaped. diff --git a/src/hyperloom/orchestrator/bus/resource_lock.py b/src/hyperloom/orchestrator/bus/resource_lock.py index 40fb75370..191837579 100644 --- a/src/hyperloom/orchestrator/bus/resource_lock.py +++ b/src/hyperloom/orchestrator/bus/resource_lock.py @@ -331,6 +331,28 @@ async def heartbeat(self, lease: Lease, *, ttl_sec: int) -> None: if cur.rowcount != len(lease.lanes): raise StaleLeaseError(f"heartbeat mismatch: expected {len(lease.lanes)} rows, got {cur.rowcount}") + async def heartbeat_by_task(self, task_id: str, *, ttl_sec: int) -> list[str]: + """Refresh every lane row a task holds, whoever the holder is. + + Args: + task_id: The task whose lane rows should be refreshed. + ttl_sec: New lifetime in seconds from now. + + Returns: + The lanes that were refreshed, sorted. + """ + new_expires_iso = datetime.fromtimestamp(time.time() + ttl_sec, tz=timezone.utc).isoformat() + now_iso = _now_iso() + async with self.db.transaction() as cur: + cur.execute("SELECT lane FROM leases WHERE task_id=?", (task_id,)) + lanes = sorted(str(r["lane"]) for r in cur.fetchall()) + if lanes: + cur.execute( + "UPDATE leases SET expires_at=?, heartbeat_at=? WHERE task_id=?", + (new_expires_iso, now_iso, task_id), + ) + return lanes + async def release(self, lease: Lease) -> int: """Drop every (lane, holder_id) row this lease owns. @@ -592,6 +614,18 @@ async def heartbeat(self, lease: Lease, *, ttl_sec: int) -> None: """ return await self.backend.heartbeat(lease, ttl_sec=ttl_sec) + async def heartbeat_by_task(self, task_id: str, *, ttl_sec: int) -> list[str]: + """Refresh every lane row a task holds. + + Args: + task_id (str): The task whose lane rows should be refreshed. + ttl_sec (int): New lifetime in seconds. + + Returns: + list[str]: The lanes that were refreshed. + """ + return await self.backend.heartbeat_by_task(task_id, ttl_sec=ttl_sec) + async def release(self, lease: Lease) -> int: """Release a lease and bump each lane's release counter. diff --git a/src/hyperloom/orchestrator/bus/storage/schema.py b/src/hyperloom/orchestrator/bus/storage/schema.py index 3fb4e2af3..5d0fb0dae 100644 --- a/src/hyperloom/orchestrator/bus/storage/schema.py +++ b/src/hyperloom/orchestrator/bus/storage/schema.py @@ -94,14 +94,13 @@ kind TEXT NOT NULL, state TEXT NOT NULL CHECK (state IN ('queued','running','succeeded','failed', - 'cancelled','needs_manual_review')), + 'cancelled')), params TEXT NOT NULL, idempotency_key TEXT NOT NULL UNIQUE, requires_lanes TEXT NOT NULL DEFAULT '[]', allowed_tools TEXT NOT NULL DEFAULT '[]', side_effects TEXT NOT NULL DEFAULT '[]', lease_ttl_sec INTEGER NOT NULL DEFAULT 0, - attempts INTEGER NOT NULL DEFAULT 0, history TEXT NOT NULL DEFAULT '[]', created_at TEXT NOT NULL, updated_at TEXT NOT NULL diff --git a/src/hyperloom/orchestrator/loop/conversation.py b/src/hyperloom/orchestrator/loop/conversation.py index 26d8bcb3d..352190e40 100644 --- a/src/hyperloom/orchestrator/loop/conversation.py +++ b/src/hyperloom/orchestrator/loop/conversation.py @@ -15,6 +15,9 @@ from .coordinator import ( _format_inbox_event, ) +from .coordinator_helpers import _parse_iso_unix +from ..state.task_registry import Task +from hyperloom.inference_optimizer.session.session_paths import runs_dir import logging as _logging log = _logging.getLogger(__name__) @@ -125,6 +128,7 @@ def _attach_orchestration_context_tools(self) -> None: inbox_reader=self._context_inbox_reader, analysis_reader=self._context_analysis_reader, recent_outcomes_reader=self._context_recent_outcomes_reader, + running_tasks_reader=self._context_running_tasks_reader, action_runner=self._run_action_now_sync, ) setter(provider) @@ -139,7 +143,7 @@ def _context_inbox_reader(self, since_seq: int = 0) -> str: included; defaults to ``0`` (all events). Returns: - A newline-joined rendering of the last 40 matching inbox events, or + A newline-joined rendering of all matching inbox events, or a placeholder string when none are available. """ try: @@ -153,7 +157,7 @@ def _context_inbox_reader(self, since_seq: int = 0) -> str: return "(no inbox events)" msgs = [Message.from_row(r) for r in rows] - lines = [_format_inbox_event(m) for m in msgs[-40:]] + lines = [_format_inbox_event(m) for m in msgs] return "\n".join(lines) def _context_recent_outcomes_reader(self, top_k: int = 8) -> str: @@ -181,13 +185,109 @@ def _context_recent_outcomes_reader(self, top_k: int = 8) -> str: if not rows: return "(no recent outcomes)" - # Flip newest-first query to newest-last for chronological reading. # Flip newest-first query to newest-last for chronological reading. msgs = [Message.from_row(r) for r in rows][::-1] lines = ["=== Recent action outcomes (newest last) ==="] lines.extend(_format_inbox_event(m) for m in msgs) return "\n".join(lines) + def _context_running_tasks_reader(self) -> str: + """Synchronous projection of in-flight tasks with their held resources. + + Returns: + One line per running task carrying elapsed time, lease expiry, held + lanes, leased GPUs and heartbeat age, or a placeholder string. + """ + try: + rows = self.bus.db.fetchall_sync( + "SELECT * FROM tasks WHERE state='running' ORDER BY updated_at ASC", + (), + ) + except Exception as exc: # noqa: BLE001 + return f"(running tasks unavailable: {exc!r})" + if not rows: + return "(no tasks in flight)" + + lanes_by_task: dict[str, list[str]] = {} + # Soonest lane expiry: the first one to lapse is when reclaim starts. + expiry_by_task: dict[str, str] = {} + for r in self.bus.db.fetchall_sync("SELECT lane, task_id, expires_at FROM leases", ()): + tid = str(r["task_id"]) + lanes_by_task.setdefault(tid, []).append(str(r["lane"])) + expires = str(r["expires_at"]) + prev = expiry_by_task.get(tid) + if prev is None or expires < prev: + expiry_by_task[tid] = expires + gpus_by_task: dict[str, list[int]] = {} + for r in self.bus.db.fetchall_sync("SELECT gpu_id, task_id FROM gpu_leases", ()): + gpus_by_task.setdefault(str(r["task_id"]), []).append(int(r["gpu_id"])) + + now_unix = time.time() + lines = ["=== Tasks in flight ==="] + for row in rows: + task = Task.from_row(row) + params = task.params or {} + started = _parse_iso_unix(task.updated_at) + running_sec = max(0.0, now_unix - started) if started > 0 else 0.0 + parts = [ + f" - task_id={task.task_id}", + f"kind={task.kind!r}", + f"running_sec={int(running_sec)}", + ] + domain = str(params.get("domain") or "") + gap = str(params.get("gap_canonical_id") or "") + if domain: + parts.append(f"domain={domain!r}") + if gap: + parts.append(f"gap={gap!r}") + parts.append(f"idempotency_key={task.idempotency_key!r}") + if task.lease_ttl_sec: + parts.append(f"lease_ttl_sec={task.lease_ttl_sec}") + expires_at = expiry_by_task.get(task.task_id, "") + if expires_at: + exp_unix = _parse_iso_unix(expires_at) + if exp_unix > 0: + parts.append(f"lease_expires_in_sec={int(exp_unix - now_unix)}") + lanes = lanes_by_task.get(task.task_id) + if lanes: + parts.append(f"lanes={sorted(lanes)}") + gpus = gpus_by_task.get(task.task_id) + if gpus: + parts.append(f"gpu_ids={sorted(gpus)}") + hb_age = self._task_heartbeat_age_sec(task, now_unix=now_unix) + if hb_age is not None: + parts.append(f"heartbeat_age_sec={int(hb_age)}") + lines.append(" ".join(parts)) + return "\n".join(lines) + + def _task_heartbeat_age_sec(self, task: "Task", *, now_unix: float) -> float | None: + """Age of a specialist's freshest liveness file, mirroring the reaper. + + The reap loop treats either ``heartbeat.json`` or ``process.log`` as + proof of life; this reports the same signal. + + Args: + task: The running task to probe. + now_unix: Current wall-clock epoch seconds. + + Returns: + Seconds since the most recent liveness write, or ``None`` when no + workspace file is readable. + """ + if (task.kind or "").strip() != "specialist": + return None + ws = runs_dir(self.session_dir, "specialist", task.task_id) + newest = 0.0 + for name in ("heartbeat.json", "process.log"): + try: + mtime = (ws / name).stat().st_mtime + except OSError: + continue + newest = max(newest, mtime) + if newest <= 0: + return None + return max(0.0, now_unix - newest) + def _context_analysis_reader(self) -> str: """Return the latest TraceLens analysis.md snapshot text. @@ -352,6 +452,9 @@ async def _compose_prompt(self, agent_name: str) -> str: if push_full: sections.append("=== Shared session state ===") sections.append(self.shared_state.to_prompt_summary()) + # Capacities a needs_gpu dispatch is admitted against. + sections.append("=== Resource pools ===") + sections.append(self.shared_state.to_resource_pools_summary()) if agent_name == "orchestration": # target_gap_pct is the gain still needed for --target-gain. obj = getattr(self, "_current_objective", None) @@ -488,11 +591,24 @@ async def _compose_prompt(self, agent_name: str) -> str: "with the read-only context tools: get_shared_state, " "get_gaps, get_warm_start, get_proposal_scores, " "get_intervention_mix, why_denied, show_analysis_md, " - "get_inbox. Reason from your own running plan; do not " - "re-derive it from scratch." + "get_inbox, get_recent_outcomes, get_running_tasks. Reason " + "from your own running plan; do not re-derive it from scratch." ) - # Robustness gets phase budget telemetry + specialist health for medium-severity alerts. + # NOTE: there is deliberately no "=== Specialist health ===" block. + # This prompt renders only on an agent's own turn, and a turn only + # comes around between blocking actions — so a running specialist is + # exactly what the agent is waiting on and is structurally absent from + # any snapshot taken here. Measured over a full 11.6h session: 33 + # renders, 0 of them overlapped a live specialist, while specialists + # held 41% of the wall clock. A block that always reports "none + # running" is worse than no block, because it manufactures a false + # belief. In-flight specialists reach the agent through + # ``specialist_progress`` observations (pushed from the reap loop, + # independent of turn timing) and are verified on demand with + # ``get_running_tasks``. + + # Robustness gets phase budget telemetry for medium-severity alerts. if agent_name == "robustness": try: budget_block = self.shared_state.to_phase_budget_telemetry( @@ -504,21 +620,6 @@ async def _compose_prompt(self, agent_name: str) -> str: if budget_block: sections.append("=== Phase budget telemetry ===") sections.append(budget_block) - try: - stale = await self._scan_stale_specialists() - running = await self.tasks.running() - specialist_running = sum(1 for t in (running or []) if (t.kind or "").strip() == "specialist") - except Exception: # noqa: BLE001 — defensive - log.exception("Coordinator: specialist health scan failed") - stale, specialist_running = [], 0 - stale_lines = [f" - task_id={row['task_id']} running_sec={int(row['running_seconds'])}" for row in stale] - sections.append("=== Specialist health ===") - sections.append( - f"running={specialist_running} stale={len(stale)} stale_threshold_sec={int(self._specialist_stale_sec)}" - ) - if stale_lines: - sections.append("stale specialists (consider kill_task):") - sections.extend(stale_lines) # Conversation no-progress circuit-breaker; Robustness is the external safety net. try: @@ -548,7 +649,8 @@ async def _compose_prompt(self, agent_name: str) -> str: # 2. Inbox tail since this agent's last cursor. cursor = await self.cursors.load(agent_name) msgs = await self.bus.replay_for(agent_name, after_seq=cursor.last_processed_seq) - rendered = list(msgs[-20:]) + # Full unread batch: events arriving in one tick must not be dropped. + rendered = list(msgs) # Durable at-least-once-until-decided delivery of proposals to the # Critic: the inbox tail is lossy, so re-present every still-undecided # proposal from the durable ``pending_proposals`` registry. diff --git a/src/hyperloom/orchestrator/loop/coordinator.py b/src/hyperloom/orchestrator/loop/coordinator.py index 2495ae246..4c27d56b5 100644 --- a/src/hyperloom/orchestrator/loop/coordinator.py +++ b/src/hyperloom/orchestrator/loop/coordinator.py @@ -266,7 +266,9 @@ def _framework_config_levers_from_done( "throughput", "tput_tok_s", ) -_OUTCOME_STATUS_KEYS: tuple[str, ...] = ("status", "verdict", "outcome") +_OUTCOME_STATUS_KEYS: tuple[str, ...] = ("status", "verdict", "outcome", "runner_status") +# Notes rendered per inbox line. +_OUTCOME_NOTES_MAX: int = 3 def _first_present(d: dict[str, Any], keys: tuple[str, ...]) -> Any | None: @@ -346,6 +348,7 @@ def _format_inbox_event(m: "Message") -> str: error = payload.get("error") result = payload.get("result") parts = [head, f"kind={kind!r}", f"state={state!r}"] + notes: list[Any] = [] if isinstance(result, dict): status = _first_present(result, _OUTCOME_STATUS_KEYS) gain = _first_present(result, _OUTCOME_GAIN_KEYS) @@ -359,8 +362,18 @@ def _format_inbox_event(m: "Message") -> str: parts.append(f"gain={gain}") if tput is not None: parts.append(f"tput={tput}") + # Executors that never raise report the failure inside the result + # envelope, leaving the top-level error None. + if not error: + error = result.get("error") + raw_notes = result.get("notes") + if isinstance(raw_notes, list): + notes = [n for n in raw_notes if n][:_OUTCOME_NOTES_MAX] if error: parts.append(f"error={str(error)[:200]!r}") + if notes: + shown = "; ".join(str(n) for n in notes) + parts.append(f"notes={shown[:300]!r}") return " ".join(parts) if topic in ("policy_denial", "denial") or (topic == "observation" and payload.get("kind") == "policy_denial"): @@ -554,19 +567,6 @@ def __init__( self._proposal_scorer: Any = proposal_scorer # Phase budget percentages, normalised once at construction. self._phase_budget_pct: dict[str, float] = _phase_state.normalize_budget_pct(phase_budget_pct) - # Specialist stale scan threshold (seconds). - try: - self._specialist_stale_sec: float = max( - 0.0, - float( - os.environ.get( - "INFERENCE_OPTIMIZER_SPECIALIST_STALE_SEC", - "600", - ) - ), - ) - except ValueError: - self._specialist_stale_sec = 600.0 self._model_class_override: str = (model_class or "").strip() # Validate every reactor has a backend wired. @@ -675,20 +675,12 @@ def _ckpt_fraction(env_key: str, default: float) -> float: "INFERENCE_OPTIMIZER_CTX_SOFT_FRACTION", _orch_mem.DEFAULT_CONTEXT_TOKEN_SOFT_FRACTION, ) - _hard_frac = _ckpt_fraction( - "INFERENCE_OPTIMIZER_CTX_HARD_FRACTION", - _orch_mem.DEFAULT_CONTEXT_TOKEN_HARD_FRACTION, - ) self._checkpoint_policy = _orch_mem.CheckpointPolicy( context_token_soft=int(_ctx_window * _soft_frac), - context_token_hard=int(_ctx_window * _hard_frac), ) self._checkpoint_tracker = _orch_mem.CheckpointTracker( last_phase=str(getattr(self.shared_state, "phase", "") or ""), ) - # Minimum ticks between orchestration-memory compactions, to avoid a - # checkpoint-every-tick loop. A near-window emergency bypasses this floor. - self._checkpoint_min_tick_gap = 3 # Consecutive degenerate checkpoint replies; resets on a good one. self._consec_degenerate_ckpt: int = 0 # Disable checkpointing entirely via env. @@ -862,10 +854,11 @@ def router(self) -> IntentRouter: "_handle_review_verdict": "router", "_handle_single_verdict": "router", "_handle_delegate": "router", - "_handle_specialist_done": "router", "_handle_request": "router", "_handle_response": "router", "_handle_kill_task": "router", + "_handle_extend_lease": "router", + "_deliver_specialist_inbox": "router", "_handle_prune_branch": "router", "_handle_escalate_strategy_change": "router", "_handle_send_message": "router", @@ -971,9 +964,9 @@ def router(self) -> IntentRouter: "_on_enter_explore": "phase_explore", "_maybe_force_stalled_domain_specialist": "phase_explore", "_seed_gaps_from_research_hints": "phase_explore", - "_scan_stale_specialists": "phase_explore", "_fan_out_specialist_wave": "phase_explore", "_maybe_auto_retry_specialist": "phase_explore", + "_record_specialist_retry_exhausted": "phase_explore", "_warm_specialist_params": "phase_explore", "_refresh_gaps": "phase_explore", "_extract_gaps_from_baseline": "phase_explore", @@ -1066,6 +1059,8 @@ def router(self) -> IntentRouter: "_attach_orchestration_context_tools": "conversation", "_context_inbox_reader": "conversation", "_context_recent_outcomes_reader": "conversation", + "_context_running_tasks_reader": "conversation", + "_task_heartbeat_age_sec": "conversation", "_context_analysis_reader": "conversation", "_record_reactor_conversation": "conversation", "_compose_prompt": "conversation", @@ -1100,6 +1095,7 @@ def router(self) -> IntentRouter: "_spawn_fitting_queued": "dispatcher", "_run_dispatched_with_gpu_release": "dispatcher", "_specialist_wall_budget_sec": "dispatcher", + "_specialist_progress_publisher": "dispatcher", "_resolve_serving_tp": "dispatcher", "_gpu_lease_ttl_sec": "dispatcher", "_reap_dispatched_task": "dispatcher", diff --git a/src/hyperloom/orchestrator/loop/dispatcher.py b/src/hyperloom/orchestrator/loop/dispatcher.py index 353cde9cf..b5120f750 100644 --- a/src/hyperloom/orchestrator/loop/dispatcher.py +++ b/src/hyperloom/orchestrator/loop/dispatcher.py @@ -397,6 +397,7 @@ async def _spawn_fitting_queued( extra_context["wall_budget_sec"] = self._specialist_wall_budget_sec( needs_gpu=needs_gpu, ) + extra_context["specialist_progress_cb"] = self._specialist_progress_publisher(task) if needs_gpu: # §3.4: probe serving-slot state immediately before admitting # this GPU specialist (not once per pass) so a serving start @@ -629,6 +630,52 @@ async def _run_dispatched_with_gpu_release( task.task_id, ) + def _specialist_progress_publisher(self, task: Task) -> Any: + """Build the callback that turns partial checkpoints into observations. + + Args: + task: The specialist task the callback reports for. + + Returns: + An async callable taking ``(payload, elapsed_sec)``. + """ + + async def _publish(payload: dict[str, Any], elapsed: float) -> None: + """Append one specialist_progress observation. + + Args: + payload: The checkpoint payload the specialist wrote. + elapsed: Seconds since the subprocess was spawned. + """ + params = task.params or {} + proposals = payload.get("proposal_set") + findings = payload.get("new_findings") + questions = payload.get("residual_questions") + await self.bus.append_and_seq( + Message.new( + "coordinator", + "*", + "observation", + { + "kind": "specialist_progress", + "task_id": task.task_id, + "domain": str(params.get("domain") or ""), + "gap_canonical_id": str(params.get("gap_canonical_id") or ""), + "elapsed_sec": int(elapsed), + "summary": str(payload.get("summary") or "")[:400], + "proposals_so_far": len(proposals) if isinstance(proposals, list) else 0, + "new_findings": [str(f)[:200] for f in (findings or [])[:3]] + if isinstance(findings, list) + else [], + "residual_questions": [str(q)[:200] for q in (questions or [])[:3]] + if isinstance(questions, list) + else [], + }, + ) + ) + + return _publish + def _specialist_wall_budget_sec(self, *, needs_gpu: bool) -> float: """Compute the explicit wall-clock budget for a specialist task. @@ -1219,7 +1266,7 @@ async def _run_action_now( already-in-flight notice, or the rendered delegated_result line. """ - # PolicyGate parity: validate synthetic delegate intent so phase/role/paths/red-line gates apply. + # PolicyGate parity: validate the synthetic delegate so phase/role/path gates apply. intent = Intent( type=IntentType.DELEGATE, payload={"action_name": action_name, "params": dict(params or {})}, @@ -1260,7 +1307,6 @@ async def _run_action_now( "succeeded", "failed", "cancelled", - "needs_manual_review", ): return ( f"(run_action_now: an identical {action_name!r} task is " diff --git a/src/hyperloom/orchestrator/loop/intent_router.py b/src/hyperloom/orchestrator/loop/intent_router.py index 651030508..4a7453c0f 100644 --- a/src/hyperloom/orchestrator/loop/intent_router.py +++ b/src/hyperloom/orchestrator/loop/intent_router.py @@ -23,10 +23,17 @@ from typing import Any from hyperloom.inference_optimizer.protocol.intent import Intent, IntentType -from .coordinator_helpers import coerce_needs_gpu, format_exc_brief, serialize_verdict_advisory +from .coordinator_helpers import ( + _parse_iso_unix, + coerce_needs_gpu, + format_exc_brief, + serialize_verdict_advisory, +) +from hyperloom.common.timeutil import now_iso +from hyperloom.inference_optimizer.session.session_paths import runs_dir from ..bus.message_bus import Message -from ..policy.gate import PolicyDenied -from ..state.task_registry import Task +from ..policy.gate import PolicyDenied, SPECIALIST_FROM_AGENT_PREFIX +from ..state.task_registry import IllegalTransition, TaskNotFound from ..kernel.request_handlers import get_handler # ``Coordinator`` is intentionally NOT imported (avoids a module-level import @@ -39,8 +46,6 @@ # IntentType -> the ``Coordinator`` handler method it dispatches to. Replaces the # former 12-branch if/elif in :meth:`IntentRouter._handle_intent`; an unknown # type falls through to the observation fallback (see the ``else`` branch there). -# ``SPECIALIST_DONE`` is a terminal specialist intent (R3 already validated); its -# handler only bookkeeps (defense-in-depth). _INTENT_DISPATCH: dict[IntentType, str] = { IntentType.PROPOSE_ACTION: "_handle_propose_action", IntentType.REVIEW_VERDICT: "_handle_review_verdict", @@ -48,12 +53,12 @@ IntentType.REQUEST: "_handle_request", IntentType.RESPONSE: "_handle_response", IntentType.KILL_TASK: "_handle_kill_task", + IntentType.EXTEND_LEASE: "_handle_extend_lease", IntentType.PRUNE_BRANCH: "_handle_prune_branch", IntentType.ESCALATE_STRATEGY_CHANGE: "_handle_escalate_strategy_change", IntentType.SEND_MESSAGE: "_handle_send_message", IntentType.ALERT: "_handle_alert", IntentType.UPDATE_STATE: "_handle_update_state", - IntentType.SPECIALIST_DONE: "_handle_specialist_done", } @@ -427,7 +432,6 @@ async def _handle_delegate(self, source: str, intent: Intent) -> None: "succeeded", "failed", "cancelled", - "needs_manual_review", } task = None was_existing = False @@ -507,41 +511,6 @@ async def _handle_delegate(self, source: str, intent: Intent) -> None: ) ) - async def _handle_specialist_done( - self, - source: str, - intent: Intent, - ) -> None: - """Handle a ``specialist_done`` intent (source ``specialist:`` per Inv-5.3 / R3); bookkeeping in _record_specialist_result. - - Args: - source: The emitting agent, expected as ``specialist:``. - intent: The SPECIALIST_DONE intent carrying the done payload. - """ - payload = dict(intent.payload or {}) - task_id = self._task_id_from_specialist_source(source) - task: Task | None = None - if task_id: - try: - task = await self.tasks.get(task_id) - except Exception: # noqa: BLE001 — TaskNotFound and friends - task = None - if task is None: - # PolicyGate R3 should have caught this; log defensively but don't crash. - log.warning( - "specialist_done from source=%r references unknown " - "task_id=%r; skipping bookkeeping (R3 should have " - "denied; defense in depth)", - source, - task_id, - ) - return - await self._record_specialist_result( - task=task, - done_payload=payload, - source=source, - ) - async def _handle_request(self, source: str, intent: Intent) -> None: """Route a REQUEST intent to its target agent (Plan A: → kernel). @@ -836,6 +805,91 @@ async def _handle_kill_task(self, source: str, intent: Intent) -> None: ) ) + async def _handle_extend_lease(self, source: str, intent: Intent) -> None: + """Grant a running task more lease time. + + Refreshes the task's ``lease_ttl_sec``, its lane rows, its GPU rows and + the live subprocess wall-clock deadline together, preserving + ``kill <= gpu_lease TTL <= gpu_research_lane TTL``. + + ``lease_ttl_sec`` is a *cumulative* budget measured from ``updated_at`` + (when the task entered ``running``), but lane / GPU rows expire at + ``now + ttl``. Feeding the cumulative TTL straight into them would hand + back the already-elapsed time, so the refresh uses the remaining budget. + + Args: + source (str): The agent requesting the extension. + intent (Intent): The EXTEND_LEASE intent; ``payload`` carries + ``task_id``, ``extra_sec`` and an optional ``reason``. + """ + task_id = str(intent.payload.get("task_id") or "").strip() + extra_sec = int(intent.payload.get("extra_sec") or 0) + try: + new_ttl = await self.tasks.extend_lease(task_id, extra_sec) + except (TaskNotFound, IllegalTransition) as exc: + await self._record_observation( + "coordinator", + "observation", + { + "kind": "extend_lease_rejected", + "task_id": task_id, + "source": source, + "error": repr(exc)[:200], + }, + ) + return + # Remaining budget = cumulative TTL minus the time already spent running. + running_sec = 0.0 + try: + task = await self.tasks.get(task_id) + started = _parse_iso_unix(task.updated_at) + if started > 0: + running_sec = max(0.0, time.time() - started) + except Exception: # noqa: BLE001 — fall back to the full TTL + log.exception("extend_lease: could not read running age for task=%s", task_id) + # A late extension can arrive after the cumulative task TTL expired but + # before the worker/reaper acted on it. It must still buy the full newly + # granted increment rather than refreshing leases for only one second. + remaining_sec = max(1, int(extra_sec), int(new_ttl - running_sec)) + lanes = await self.locks.heartbeat_by_task(task_id, ttl_sec=remaining_sec) + gpu_error = "" + try: + gpus = await self.gpu_specialist_pool.extend(task_id, remaining_sec) + except Exception as exc: # noqa: BLE001 — lane extension already landed + log.exception("extend_lease: GPU lease refresh failed for task=%s", task_id) + gpus = 0 + gpu_error = repr(exc)[:200] + # Push the live subprocess's hard wall-clock kill deadline out too, so + # the extension actually buys the specialist more time to run. + wall_budget_error = "" + try: + from ..specialists.subprocess_ import grant_wall_budget_extension + + grant_wall_budget_extension(task_id, extra_sec) + except Exception as exc: # noqa: BLE001 — lease rows already moved + log.exception("extend_lease: wall-budget extension failed for task=%s", task_id) + wall_budget_error = repr(exc)[:200] + # A swallowed GPU or wall-budget failure would leave the lane extended + # while the GPU reaper or subprocess wall-clock cap can still interrupt + # the work — report the partial extension as degraded. + await self._record_observation( + "coordinator", + "observation", + { + "kind": "extend_lease_degraded" if gpu_error or wall_budget_error else "extend_lease", + "task_id": task_id, + "source": source, + "extra_sec": extra_sec, + "lease_ttl_sec": new_ttl, + "lease_expires_in_sec": remaining_sec, + "lanes": lanes, + "gpu_rows": gpus, + **({"gpu_refresh_error": gpu_error} if gpu_error else {}), + **({"wall_budget_extension_error": wall_budget_error} if wall_budget_error else {}), + "reason": str(intent.payload.get("reason") or "")[:200], + }, + ) + async def _handle_prune_branch(self, source: str, intent: Intent) -> None: """Prune an action family and cancel its in-flight tasks. @@ -945,8 +999,9 @@ async def _handle_escalate_strategy_change(self, source: str, intent: Intent) -> async def _handle_send_message(self, source: str, intent: Intent) -> None: """Publish a free-form message onto the bus. - Soft-degrades an unknown topic to ``observation`` and - routes to the requested recipient (defaulting to broadcast). + Soft-degrades an unknown topic to ``observation`` and routes to the + requested recipient (defaulting to broadcast). A ``specialist:`` + recipient additionally gets the message in its workspace inbox. Args: source (str): The sending agent. @@ -969,6 +1024,47 @@ async def _handle_send_message(self, source: str, intent: Intent) -> None: {k: v for k, v in intent.payload.items() if k != "to"}, ) ) + if str(to_agent).startswith(SPECIALIST_FROM_AGENT_PREFIX): + self._deliver_specialist_inbox(source, str(to_agent), intent.payload) + + def _deliver_specialist_inbox(self, source: str, to_agent: str, payload: dict[str, Any]) -> None: + """Append a message to a running specialist's workspace inbox. + + A specialist reads ``inbox.json`` between turns; the reaper ignores the + file, so this steers a live run without ending it. + + Args: + source (str): The sending agent. + to_agent (str): Recipient of the form ``specialist:``. + payload (dict[str, Any]): The send_message payload. + """ + task_id = to_agent[len(SPECIALIST_FROM_AGENT_PREFIX) :].strip() + if not task_id: + return + try: + workspace = runs_dir(self.session_dir, "specialist", task_id) + workspace.mkdir(parents=True, exist_ok=True) + # The prompt advertises the worktree when one exists; match it. + worktree = workspace / "worktree" + inbox = (worktree if worktree.is_dir() else workspace) / "inbox.json" + existing: list[Any] = [] + if inbox.exists(): + loaded = json.loads(inbox.read_text(encoding="utf-8")) + if isinstance(loaded, list): + existing = loaded + existing.append( + { + "from": source, + "ts": now_iso(), + "body": {k: v for k, v in payload.items() if k not in ("to", "topic")}, + } + ) + # Keep the last 32 so the file stays prompt-sized. + tmp = inbox.with_suffix(".json.tmp") + tmp.write_text(json.dumps(existing[-32:], indent=2), encoding="utf-8") + tmp.replace(inbox) + except Exception: # noqa: BLE001 — steering is best-effort + log.exception("failed to deliver inbox message to %s", to_agent) async def _handle_alert(self, source: str, intent: Intent) -> None: """Broadcast an alert message, prioritized by severity. diff --git a/src/hyperloom/orchestrator/loop/maintenance.py b/src/hyperloom/orchestrator/loop/maintenance.py index 709499d18..e42c48944 100644 --- a/src/hyperloom/orchestrator/loop/maintenance.py +++ b/src/hyperloom/orchestrator/loop/maintenance.py @@ -194,46 +194,16 @@ async def _maybe_checkpoint_orchestration( tracker = self._checkpoint_tracker ticks_since = max(0, tick - tracker.last_tick) minutes_since = max(0.0, now_min - tracker.last_minute_mark) - # Hard context-token guardrail: near the window we MUST compact even when - # the LLM summary is degenerate (deterministic fallback) to avoid overflow. - hard = self._checkpoint_policy.is_hard_compaction(tracker.context_tokens_now) - # Anti-thrash floor for the TOKEN-budget triggers only: require a minimum - # tick gap between compactions so a re-seeded conversation doesn't - # re-trip the budget every tick. Cadence triggers are unaffected; a true - # near-window emergency (>= 98%) always bypasses the floor. - suppress_token_trigger = False - if not force and ticks_since < max(1, int(getattr(self, "_checkpoint_min_tick_gap", 2) or 2)): - ctx_token_hard = int(getattr(self._checkpoint_policy, "context_token_hard", 0) or 0) - ctx_token_soft = int(getattr(self._checkpoint_policy, "context_token_soft", 0) or 0) - ctx_now = int(tracker.context_tokens_now) - token_due = (ctx_token_hard > 0 and ctx_now >= ctx_token_hard) or ( - ctx_token_soft > 0 and ctx_now >= ctx_token_soft - ) - emergency_ceiling = ( - int(ctx_token_hard / max(0.01, _orch_mem.DEFAULT_CONTEXT_TOKEN_HARD_FRACTION) * 0.98) - if ctx_token_hard > 0 - else 0 - ) - in_emergency = emergency_ceiling > 0 and ctx_now >= emergency_ceiling - suppress_token_trigger = token_due and not in_emergency - if suppress_token_trigger: - # Suppress the token-driven hard flag so the freshly-seeded - # conversation can persist; fall through to the cadence check. - hard = False - # Authoritative growth signal is the context-token water level; the char - # count is a fallback for backends that don't report token usage. + # Growth signal is the context-token water level; char count is the + # fallback for backends that don't report token usage. if ( not force - and not hard and not self._checkpoint_policy.should_checkpoint( ticks_since_last=ticks_since, minutes_since_last=minutes_since, chars_since_last=tracker.chars_since_last, phase_changed=phase_changed, - # During the anti-thrash window, zero out the token level so the - # soft-token trigger can't re-fire the compaction we just - # suppressed; cadence triggers still evaluate normally. - context_tokens_now=0 if suppress_token_trigger else tracker.context_tokens_now, + context_tokens_now=tracker.context_tokens_now, ) ): return False @@ -252,10 +222,9 @@ async def _maybe_checkpoint_orchestration( parsed = _orch_mem.parse_checkpoint_reply(raw_text) degenerate = _orch_mem.is_degenerate_checkpoint(parsed) cur_phase = str(getattr(self.shared_state, "phase", "") or "") - # Path 1 — degenerate reply, NOT near the window: skip compaction, - # preserve the live conversation + prior memory, but reset the - # tracker to avoid a checkpoint storm. - if degenerate and not hard: + # Degenerate reply: skip compaction, preserve the live conversation + + # prior memory, but reset the tracker to avoid a checkpoint storm. + if degenerate: self._coord._consec_degenerate_ckpt += 1 tracker.reset(tick=tick, minute_mark=now_min, phase=cur_phase) await self._record_observation( @@ -283,11 +252,7 @@ async def _maybe_checkpoint_orchestration( }, ) return False - # Path 2 — degenerate but near the window: compact anyway using a - # deterministic fallback synthesised from authoritative state. - if degenerate and hard: - parsed = _orch_mem.deterministic_memory_fallback(self.shared_state) - # Path 3 (and post-fallback): a usable summary — compact for real. + # Usable summary — compact for real. self._coord._consec_degenerate_ckpt = 0 seq = 0 try: @@ -331,7 +296,6 @@ async def _maybe_checkpoint_orchestration( "seq": seq, "checkpoint_count": record.get("checkpoint_count", 0), "phase_changed": bool(phase_changed), - "hard_compaction": bool(hard), "context_tokens": int(tracker.context_tokens_now), }, ) diff --git a/src/hyperloom/orchestrator/loop/sub_agent_runner.py b/src/hyperloom/orchestrator/loop/sub_agent_runner.py index 5be3a1a19..92f625683 100644 --- a/src/hyperloom/orchestrator/loop/sub_agent_runner.py +++ b/src/hyperloom/orchestrator/loop/sub_agent_runner.py @@ -21,7 +21,7 @@ from hyperloom.inference_optimizer.session.session_paths import _runs_actions, runs_dir from ..bus.resource_lock import Lease, ResourceLockManager from ..policy.gate import PolicyDenied -from ..state.task_registry import Task, TaskNotFound, TaskRegistry +from ..state.task_registry import IllegalTransition, Task, TaskNotFound, TaskRegistry if TYPE_CHECKING: from ..policy.gate import PolicyGate @@ -56,14 +56,13 @@ class SubAgentResult: Attributes: task_id (str): Id of the task that ran. - state (str): Terminal state — ``"succeeded"`` / ``"failed"`` / - ``"needs_manual_review"``. + state (str): Terminal state — ``"succeeded"`` / ``"failed"``. result (dict): Executor result payload (empty on failure). error (str | None): Error string when the task failed, else None. """ task_id: str - state: str # "succeeded" / "failed" / "needs_manual_review" + state: str # "succeeded" / "failed" result: dict error: str | None = None @@ -149,37 +148,43 @@ async def _transition_resilient( *, evidence: dict | None = None, context: str, - ) -> bool: - """Transition a task to ``new_state`` but tolerate ``TaskNotFound``. - - A long-running task's row can vanish before its terminal - transition; swallowing TaskNotFound keeps the pipeline running. - Returns True on success, False on the swallowed-TaskNotFound branch. + allow_terminal: bool = False, + ) -> None: + """Transition a task to ``new_state``, tolerating a row lost to retention. Args: task_id: The task to transition. new_state: The target state. evidence: Optional evidence dict recorded with the transition. context: Short label describing the transition call site. + allow_terminal: Also tolerate an already-terminal row. Only for + terminal transitions; on ``queued -> running`` the rejection is + the double-spawn guard and must propagate. - Returns: - ``True`` on success, ``False`` on the swallowed-``TaskNotFound`` - branch. + Raises: + IllegalTransition: When the row is already terminal and + ``allow_terminal`` is False. """ try: await self.tasks.transition(task_id, new_state, evidence=evidence or {}) - return True except TaskNotFound: log.warning( "sub_agent_runner: tasks row for task_id=%s vanished before " - "transition→%s (context=%s); continuing so the executor " - "result is not lost. See sub_agent_runner._transition_" - "resilient docstring for the disappearing-row hypothesis.", + "transition→%s (context=%s); keeping the executor result", + task_id, + new_state, + context, + ) + except IllegalTransition: + if not allow_terminal: + raise + log.warning( + "sub_agent_runner: task_id=%s already terminal before " + "transition→%s (context=%s); keeping the executor result", task_id, new_state, context, ) - return False async def run_task( self, @@ -243,6 +248,7 @@ async def run_task( "failed", evidence={"reason": "no_executor", "kind": task.kind}, context="no_executor", + allow_terminal=True, ) if prebound_lease is not None: await self.locks.release(prebound_lease) @@ -283,6 +289,7 @@ async def run_task( "failed", evidence={"error": repr(exc)}, context="executor_exception", + allow_terminal=True, ) return SubAgentResult( task_id=task.task_id, @@ -295,6 +302,7 @@ async def run_task( "succeeded", evidence={"result_keys": sorted(result_payload.keys())}, context="executor_success", + allow_terminal=True, ) return SubAgentResult( task_id=task.task_id, diff --git a/src/hyperloom/orchestrator/phases/close.py b/src/hyperloom/orchestrator/phases/close.py index c803c6b9e..f02e571eb 100644 --- a/src/hyperloom/orchestrator/phases/close.py +++ b/src/hyperloom/orchestrator/phases/close.py @@ -550,5 +550,4 @@ async def _closing_report_terminal(self) -> bool: "succeeded", "failed", "cancelled", - "needs_manual_review", } diff --git a/src/hyperloom/orchestrator/phases/explore.py b/src/hyperloom/orchestrator/phases/explore.py index cff98a551..414faa0c8 100644 --- a/src/hyperloom/orchestrator/phases/explore.py +++ b/src/hyperloom/orchestrator/phases/explore.py @@ -14,13 +14,13 @@ from typing import Any from . import machine_state as _phase_state from hyperloom.common.coerce import to_float -from ..loop.coordinator_helpers import _parse_iso_unix from hyperloom.inference_optimizer.protocol.intent import Intent, IntentType from ..bus.message_bus import Message from ..policy.gate import ( SPECIALIST_FROM_AGENT_PREFIX, ) from ..loop.sub_agent_runner import SubAgentResult +from ..specialists.runner import SpecialistFailureType from ..state.task_registry import Task from ..loop.coordinator import ( FORCE_STALLED_KEEP_ROUNDS, @@ -610,40 +610,6 @@ def _seed_gaps_from_research_hints(self) -> None: cid, ) - async def _scan_stale_specialists(self) -> list[dict[str, Any]]: - """Return specialist task rows running longer than ``_specialist_stale_sec``; never raises, returns [] on failure. - - Returns: - A list of stale specialist task row dicts; empty on failure or when - none are stale. - """ - try: - running = await self.tasks.running() - except Exception: # noqa: BLE001 — defensive - log.exception("Coordinator: tasks.running() failed during stale scan") - return [] - if not running: - return [] - stale: list[dict[str, Any]] = [] - now_unix = time.time() - for t in running: - if (t.kind or "").strip() != "specialist": - continue - # updated_at on a running task = when the dispatcher promoted it. - started_unix = _parse_iso_unix(t.updated_at) - if started_unix <= 0: - continue - running_sec = max(0.0, now_unix - started_unix) - if running_sec >= self._specialist_stale_sec: - stale.append( - { - "task_id": t.task_id, - "kind": t.kind, - "running_seconds": running_sec, - } - ) - return stale - async def _fan_out_specialist_wave( self, source: str, @@ -702,6 +668,51 @@ async def _fan_out_specialist_wave( Intent(type=intent.type, payload=sub_payload), ) + async def _record_specialist_retry_exhausted( + self, + *, + task: "Task", + ftype: SpecialistFailureType, + error: str, + attempts_used: int, + cap: int, + detail: str, + ) -> None: + """Broadcast that an infra-failed specialist is being abandoned. + + Args: + task: The specialist task whose final attempt failed. + ftype: The classified failure type. + error: The failure reason carried by the attempt. + attempts_used: Retry attempts already spent. + cap: Configured retry ceiling. + detail: Why no further retry was scheduled. + """ + params = task.params or {} + await self._record_observation( + "coordinator", + "observation", + { + "kind": "specialist_auto_retry_exhausted", + "task_id": task.task_id, + "domain": str(params.get("domain") or ""), + "gap_canonical_id": str(params.get("gap_canonical_id") or ""), + "attempts_used": attempts_used, + "max_attempts": cap, + "failure_type": ftype.value, + "reason": error[:200], + "detail": detail, + }, + ) + log.warning( + "specialist auto-retry exhausted: task=%s failure=%s attempts=%d/%d (%s)", + task.task_id, + ftype.value, + attempts_used, + cap, + detail, + ) + async def _maybe_auto_retry_specialist( self, task: "Task", @@ -750,13 +761,23 @@ async def _maybe_auto_retry_specialist( result_dict = result.result if isinstance(result.result, dict) else {} runner_status = str(result_dict.get("runner_status") or "") - error = str(result.error or "") + # The specialist executor never raises, so the reason lives in the + # result envelope rather than on SubAgentResult. + error = str(result.error or result_dict.get("error") or "") ftype, retry_eligible = classify_specialist_failure(runner_status, error) if not retry_eligible: return False params = task.params or {} attempt = int(params.get("_auto_retry_attempt", 0) or 0) if attempt >= cap: + await self._record_specialist_retry_exhausted( + task=task, + ftype=ftype, + error=error, + attempts_used=attempt, + cap=cap, + detail="retry cap reached", + ) return False next_attempt = attempt + 1 @@ -804,6 +825,14 @@ async def _maybe_auto_retry_specialist( ) if was_existing: # Retry slot already taken: let normal bookkeeping record this attempt. + await self._record_specialist_retry_exhausted( + task=task, + ftype=ftype, + error=error, + attempts_used=attempt, + cap=cap, + detail="retry slot already taken", + ) return False await self._record_observation( "coordinator", @@ -1020,13 +1049,6 @@ async def _warm_specialist_params(self, params: dict[str, Any]) -> None: "hot_kernels_top15": hot_kernels, } - # proposal_set cap into params so SpecialistRunner reads it. - from hyperloom.orchestrator.policy.gate import ( - DEFAULT_SPECIALIST_MAX_PROPOSALS, - ) - - params.setdefault("max_proposals", DEFAULT_SPECIALIST_MAX_PROPOSALS) - async def _refresh_gaps(self, *, reason: str) -> None: """Refresh :attr:`SharedState.gaps` from observable signals. Additive upsert deduped by canonical_id; best-effort. @@ -1775,7 +1797,6 @@ def _build_specialist_round_entry( if not isinstance(proposals, list): proposals = [] round_id = str((task.params or {}).get("round_id") or task.task_id) - truncated_from = done_payload.get("proposals_truncated_from") from ..specialists.domains import normalize_dispatch_tags # Knowledge-domain tags; reported tags win over dispatch params. @@ -1804,6 +1825,4 @@ def _build_specialist_round_entry( entry["allocated_gpu_ids"] = [ int(g) for g in gpu_ids if isinstance(g, (int, str)) and str(g).strip().lstrip("-").isdigit() ] - if isinstance(truncated_from, int) and truncated_from > len(proposals): - entry["proposals_truncated_from"] = truncated_from return entry diff --git a/src/hyperloom/orchestrator/policy/gate.py b/src/hyperloom/orchestrator/policy/gate.py index 45c5cafc0..371ee478a 100644 --- a/src/hyperloom/orchestrator/policy/gate.py +++ b/src/hyperloom/orchestrator/policy/gate.py @@ -7,7 +7,6 @@ import logging import os -import re from dataclasses import dataclass from pathlib import Path from typing import TYPE_CHECKING, Any @@ -134,7 +133,7 @@ def __init__(self, reason: str, *, rule: str | None = None, hint: str | None = N } -# Specialist dispatch action name (central so R2 sub-rules enforce the contract uniformly). +# Specialist dispatch action name. SPECIALIST_ACTION_NAME: str = "specialist" # Orchestrator-side patch integration step (EXPLORE phase, gated by a Critic verdict). @@ -282,8 +281,6 @@ def _whole_machine_pool_size() -> int: return len(resolve_whole_machine_devices()) -DEFAULT_SPECIALIST_MAX_PROPOSALS: int = 12 - # Verdicts that allow ``integrate_patch`` without an operator override (``advise`` = soft approval, ``approve`` = green light). INTEGRATE_PATCH_PERMISSIVE_VERDICTS: frozenset[str] = frozenset( { @@ -299,21 +296,6 @@ def _whole_machine_pool_size() -> int: # research_lane capacity and the GPU specialist pool. SPECIALIST_FREEFORM_WAVE_MAX: int = 16 SPECIALIST_FREEFORM_TASK_DESC_MAX_CHARS: int = 8000 -# Fail-fast tripwire for obviously-destructive host commands in free-form task -# descriptions. NOT a security boundary — the worktree, Critic review, and -# integrate_patch gate remain the real boundaries. -_FREEFORM_REDLINE_PATTERNS: tuple[re.Pattern[str], ...] = ( - re.compile(r"\brm\s+-rf?\s+(?:/|~|\$HOME|\*)", re.IGNORECASE), - re.compile(r"\bmkfs\.", re.IGNORECASE), - re.compile(r"\bdd\s+if=.*\bof=/dev/", re.IGNORECASE), - re.compile(r">\s*/dev/sd[a-z]"), - re.compile(r":\(\)\s*\{.*\};\s*:"), # fork bomb - re.compile(r"\bshutdown\b|\breboot\b", re.IGNORECASE), - # Ban pipe-to-kill and killall (can kill the serving / benchmark process). - re.compile(r"\bps\s+(?:aux|-\w+)\b.*\|.*\bkill\b", re.IGNORECASE), - re.compile(r"\bpgrep\b.*\|.*\bkill\b", re.IGNORECASE), - re.compile(r"\bkillall\b", re.IGNORECASE), -) # Prefix the SubAgentRunner stamps on specialist emit-intents (``from_agent='specialist:'``). SPECIALIST_FROM_AGENT_PREFIX: str = "specialist:" @@ -356,25 +338,6 @@ def _whole_machine_pool_size() -> int: ALL_KNOWN_EXTERNAL_TOOL_NAMES: frozenset[str] = PR_MONITOR_TOOL_NAMES | WEB_TOOL_NAMES -# Synthetic stub used as ``role`` for specialist path-containment checks (only ``name`` is needed). -class _SpecialistPseudoRole: - """Minimal stand-in role used when path-validating specialist intents. - - Specialist intents are routed through ``_validate_specialist_*`` rather - than the conventional ``role.allowed_intents`` matrix, so the path - containment check only needs a ``name`` attribute for its error - messages. This stub supplies that single field. - - Attributes: - name (str): the synthetic role name, always ``"specialist"``. - """ - - name = "specialist" - - -_SPECIALIST_PSEUDO_ROLE = _SpecialistPseudoRole() - - # REQUEST/RESPONSE routing matrix: source role → allowed target_agents (only orchestration→kernel). REQUEST_ROUTING: dict[str, frozenset[str]] = { "orchestration": frozenset({"kernel_agent"}), @@ -396,10 +359,13 @@ class _SpecialistPseudoRole: ) -# Robustness-only: kill_task + scheduling-police intents -KILL_TASK_SOURCE_ALLOWLIST: frozenset[str] = frozenset({"robustness"}) +# kill_task sources; the other scheduling-police intents stay robustness-only. +KILL_TASK_SOURCE_ALLOWLIST: frozenset[str] = frozenset({"robustness", "orchestration"}) KILL_TASK_ALLOWED_SCOPES: frozenset[str] = frozenset({"task"}) +# Ceiling on a single extend_lease step; repeated extensions are allowed. +EXTEND_LEASE_MAX_SEC: int = 3600 + ROBUSTNESS_ONLY_INTENTS: frozenset[IntentType] = frozenset( { IntentType.PRUNE_BRANCH, @@ -558,7 +524,7 @@ def _resolved_within(value: str, root: str) -> bool: # operator-facing lifecycle event log; Coordinator-only writer so the # LLM cannot forge lifecycle events. "lifecycle", - # specialist sub-agent ledger; LLM cannot inject entries (proposals go via the R3 path). + # specialist sub-agent ledger; Coordinator-only writer. "specialist_rounds", # per-kb_anchor coverage counters; Coordinator-only writers. "rounds_since_last_specialist", @@ -657,25 +623,13 @@ def validate_intent(self, from_agent: str, intent: Intent) -> None: """Raise :class:`PolicyDenied` if the intent is not allowed (cheapest checks first: role → allowed_intents → structural → cross-source). Args: - from_agent (str): the identity of the emitting agent; a - ``specialist:`` prefix routes to the specialist - validators. + from_agent (str): the identity of the emitting agent. intent (Intent): the parsed intent to validate. Raises: PolicyDenied: when the intent is not permitted; the ``rule`` attribute identifies which guard fired. """ - # specialist sub-agents emit under an ephemeral ``specialist:`` identity routed to a synthetic role. - if from_agent.startswith(SPECIALIST_FROM_AGENT_PREFIX): - self._validate_specialist_intent(from_agent, intent) - self._validate_payload_paths( - _SPECIALIST_PSEUDO_ROLE, - intent.type, - intent.payload or {}, - ) - return - role = self.role_registry.get(from_agent) if role is None: raise PolicyDenied(f"unknown agent {from_agent!r}", rule="role") @@ -709,6 +663,8 @@ def validate_intent(self, from_agent: str, intent: Intent) -> None: self._validate_review_verdict(role, payload) elif intent.type == IntentType.KILL_TASK: self._validate_kill_task(role, payload) + elif intent.type == IntentType.EXTEND_LEASE: + self._validate_extend_lease(payload) elif intent.type in ROBUSTNESS_ONLY_INTENTS: self._validate_robustness_only(role, intent.type, payload) # ALERT carries no extra checks beyond the role gate. @@ -878,7 +834,7 @@ def _validate_delegate_body( f"of delegate(action_name={action_name!r})", rule="kernel_owned_by_kernel_agent", ) - # R2 ``specialist`` bypasses ActionRegistry; its contract is enforced by ``_validate_specialist_dispatch``. + # ``specialist`` bypasses ActionRegistry; ``_validate_specialist_dispatch`` owns its contract. if action_name == SPECIALIST_ACTION_NAME: self._validate_specialist_dispatch(role, payload) if check_phase: @@ -1528,7 +1484,7 @@ def _validate_specialist_dispatch( role: "AgentRole", payload: dict[str, Any], ) -> None: - """Enforce the specialist-delegate contract (Inv-11.2): orchestration-only, tags ∈ vocab, gap_canonical_id required, max_turns ≤ cap. + """Enforce the specialist-delegate contract (Inv-11.2): orchestration-only, gap_canonical_id required, max_turns ≤ cap. Args: role (AgentRole): the resolved role of the emitting agent. @@ -1537,8 +1493,8 @@ def _validate_specialist_dispatch( Raises: PolicyDenied: when the role may not dispatch, params are malformed, - tags are unknown, the scope is invalid, the gap id is missing, - or max_turns is out of range. + the gap id is missing, or max_turns exceeds the hard cap. Tag / + scope incoherence is logged rather than denied. """ if role.name not in SPECIALIST_DISPATCH_SOURCE_ALLOWLIST: raise PolicyDenied( @@ -1576,49 +1532,28 @@ def _validate_specialist_dispatch( self._validate_freeform_specialist_dispatch(params) return + # Observed, not enforced: resolve_specialist_profile re-infers the scope + # and the runner synthesizes an empty result for an unresolvable anchor. if not tags: - raise PolicyDenied( - "delegate{action='specialist'}: at least one tag is " - "required (params.tags or the legacy params.domain alias)", - rule="specialist_unknown_domain", - hint=(f"set params.tags to a non-empty subset of {sorted(KNOWLEDGE_DOMAIN_TAG_SET)!r}"), - ) - # Each tag must belong to the controlled knowledge-domain vocabulary. + log.info("specialist dispatch declares a scope but no tags; profile will re-infer") unknown_tags = [t for t in tags if t not in KNOWLEDGE_DOMAIN_TAG_SET] if unknown_tags: - raise PolicyDenied( - f"delegate{{action='specialist'}}: unknown knowledge-domain tag(s)={unknown_tags!r}", - rule="specialist_unknown_domain", - hint=(f"every tag must be one of {sorted(KNOWLEDGE_DOMAIN_TAG_SET)!r}"), + log.info( + "specialist dispatch carries out-of-vocabulary tag(s)=%r (known: %r)", + unknown_tags, + sorted(KNOWLEDGE_DOMAIN_TAG_SET), ) - # ``scope`` dial (domain | domains | freeform). Absent => single-domain - # default. ``domains`` requires >1 distinct tag; ``domain`` is single-tag. - scope = str(params.get("scope") or "").strip().lower() - if scope and scope not in SPECIALIST_SCOPE_VALUES: - raise PolicyDenied( - f"delegate{{action='specialist'}}: unknown scope={scope!r}", - rule="specialist_scope_invalid", - hint=(f"scope must be one of {sorted(SPECIALIST_SCOPE_VALUES)!r}"), - ) - if scope == SPECIALIST_SCOPE_DOMAINS and len(tags) < 2: - raise PolicyDenied( - "delegate{action='specialist'}: scope='domains' is the " - "cross-domain channel and requires at least 2 distinct " - f"tags; got {tags!r}", - rule="specialist_scope_too_narrow", - hint=( - "Declare every domain the patch must touch together in " - "params.tags, or use scope='domain' for a single-domain " - "specialist." - ), - ) - if scope == SPECIALIST_SCOPE_DOMAIN and len(tags) > 1: - raise PolicyDenied( - f"delegate{{action='specialist'}}: scope='domain' is single-domain but got {len(tags)} tags {tags!r}", - rule="specialist_scope_mismatch", - hint=("Use scope='domains' for a cross-domain specialist, or pass a single tag."), + if scope_raw and scope_raw not in SPECIALIST_SCOPE_VALUES: + log.info( + "specialist dispatch scope=%r not in %r; re-inferred from tags", + scope_raw, + sorted(SPECIALIST_SCOPE_VALUES), ) + elif scope_raw == SPECIALIST_SCOPE_DOMAINS and len(tags) < 2: + log.info("specialist dispatch scope='domains' with %d tag(s)=%r", len(tags), tags) + elif scope_raw == SPECIALIST_SCOPE_DOMAIN and len(tags) > 1: + log.info("specialist dispatch scope='domain' with %d tags=%r", len(tags), tags) gap = str(params.get("gap_canonical_id") or params.get("gap") or "").strip() if not gap: @@ -1644,15 +1579,15 @@ def _validate_specialist_dispatch( f"delegate{{action='specialist'}}: max_turns must be int, got {max_turns_raw!r}", rule="specialist_dispatch_source", ) from exc - # ``max_turns=0`` means unbounded (depth bounded by wall-clock); - # negatives and values above the hard cap are rejected. - if max_turns < 0 or max_turns > SPECIALIST_MAX_TURNS_HARD_CAP: + # The in-process backend's turn loop has no wall-clock check, so this + # cap is the only bound on that path. + if max_turns > SPECIALIST_MAX_TURNS_HARD_CAP: raise PolicyDenied( f"delegate{{action='specialist'}}: max_turns={max_turns} " - f"outside [0, {SPECIALIST_MAX_TURNS_HARD_CAP}]", + f"exceeds the hard cap {SPECIALIST_MAX_TURNS_HARD_CAP}", rule="specialist_dispatch_source", hint=( - f"max_turns must be in [0, {SPECIALIST_MAX_TURNS_HARD_CAP}] " + f"max_turns must be <= {SPECIALIST_MAX_TURNS_HARD_CAP} " "(0 = unbounded; depth is bounded by the wall-clock " "budget, so omit max_turns unless capping a probe early)." ), @@ -1714,11 +1649,10 @@ def _validate_specialist_gpu_request(self, params: dict[str, Any]) -> None: gpu_count_raw = default_gpu_count try: gpu_count = int(gpu_count_raw) - except (TypeError, ValueError) as exc: - raise PolicyDenied( - f"delegate{{action='specialist'}}: gpu_count must be an integer, got {gpu_count_raw!r}", - rule="specialist_gpu_request_invalid", - ) from exc + except (TypeError, ValueError): + # The dispatcher re-parses with the same default. + log.info("specialist dispatch gpu_count=%r not an integer; using %d", gpu_count_raw, default_gpu_count) + gpu_count = int(default_gpu_count) if gpu_count <= 0: raise PolicyDenied( "delegate{action='specialist'}: gpu_count must be > 0 when needs_gpu=true", @@ -1853,34 +1787,24 @@ def _validate_freeform_specialist_dispatch( specialists. Free-form dispatches carry no domain/tag/gap anchor, so this validates only structural shape: a single ``task_description`` or a ``tasks=[...]`` wave (bounded by SPECIALIST_FREEFORM_WAVE_MAX), each - with a non-empty, length-bounded description that survives the - red-line tripwire. + with a non-empty, length-bounded description. Args: params (dict[str, Any]): the freeform dispatch ``params`` carrying a single ``task_description`` or a ``tasks`` wave. Raises: - PolicyDenied: when the GPU request fails, the wave is malformed or - too large, or a task description is empty / too long / trips the - red-line tripwire. + PolicyDenied: when the GPU request fails, the wave is too large, or + a task description is empty / too long. """ # Freeform skips the domain-anchored max_turns gate (bounded by the task # timeout instead), but a GPU request must still clear the same pool # ceiling as a domain specialist. self._validate_specialist_gpu_request(params) wave = params.get("tasks") - if wave is not None: - if not isinstance(wave, list) or not wave: - raise PolicyDenied( - "delegate{action='specialist',scope='freeform'}: params.tasks must be a non-empty list", - rule="specialist_freeform_wave_invalid", - hint=( - "Pass tasks=[{task_description: ...}, ...] or a single " - "params.task_description for a one-off freeform " - "specialist." - ), - ) + # A malformed or empty wave falls through to the single-task path in the + # fan-out, which re-checks shape per entry. + if isinstance(wave, list) and wave: if len(wave) > SPECIALIST_FREEFORM_WAVE_MAX: raise PolicyDenied( f"delegate{{action='specialist',scope='freeform'}}: wave " @@ -1891,20 +1815,17 @@ def _validate_freeform_specialist_dispatch( ) for i, task in enumerate(wave): if not isinstance(task, dict): - raise PolicyDenied( - f"delegate{{action='specialist',scope='freeform'}}: tasks[{i}] must be an object", - rule="specialist_freeform_task_invalid", - ) + continue desc = str(task.get("task_description") or task.get("task_summary") or "").strip() - self._check_freeform_task_description(desc, where=f"tasks[{i}]") + if desc: + self._check_freeform_task_description(desc, where=f"tasks[{i}]") return desc = str(params.get("task_description") or "").strip() self._check_freeform_task_description(desc, where="params") @staticmethod def _check_freeform_task_description(desc: str, *, where: str) -> None: - """Per-task structural checks for a free-form ``task_description``: - non-empty, length-bounded, and clear of the red-line tripwire. + """Per-task structural checks for a free-form ``task_description``: non-empty and length-bounded. Args: desc (str): the freeform task description to validate. @@ -1912,8 +1833,7 @@ def _check_freeform_task_description(desc: str, *, where: str) -> None: messages. Raises: - PolicyDenied: when ``desc`` is empty, exceeds the length cap, or - matches a red-line pattern. + PolicyDenied: when ``desc`` is empty or exceeds the length cap. """ if not desc: raise PolicyDenied( @@ -1928,173 +1848,45 @@ def _check_freeform_task_description(desc: str, *, where: str) -> None: f"{SPECIALIST_FREEFORM_TASK_DESC_MAX_CHARS}", rule="specialist_freeform_description_too_long", ) - for pat in _FREEFORM_REDLINE_PATTERNS: - if pat.search(desc): - raise PolicyDenied( - f"delegate{{action='specialist',scope='freeform'}}: " - f"{where} task_description tripped the red-line scan " - f"(pattern={pat.pattern!r})", - rule="specialist_freeform_redline", - hint=( - "Free-form mandates must not embed destructive host " - "commands. Describe the investigation, not raw " - "destructive shell." - ), - ) - # R3 ``specialist_done_source`` - def _validate_specialist_intent( - self, - from_agent: str, - intent: Intent, - ) -> None: - """Validate any intent emitted under a ``specialist:`` identity (Inv-5.2: only SEND_MESSAGE, ALERT, and one SPECIALIST_DONE). + def _validate_extend_lease(self, payload: dict[str, Any]) -> None: + """Validate an ``EXTEND_LEASE`` intent. Args: - from_agent (str): the ``specialist:`` identity of the - emitter. - intent (Intent): the intent emitted under that identity. + payload (dict[str, Any]): the payload carrying ``task_id``, + ``extra_sec`` and an optional ``reason``. Raises: - PolicyDenied: when the task_id suffix is missing or the intent type - is not one a specialist may emit. + PolicyDenied: when ``task_id`` is missing or ``extra_sec`` is not a + positive integer within :data:`EXTEND_LEASE_MAX_SEC`. """ - task_id = from_agent.removeprefix(SPECIALIST_FROM_AGENT_PREFIX).strip() + task_id = str(payload.get("task_id", "")).strip() if not task_id: + raise PolicyDenied("extend_lease missing task_id", rule="payload") + try: + extra_sec = int(payload.get("extra_sec") or 0) + except (TypeError, ValueError) as exc: raise PolicyDenied( - f"specialist from_agent missing task_id suffix (got {from_agent!r})", - rule="specialist_done_source", - hint=( - "Specialist sub-agents must stamp " - "from_agent='specialist:' where " - "matches the dispatched task." - ), - ) - if intent.type == IntentType.SPECIALIST_DONE: - self._validate_specialist_done_payload(task_id, intent.payload or {}) - return - # Allowed ancillary intents (heartbeat / advice / alert). - if intent.type in ( - IntentType.SEND_MESSAGE, - IntentType.ALERT, - ): - return - raise PolicyDenied( - f"specialist={from_agent!r} cannot emit intent_type={intent.type.value!r}", - rule="specialist_done_source", - hint=( - "Specialists may only emit specialist_done (exit), " - "send_message (heartbeat/advice), or alert. Use " - "specialist_done with proposal_set + summary instead." - ), - ) - - def _validate_specialist_done_payload( - self, - task_id: str, - payload: dict[str, Any], - ) -> None: - """Per-field R3 structural checks for the ``specialist_done`` payload (gap_canonical_id, domain ∈ keys, proposal_set, empty+reason, summary ≤4096, confidence ∈ [0,1]). - - Args: - task_id (str): the specialist task id taken from the emitting - identity. - payload (dict[str, Any]): the specialist_done payload to validate. - - Raises: - PolicyDenied: when any required field is missing or malformed (gap - id, domain, proposal_set, empty+reason, summary length, or - confidence range). - """ - gap = str(payload.get("gap_canonical_id") or "").strip() - if not gap: + f"extend_lease extra_sec must be an integer, got {payload.get('extra_sec')!r}", + rule="payload", + ) from exc + if extra_sec <= 0 or extra_sec > EXTEND_LEASE_MAX_SEC: raise PolicyDenied( - "specialist_done missing gap_canonical_id", - rule="specialist_done_source", + f"extend_lease extra_sec={extra_sec} outside (0, {EXTEND_LEASE_MAX_SEC}]", + rule="extend_lease_bounds", hint=( - "Payload must echo the gap_canonical_id that was " - "passed to delegate{action='specialist'} so " - "Coordinator can cross-check the dispatch." + "Extend in bounded steps and re-check get_running_tasks; " + "a lease must not outlive the session budget." ), ) - domain = str(payload.get("domain") or "").strip() - if not domain: - raise PolicyDenied( - "specialist_done missing domain", - rule="specialist_done_source", - ) - if domain not in SPECIALIST_DOMAIN_KEYS: - raise PolicyDenied( - f"specialist_done: unknown domain={domain!r}", - rule="specialist_done_source", - hint=(f"domain must be one of {sorted(SPECIALIST_DOMAIN_KEYS)!r}"), - ) - proposal_set = payload.get("proposal_set") - if not isinstance(proposal_set, list): - raise PolicyDenied( - "specialist_done.proposal_set must be a list", - rule="specialist_done_source", - hint="set proposal_set=[] when empty=true", - ) - empty_flag = bool(payload.get("empty")) - if empty_flag: - if proposal_set: - raise PolicyDenied( - "specialist_done: empty=true implies proposal_set=[]", - rule="specialist_done_source", - ) - reason_field = str(payload.get("reason") or payload.get("summary") or "").strip() - if not reason_field: - raise PolicyDenied( - "specialist_done: empty=true requires a reason / summary describing why no proposals were emitted", - rule="specialist_done_source", - ) - else: - for i, variant in enumerate(proposal_set): - if not isinstance(variant, dict): - raise PolicyDenied( - f"specialist_done.proposal_set[{i}] must be a dict", - rule="specialist_done_source", - ) - if not str(variant.get("name") or "").strip(): - raise PolicyDenied( - f"specialist_done.proposal_set[{i}].name required", - rule="specialist_done_source", - hint=( - "Every variant needs a unique name " - "(round-scoped). See §3.4 §5.1 for the full " - "variant schema." - ), - ) - summary = str(payload.get("summary") or "") - if len(summary) > 4096: - raise PolicyDenied( - f"specialist_done.summary too long ({len(summary)} > 4096 chars)", - rule="specialist_done_source", - hint="KB_design §3.5 §7 caps summary at ~500 chars; 4096 is the defensive hard limit.", - ) - confidence_raw = payload.get("confidence") - if confidence_raw is not None: - try: - confidence = float(confidence_raw) - except (TypeError, ValueError) as exc: - raise PolicyDenied( - f"specialist_done.confidence must be float, got {confidence_raw!r}", - rule="specialist_done_source", - ) from exc - if not 0.0 <= confidence <= 1.0: - raise PolicyDenied( - f"specialist_done.confidence={confidence} not in [0, 1]", - rule="specialist_done_source", - ) def _validate_kill_task(self, role: "AgentRole", payload: dict[str, Any]) -> None: - """Validate a ``KILL_TASK`` intent (robustness-only). + """Validate a ``KILL_TASK`` intent. Requires the source role to be on :data:`KILL_TASK_SOURCE_ALLOWLIST`, a non-empty ``task_id`` and ``reason``, and a ``scope`` within :data:`KILL_TASK_ALLOWED_SCOPES` - (``task`` only — server/process kills stay out per IR-5). + (``task`` only — server/process kills stay out). Args: role (AgentRole): the resolved role of the emitting agent. @@ -2391,6 +2183,7 @@ def to_policy_denial_summary(state, *, top_k: int = 6) -> str: "CORE_STATE_FIELDS", "DELEGATE_ACTION_REQUIRED_PAYLOAD", "DELEGATE_ACTION_SOURCE_ALLOWLIST", + "EXTEND_LEASE_MAX_SEC", "INTERNAL_ONLY_ACTION_NAMES", "KERNEL_AGENT_OWNED_ACTIONS", "KILL_TASK_ALLOWED_SCOPES", diff --git a/src/hyperloom/orchestrator/prompts/orchestration.md b/src/hyperloom/orchestrator/prompts/orchestration.md index 2d6c5ed43..cfca280ab 100644 --- a/src/hyperloom/orchestrator/prompts/orchestration.md +++ b/src/hyperloom/orchestrator/prompts/orchestration.md @@ -20,17 +20,8 @@ is usually a **thin delta**, not a full state dump: (recovered) ===` block summarising your own prior plan. - Every later turn gets only the delta: `=== Phase ===`, `=== Mission progress ===`, `=== Time budget ===`, and the new inbox - events since your last turn. A `=== Context (pull on demand) ===` - note marks these delta turns. - -On a delta turn the verbose state is intentionally NOT re-pasted. **Pull -exactly what you need** with the read-only context tools: -`get_shared_state`, `get_gaps`, `get_warm_start`, `get_proposal_scores`, -`get_intervention_mix`, `why_denied`, `show_analysis_md`, `get_inbox`, -`get_recent_outcomes` (and `Read` for sandboxed files). They return the -same projections the old prompt used to push. Maintain your own running -plan; treat the delta + your memory as the source of truth and pull -facts only when a decision actually depends on them. + events since your last turn. A `=== Context (pull on demand) ===` note + marks these delta turns. ### Web search (upstream comparison) @@ -55,6 +46,11 @@ waiting for the next tick: outcomes (kind / state / status / kept / gain / tput / error) plus review verdicts. Use this to check how your prior delegated work landed before deciding the next move, instead of re-emitting blindly. +- **`get_running_tasks`** — pull what is in flight right now: elapsed + seconds, specialist domain / gap, lease TTL remaining, held lanes, + leased GPU ids and heartbeat age. `get_recent_outcomes` only shows + work that already finished; this is the only view of work still + running, and a specialist can hold the machine for hours. - **`run_action_now{action_name, params}`** — run a CHEAP, lane-light action synchronously and get its result back IN THIS TURN. Only a small whitelist of fast, non-GPU / non-serving actions is eligible @@ -76,6 +72,52 @@ of your working memory; it persists that and re-seeds a fresh conversation from it so the context stays bounded on long runs. Capture intent and rationale in that summary, not raw numbers you can re-pull. +### Watching a running specialist + +Nothing in this message reports in-flight specialists: the prompt renders +between blocking actions, so a running specialist is exactly what you are +waiting on and never appears here. Never read silence as "nothing is +running". Two signals do reach you: + +- **`specialist_progress` inbox observations** — pushed whenever a + specialist rewrites its checkpoint, carrying `task_id`, `elapsed_sec`, + summary, proposal count, findings and `residual_questions`. Sparse (often + 2-3 per specialist, the first lagging dispatch by minutes), so read each + as a sample of work that has been running unobserved. +- **`get_running_tasks`** — the live view (see above). Call it whenever a + `specialist_progress` lands, before a phase change, and when a stretch of + turns has passed with no specialist news. + +Elapsed time alone decides nothing: an offline autotune legitimately runs +for an hour, a five-minute agent can already be wedged. Judge on what you +asked for, whether successive checkpoints advance or repeat, and what is +queued behind the lane or GPUs it holds. Three moves: + +- `kill_task{task_id, scope='task', reason}` — cancel the coordinator task. + This does **not** terminate an already-running specialist process: its lane + and GPU leases release only when its worker exits or its reaper terminates + it. Do not use this to promptly free capacity; use it when the mandate is a + dead end and the remaining task budget is not worth spending. +- `send_message{to='specialist:', body_md}` — lands in its inbox + and it acts without restarting. Prefer this when the agent works well but + on the wrong question, or to answer its `residual_questions`. +- `extend_lease{task_id, extra_sec, reason}` — grows the lease TTL and its + lane rows, in bounded steps. For live work near expiry that the TTL + watchdog would otherwise fail out. + +Doing nothing is a legitimate choice; doing nothing because nothing +prompted you is not. + +On a delta turn the verbose state is intentionally NOT re-pasted. **Pull +exactly what you need** with the read-only context tools: +`get_shared_state`, `get_gaps`, `get_warm_start`, `get_proposal_scores`, +`get_intervention_mix`, `why_denied`, `show_analysis_md`, `get_inbox`, +`get_recent_outcomes`, `get_running_tasks` (and `Read` for sandboxed +files). They return the +same projections the old prompt used to push. Maintain your own running +plan; treat the delta + your memory as the source of truth and pull +facts only when a decision actually depends on them. + ### Phase awareness The 6-phase chain, per-phase allowed actions, and transition gates are in @@ -242,9 +284,11 @@ on the next tick. `integrate_patch`, is one route worth weighing against another config round. A `code_patch` KEEP resets the consecutive counter. * **You CANNOT** delegate kernel_agent-owned actions; mutate core state fields - (`current_best` / `stop_reason` / `baseline_tput` / ...); emit - `kill_task` (Robustness-only); read or write KB - directly (Critic owns it). You **CAN** emit `escalate_strategy_change` + (`current_best` / `stop_reason` / `baseline_tput` / ...); read or write KB + directly (Critic owns it). You **CAN** emit `kill_task` with + `scope='task'` to reap work you dispatched (server / process kills stay + out — those go through Robustness `delegate(recover)`), and + `escalate_strategy_change` with a phase-advance / budget hint (`skip_to_kernel` / `skip_to_sweep` / `skip_to_close` / `extend_explore_budget` / `extend_kernel_budget`) — PolicyGate allows this intent from both Robustness and Orchestration — @@ -317,6 +361,13 @@ defaults the rest; omitting a dial is safe): Both `bench` and `needs_gpu` acquire `gpu_research_lane` (see Phase awareness — GPU specialists serialize against serving). +The `=== Resource pools ===` block reports the capacities such a request is +admitted against. A `bench` / framework-authoring specialist admits against +`whole_machine_gpu_pool`; any other `needs_gpu` specialist admits against +`serving_disjoint_gpu_pool`, which is `serving_tp` cards smaller and is `0` +whenever serving owns every card — in that case dispatch CPU specialists, or +use `bench` when the work genuinely needs to measure. + **Domain-anchored example:** ``` emit_intent({ diff --git a/src/hyperloom/orchestrator/prompts/specialist_prompt_builder.py b/src/hyperloom/orchestrator/prompts/specialist_prompt_builder.py index 7abe53b48..4c8cb907d 100644 --- a/src/hyperloom/orchestrator/prompts/specialist_prompt_builder.py +++ b/src/hyperloom/orchestrator/prompts/specialist_prompt_builder.py @@ -49,13 +49,6 @@ ) -# Soft cap on ``proposal_set`` size; re-exported so the prompt-side cap and the -# runner-side hard truncate stay aligned. -from hyperloom.orchestrator.policy.gate import ( - DEFAULT_SPECIALIST_MAX_PROPOSALS, -) - - # Per-domain focus templates: each injects a "Domain focus" block into # Section 1; a missing key falls back to the generic body. @@ -692,8 +685,6 @@ class SpecialistPromptInputs: task_id: str domain: SpecialistDomain max_turns: int = DEFAULT_SPECIALIST_MAX_TURNS - # Soft cap on ``proposal_set`` size (rendered into Sections 1 + 8). - max_proposals: int = DEFAULT_SPECIALIST_MAX_PROPOSALS # ``tp`` defaults to 0 (sentinel for "unspecified"), not 1, so # comm_specialist doesn't veto its own TP proposals. @@ -823,7 +814,7 @@ def _section_identity(inp: SpecialistPromptInputs) -> list[str]: "to be thorough. Be creative. Investigate deeply. One-turn shortcuts", "are discouraged when a real bottleneck is on the table. Quality is", "scored over quantity: cap your final ``proposal_set`` at the", - f"**top-{inp.max_proposals}** ranked picks (see Section 8).", + "**top-6** ranked picks (see Section 8).", "", "Division of labour: the Coordinator owns the serving GPU, runs the E2E", "benchmark, and decides KEEP/REVERT — you do not have to validate final", @@ -929,9 +920,8 @@ def _gpu_autonomy_block(inp: SpecialistPromptInputs) -> list[str]: "- Write and run arbitrary scripts — autotune harnesses, " + "microbenchmarks, profilers (rocprof / torch.profiler / your own " + "breakdown).", - "- Start / restart a real server on your own cards (any port that is " - + "NOT the production serving port 8888) and benchmark it however you " - + "see fit.", + "- Start / restart a real server on your own cards and benchmark it " + + "however you see fit.", "- Profile freely to get a fresh trace after a change — don't rely only " + "on the static roofline snapshot you were handed.", "- Tune the framework's config-file levers (e.g. MoE/GEMM/attention " @@ -942,9 +932,8 @@ def _gpu_autonomy_block(inp: SpecialistPromptInputs) -> list[str]: + "not a requirement.", "", "Optional helper: a ``rebench`` convenience reuses the real Magpie " - + "serving + benchmark path on your leased cards + a non-8888 port, so " - + "you can get numbers directly comparable to the ``integrate_patch`` " - + "gate in one call:", + + "serving + benchmark path on your leased cards, so you can get numbers " + + "directly comparable to the ``integrate_patch`` gate in one call:", " python -m hyperloom.orchestrator.specialists.rebench \\", " --config --output ./scratch/rebench " + "[--extra-args '']", " It prints a JSON result with ``output_throughput``. It is OPTIONAL " @@ -1748,6 +1737,14 @@ def _section_output_protocol(inp: SpecialistPromptInputs) -> list[str]: "for that file and treats its appearance as the run's exit", "signal. After writing it, stop — do not call any further tools.", "", + "**Messages from the Orchestrator (check this as you work):** read", + f"``{workspace}/inbox.json`` whenever you finish a step. It is a JSON", + "list of ``{from, ts, body}`` entries, absent until the Orchestrator", + "sends one. It is how the Orchestrator answers a question you raised", + "or redirects you mid-run — if it tells you the mandate changed,", + "follow it rather than finishing the original plan. Never write to", + "this file.", + "", "**Incremental checkpoint (do this throughout the run):** every time", "you reach a new finding or finish a candidate, rewrite your", "best-so-far payload to", @@ -1757,7 +1754,10 @@ def _section_output_protocol(inp: SpecialistPromptInputs) -> list[str]: "partial uses the **same payload schema** as the final file but does", "**NOT** end the run — keep working. There is a wall-clock budget; if", "you are stopped before finishing, whatever is in the partial is", - "preserved as your result, so keep it current. Write the final", + "preserved as your result, so keep it current. The Orchestrator also", + "reads each rewrite while you are still running — it is how you report", + "direction and raise ``residual_questions`` early enough to get an", + "answer back through ``inbox.json``. Write the final", "``specialist_done.json`` (which ends the run) only once, as your", "absolute last action.", "", @@ -1823,12 +1823,11 @@ def _section_output_protocol(inp: SpecialistPromptInputs) -> list[str]: "coupling across several proposals." ), ( - f"- ``proposal_set`` MUST contain AT MOST **{inp.max_proposals}** " - "entries. You are a curator, not a brainstormer: rank candidates " - "by expected gain x your confidence, drop everything that " - "contradicts ``kb_subgraph`` / ``pr_evidence`` already in " - f"your prompt, and only emit the surviving top {inp.max_proposals}. " - "Fewer is better than padding." + "- ``proposal_set`` MUST contain AT MOST **6** entries. You are a " + "curator, not a brainstormer: rank candidates by expected gain x " + "your confidence, drop everything that contradicts ``kb_subgraph`` " + "/ ``pr_evidence`` already in your prompt, and only emit the " + "surviving top 6. Fewer is better than padding." ), ( "- The Critic reviews each surviving variant against the KB " @@ -1892,12 +1891,11 @@ def _section_iron_rules(inp: SpecialistPromptInputs) -> list[str]: gpu_rule = [ f"1. You EXCLUSIVELY own GPU card(s) [{cards}] for this task. On", " those cards do whatever you want: edit code, build, start/stop", - " your own servers (on any port that is NOT 8888), profile,", - " autotune, install tuned artifacts, run real benchmark loops.", - " The ONE thing you must NOT do: touch the production serving", - " process, its cards, or port 8888 — co-residing on them would", - " corrupt both your measurement and production. Manage only", - " processes YOU started, by their own PID/PGID.", + " your own servers, profile, autotune, install tuned artifacts,", + " and run real benchmark loops. The ONE thing you must NOT do:", + " touch the production serving process or its cards — co-residing", + " on them would corrupt both your measurement and production.", + " Manage only processes YOU started, by their own PID/PGID.", ] else: gpu_rule = [ @@ -1908,7 +1906,7 @@ def _section_iron_rules(inp: SpecialistPromptInputs) -> list[str]: " try and optionally author patches.", ] return [ - "## 9. IRON RULES (Inv-5.1 / Inv-5.2 / Inv-5.3)", + "## 9. IRON RULES (Inv-5.1 / Inv-5.3)", "", *gpu_rule, "2. **You MAY** produce changes for integration, but stage them ONLY", @@ -1934,8 +1932,8 @@ def _section_iron_rules(inp: SpecialistPromptInputs) -> list[str]: " read context is pre-warmed into Section 4 of this prompt; the", " specialist subprocess has no live KB connection.", "4. **NEVER** emit any intent other than ``specialist_done``,", - " ``send_message`` (heartbeat), or ``alert``. Any other intent", - " type triggers PolicyGate R3 ``specialist_done_source``.", + " ``send_message`` (heartbeat), or ``alert``. Other intent types", + " are dropped and recorded as a tool violation.", "5. You **MUST** finish within ``max_turns`` LLM turns and end with", " a single ``specialist_done`` exit signal (intent OR", " ``specialist_done.json`` file write per Section 8). Sub-agent", diff --git a/src/hyperloom/orchestrator/roles/agent_role.py b/src/hyperloom/orchestrator/roles/agent_role.py index 00f0b8b7f..32b674e0c 100644 --- a/src/hyperloom/orchestrator/roles/agent_role.py +++ b/src/hyperloom/orchestrator/roles/agent_role.py @@ -17,7 +17,7 @@ │ name │ backend │ allowed intents (high level) │ ├──────────────┼──────────┼─────────────────────────────────────────┤ │ orchestration│ Claude │ propose_action / delegate / request / │ - │ │ │ update_state / send_message / ... │ + │ │ │ update_state / kill_task / ... │ │ kernel │ Claude │ response (only) / send_message / alert │ │ critic │ Codex │ review_verdict (only) / send_message / │ │ │ no-tools │ alert │ @@ -71,6 +71,8 @@ class BackendType(str, Enum): IntentType.DELEGATE, IntentType.UPDATE_STATE, IntentType.REQUEST, + IntentType.KILL_TASK, + IntentType.EXTEND_LEASE, IntentType.PRUNE_BRANCH, IntentType.ESCALATE_STRATEGY_CHANGE, } diff --git a/src/hyperloom/orchestrator/roles/claude.py b/src/hyperloom/orchestrator/roles/claude.py index e2cf5b343..0392264b6 100644 --- a/src/hyperloom/orchestrator/roles/claude.py +++ b/src/hyperloom/orchestrator/roles/claude.py @@ -63,7 +63,7 @@ {{ "intent_type": "", "payload": {{ /* per-intent fields — see tool description */ }} }} diff --git a/src/hyperloom/orchestrator/roles/mcp_context_tools.py b/src/hyperloom/orchestrator/roles/mcp_context_tools.py index 474437dbc..7181419db 100644 --- a/src/hyperloom/orchestrator/roles/mcp_context_tools.py +++ b/src/hyperloom/orchestrator/roles/mcp_context_tools.py @@ -47,6 +47,7 @@ class ContextProvider: analysis_reader: Callable[[], str] | None = None denial_reader: Callable[[int], str] | None = None recent_outcomes_reader: Callable[[int], str] | None = None + running_tasks_reader: Callable[[], str] | None = None # Whitelisted lane-light action runner; ``None`` => unavailable. action_runner: Callable[[str, dict[str, Any]], str] | None = None @@ -170,6 +171,17 @@ def recent_outcomes(self, top_k: int = 8) -> str: return "(recent outcomes reader not wired)" return self._safe(lambda: self.recent_outcomes_reader(top_k), "recent_outcomes") + def running_tasks(self) -> str: + """Return the in-flight task set with the resources each one holds. + + Returns: + The running-task summary, or a not-wired marker when no reader is + bound. + """ + if self.running_tasks_reader is None: + return "(running tasks reader not wired)" + return self._safe(self.running_tasks_reader, "running_tasks") + def run_action_now( self, action_name: str = "", @@ -284,7 +296,7 @@ def run_action_now( ( "get_inbox", "Return inbox events addressed to orchestration. Pass since_seq to " - "page from a given sequence; omit for the recent tail.", + "page from a given sequence; omit for the full history.", _SINCE_SCHEMA, "inbox", ), @@ -298,6 +310,17 @@ def run_action_now( _TOPK_SCHEMA, "recent_outcomes", ), + ( + "get_running_tasks", + "Return the tasks currently in flight and what each one holds: " + "elapsed running seconds, specialist domain / gap, idempotency key, " + "lease TTL and remaining time, held lanes, leased GPU ids, and " + "heartbeat age. A dispatched task is otherwise invisible until it " + "terminates, so use this to judge whether to keep waiting on it, " + "plan around it, or escalate.", + _NO_ARGS_SCHEMA, + "running_tasks", + ), ( "run_action_now", "Run a CHEAP, lane-light action synchronously and get its result " diff --git a/src/hyperloom/orchestrator/roles/mcp_emit_intent.py b/src/hyperloom/orchestrator/roles/mcp_emit_intent.py index bc857bc3c..71f81ee89 100644 --- a/src/hyperloom/orchestrator/roles/mcp_emit_intent.py +++ b/src/hyperloom/orchestrator/roles/mcp_emit_intent.py @@ -48,6 +48,7 @@ "review_verdict:{target_proposal_msg_id,verdict ∈ " "approve|reject|redirect|advise|needs_review}, " "kill_task:{task_id,reason}, " + "extend_lease:{task_id,extra_sec,reason}, " "prune_branch:{family,reason}, escalate_strategy_change:" "{reason,next_action_hint}, update_state:{changes}, " "alert:{severity,summary}." diff --git a/src/hyperloom/orchestrator/specialists/domains.py b/src/hyperloom/orchestrator/specialists/domains.py index bff5907ce..e54add3b1 100644 --- a/src/hyperloom/orchestrator/specialists/domains.py +++ b/src/hyperloom/orchestrator/specialists/domains.py @@ -4,14 +4,15 @@ """Specialist sub-agent domain catalogue. LLM sub-agent form factor parameterized by a ``domain`` — a stable -id used by PolicyGate R2. A runtime constant, not per-domain yaml. +id used across dispatch and prompt assembly. A runtime constant, not +per-domain yaml. Field reference: * ``key`` — canonical id used in ``delegate{params.domain}``. * ``layer`` — short human label (analysis layer covered). * ``kb_anchor`` — knowledge-domain label for prompt grouping. -* ``available_in`` — ``"M5"`` / ``"M6"``; PolicyGate R2 accepts both. +* ``available_in`` — ``"M5"`` / ``"M6"``; both are dispatchable. All specialists share the global :data:`PR_QUERY_REPOS` allowlist and query the PR Monitor directly via ``mcp__pr_monitor__*`` tools. @@ -63,7 +64,7 @@ class SpecialistDomain: ) -# Canonical catalogue; PolicyGate R2's `specialist_unknown_domain` rule reads this set. +# Canonical catalogue of knowledge-domain anchors. SPECIALIST_DOMAINS: tuple[SpecialistDomain, ...] = ( SpecialistDomain( key="serving_specialist", @@ -271,8 +272,8 @@ def _tag_to_kb_anchor(tag: str) -> str: (``serving_specialist`` …) and the ``kb_anchor`` the PolicyGate whitelist validates against (``framework`` …). A tag naming a domain **key** is translated to that domain's anchor; a tag that is already a valid anchor is - kept as-is; anything else is returned verbatim so a genuinely unknown tag - still surfaces as ``specialist_unknown_domain``. + kept as-is; anything else is returned verbatim rather than invented into + an anchor, so an unresolvable tag stays visible downstream. Args: tag: A single (already-stripped, non-empty) dispatch tag. @@ -293,8 +294,7 @@ def normalize_dispatch_tags(params: dict) -> list[str]: Reads ``params.tags`` (each element is translated key→``kb_anchor`` via :func:`_tag_to_kb_anchor`); falls back to the ``params.domain`` alias (same translation) when absent. Order-preserving dedup; empty entries - dropped. Genuinely unknown tags pass through untranslated so PolicyGate's - ``specialist_unknown_domain`` still rejects them. + dropped. Genuinely unknown tags pass through untranslated. Args: params: The dispatch payload (reads ``tags`` then ``domain``). @@ -353,7 +353,8 @@ def get_domain(key: str) -> SpecialistDomain | None: # real stop is the wall-clock budget. DEFAULT_SPECIALIST_MAX_TURNS: int = 1000 -# Hard cap (PolicyGate R2 ``specialist_max_turns_excess`` enforces this). +# Hard cap; PolicyGate denies a dispatch above it because the in-process +# backend's turn loop has no wall-clock bound. SPECIALIST_MAX_TURNS_HARD_CAP: int = 1000 diff --git a/src/hyperloom/orchestrator/specialists/rebench.py b/src/hyperloom/orchestrator/specialists/rebench.py index c0cf923ff..52888e027 100644 --- a/src/hyperloom/orchestrator/specialists/rebench.py +++ b/src/hyperloom/orchestrator/specialists/rebench.py @@ -4,11 +4,11 @@ """Optional gate-comparable rebench helper for GPU specialists. Reuses the real Magpie serving + benchmark path (``run_grid``) on the -specialist's leased cards and a non-production port. The ``integrate_patch`` -gate stays the single authoritative measure of truth. +specialist's leased cards. The ``integrate_patch`` gate stays the single +authoritative measure of truth. The helper runs the server on the cards the subprocess already has pinned via -``ROCR_VISIBLE_DEVICES`` and refuses the production serving port 8888. +``ROCR_VISIBLE_DEVICES``. CLI usage (from inside a specialist subprocess):: @@ -37,50 +37,17 @@ ) -# The production serving process owns this port; a specialist rebench must never bind it. -PRODUCTION_SERVING_PORT = 8888 - # Default per-variant timeout (s) for a one-off specialist rebench. DEFAULT_REBENCH_TIMEOUT_SEC = 7800 -def _pick_free_port() -> int: - """Pick a free localhost TCP port (never the production serving port). - - Returns: - int: An ephemeral port the OS reported free, guaranteed != 8888. - """ - for _ in range(8): - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: - sock.bind(("127.0.0.1", 0)) - port = int(sock.getsockname()[1]) - if port != PRODUCTION_SERVING_PORT: - return port - # Fallthrough: pick a fixed non-8888 high port. - return 18888 - - def _resolve_port(port: int | None) -> int: - """Resolve and validate the rebench server port. - - Args: - port: Requested port; ``None`` or ``0`` auto-picks a free one. - - Returns: - int: A concrete port that is not the production serving port. - - Raises: - ValueError: If the production serving port 8888 is requested explicitly. - """ - if port in (None, 0): - return _pick_free_port() - p = int(port) - if p == PRODUCTION_SERVING_PORT: - raise ValueError( - f"refusing port {PRODUCTION_SERVING_PORT}: that is the production " - "serving port; a specialist rebench must use its own port." - ) - return p + """Resolve the requested port, using an OS-assigned port for ``None`` or ``0``.""" + if port not in (None, 0): + return int(port) + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind(("127.0.0.1", 0)) + return int(sock.getsockname()[1]) def _current_leased_cards() -> str: @@ -113,7 +80,7 @@ async def run_specialist_rebench( Reuses the real Magpie serving + benchmark path (``run_grid``) with a single identity variant. The server runs on the cards already pinned into - the subprocess env (``ROCR_VISIBLE_DEVICES``) and on a non-production port. + the subprocess env (``ROCR_VISIBLE_DEVICES``). Args: config_path: Base Magpie YAML to template from; ``None`` falls back to @@ -123,8 +90,7 @@ async def run_specialist_rebench( base_extra_args: Server args merged ahead of the variant's args (e.g. the current-best stack args, to compare apples-to-apples). extra_envs: Per-run environment overrides (e.g. tuned config env vars). - port: Server port; ``None``/``0`` auto-picks a free one. 8888 is - rejected. + port: Server port; ``None``/``0`` uses an OS-assigned port. variant_timeout_sec: Per-variant wall-clock timeout. model_path: Overrides the benchmark model path when set. gpu_type: Pins the generic ``{framework}_{gpu_type}.sh`` benchmark. @@ -180,6 +146,7 @@ async def run_specialist_rebench( benchmark_script=benchmark_script or None, magpie_python=magpie_python or None, server_lifecycle=server_lifecycle, + preclean_before_run=False, ) except Exception as exc: # noqa: BLE001 — surface as a structured failure return { @@ -244,13 +211,13 @@ def main(argv: list[str] | None = None) -> int: prog="specialist_rebench", description=( "Optional gate-comparable rebench on the specialist's leased " - "cards + a non-production port (reuses the Magpie serving path)." + "cards using the Magpie serving path." ), ) parser.add_argument("--config", default=None, help="Base Magpie YAML (defaults to packaged baseline).") parser.add_argument("--output", required=True, help="Output directory (conventionally inside the worktree).") parser.add_argument("--extra-args", default="", help="Server args merged ahead of the variant args.") - parser.add_argument("--port", type=int, default=0, help="Server port (0 auto-picks a free one; 8888 rejected).") + parser.add_argument("--port", type=int, default=0, help="Server port (0 uses an OS-assigned port).") parser.add_argument("--timeout", type=int, default=DEFAULT_REBENCH_TIMEOUT_SEC, help="Per-variant timeout (s).") parser.add_argument("--model-path", default=None, help="Override benchmark model path.") parser.add_argument("--gpu-type", default=None, help="Pin the generic benchmark script GPU type.") @@ -283,7 +250,6 @@ def main(argv: list[str] | None = None) -> int: __all__ = [ "DEFAULT_REBENCH_TIMEOUT_SEC", - "PRODUCTION_SERVING_PORT", "run_specialist_rebench", ] diff --git a/src/hyperloom/orchestrator/specialists/runner.py b/src/hyperloom/orchestrator/specialists/runner.py index 455429b4d..4f4c04bbc 100644 --- a/src/hyperloom/orchestrator/specialists/runner.py +++ b/src/hyperloom/orchestrator/specialists/runner.py @@ -47,7 +47,6 @@ _setup_worktree, ) from . import patch_safety as _patch_safety -from ..policy.gate import DEFAULT_SPECIALIST_MAX_PROPOSALS from .profile import SpecialistProfile, resolve_specialist_profile from ..loop.sub_agent_runner import RunnerContext from ..prompts.specialist_prompt_builder import ( @@ -240,7 +239,7 @@ class SpecialistRunResult: task_id: str domain: str gap_canonical_id: str - status: str # "succeeded" / "stale" / "empty_synthesised" / "tool_violation" + status: str # "succeeded" / "partial" / "stale" / "empty_synthesised" / "tool_violation" specialist_done: dict[str, Any] turns_used: int = 0 workspace: str = "" @@ -286,8 +285,10 @@ def classify_specialist_failure( type + retry-eligibility flag. ``status == 'stale'`` marks a subprocess that died with a ``backend_error`` - (timeout / stale-heartbeat / crash); ``empty_synthesised`` means it exited - cleanly without a usable ``specialist_done``. + (timeout / stale-heartbeat / crash) and left nothing usable behind; + ``partial`` means it died the same way but a checkpoint was salvaged, so the + failure is reported without discarding the work; ``empty_synthesised`` means + it exited cleanly without a usable ``specialist_done``. Args: runner_status: The :class:`SpecialistRunResult` status string. @@ -302,6 +303,13 @@ def classify_specialist_failure( return SpecialistFailureType.NONE, False if status == "tool_violation": return SpecialistFailureType.TOOL_VIOLATION, False + if status == "partial": + # Salvaged work: classify the failure but never retry over it. + if "timeout" in err: + return SpecialistFailureType.TIMEOUT, False + if "stale_heartbeat" in err: + return SpecialistFailureType.STALE_HEARTBEAT, False + return SpecialistFailureType.CRASH, False if status == "stale": if "timeout" in err: ftype = SpecialistFailureType.TIMEOUT @@ -326,8 +334,7 @@ def build_empty_specialist_done( ) -> dict[str, Any]: """Return the canonical empty ``specialist_done`` payload. - Satisfies PolicyGate R3 schema (``empty=true``, ``proposal_set=[]``, - non-empty summary). + Shape: ``empty=true``, ``proposal_set=[]``, non-empty summary. Args: gap_canonical_id: Canonical id of the gap the specialist addressed. @@ -605,8 +612,6 @@ async def _prepare( # WS1 wall-clock budget so the specialist can self-throttle. wall_budget_sec=float((ctx.extra or {}).get("wall_budget_sec") or 0.0), started_at_iso=datetime.now(timezone.utc).isoformat(), - # proposal_set self-curation target; shapes the prompt, not a hard cap. - max_proposals=max(1, int(params.get("max_proposals") or DEFAULT_SPECIALIST_MAX_PROPOSALS)), ) system_prompt, user_prompt = build_specialist_prompts(prompt_inputs) @@ -1041,6 +1046,7 @@ async def _run_via_subprocess( gpu_ids=tuple((ctx.extra or {}).get("gpu_ids") or ()), wall_budget_sec=wall_budget_sec, gpu_lease=(ctx.extra or {}).get("gpu_specialist_lease"), + progress_cb=(ctx.extra or {}).get("specialist_progress_cb"), ) self._append_transcript( workspace, @@ -1119,13 +1125,16 @@ async def _run_via_subprocess( ) # Decode subprocess error: backend_error → 'stale', clean miss → empty_synthesised. + # The classifier keys off the leading token; the reaper's own text is kept + # after it so the reader sees the elapsed/threshold numbers. + detail = (sub_result.error or "").strip() backend_error = "" if sub_result.timed_out: - backend_error = "subprocess_timeout" + backend_error = f"subprocess_timeout: {detail}" if detail else "subprocess_timeout" elif sub_result.stale_heartbeat: - backend_error = "subprocess_stale_heartbeat" - elif sub_result.error: - backend_error = f"subprocess_error:{sub_result.error}" + backend_error = f"subprocess_stale_heartbeat: {detail}" if detail else "subprocess_stale_heartbeat" + elif detail: + backend_error = f"subprocess_error:{detail}" elif sub_result.exit_code not in (None, 0) and sub_result.done_payload is None: backend_error = f"subprocess_exit_code:{sub_result.exit_code}" @@ -1155,9 +1164,9 @@ def _finalize( ) -> SpecialistRunResult: """Persist the ``specialist_done`` artifact and build the result. - Synthesises an empty payload when none was produced, sanitises and - truncates the proposal set, merges discovered patches and writes the - on-disk ``specialist_done.json``. + Synthesises an empty payload when none was produced, sanitises the + proposal set, merges discovered patches and writes the on-disk + ``specialist_done.json``. Args: ctx (RunnerContext): Dispatch context carrying the task. @@ -1217,7 +1226,6 @@ def _finalize( done_payload["allocated_gpu_ids"] = list(gpu_ids) if "proposal_set" not in done_payload: done_payload["proposal_set"] = [] - # ``max_proposals`` is a prompt-side target, not a hard cap. if "empty" not in done_payload: done_payload["empty"] = not bool(done_payload["proposal_set"]) if "summary" not in done_payload: @@ -1310,10 +1318,17 @@ def _resolve_existing_patch(p: Any) -> str | None: notes.extend(safety.notes()) self._write_specialist_done(workspace, done_payload) + recovered = bool(done_payload.get("_recovered_from_partial")) + # ``partial`` keeps an infra failure visible without making the attempt + # retry-eligible, which would discard whatever was salvaged. status = "succeeded" if tool_violations: status = "tool_violation" notes.append(f"tool_violations:{tool_violations}") + elif backend_error or recovered: + status = "partial" + if recovered: + notes.append("recovered_from_partial") return SpecialistRunResult( task_id=ctx.task.task_id, @@ -1325,6 +1340,7 @@ def _resolve_existing_patch(p: Any) -> str | None: workspace=str(workspace) if workspace else "", transcript_path=str(self._transcript_path(workspace)) if workspace else "", done_path=str(self._done_path(workspace)) if workspace else "", + error=backend_error, notes=notes, ) diff --git a/src/hyperloom/orchestrator/specialists/subprocess_.py b/src/hyperloom/orchestrator/specialists/subprocess_.py index b2a6950c6..dcd753546 100644 --- a/src/hyperloom/orchestrator/specialists/subprocess_.py +++ b/src/hyperloom/orchestrator/specialists/subprocess_.py @@ -97,6 +97,57 @@ def _build_specialist_env() -> dict[str, str]: return env +# Live wall-budget extensions granted by ``extend_lease`` while a specialist is +# already spawned. The reap loop re-reads this every poll, so an extension moves +# the hard kill deadline of a run that is in flight — without it, extend_lease +# would push the task / lane / GPU leases out while the subprocess still died at +# its original ``wall_budget_sec``. Keyed by task_id; the dispatcher clears the +# entry when the run finishes. +_WALL_BUDGET_EXTENSIONS: dict[str, float] = {} + + +def grant_wall_budget_extension(task_id: str, extra_sec: float) -> float: + """Add ``extra_sec`` to a live specialist's hard wall-clock deadline. + + Safe to call for a task that is not running a subprocess (the entry is + simply never read and is cleared on the next dispatch of that task_id). + + Args: + task_id: The specialist task whose deadline should move. + extra_sec: Seconds to add; non-positive values are a no-op. + + Returns: + The task's cumulative granted extension in seconds. + """ + key = str(task_id or "").strip() + if not key or extra_sec <= 0: + return _WALL_BUDGET_EXTENSIONS.get(key, 0.0) + total = _WALL_BUDGET_EXTENSIONS.get(key, 0.0) + float(extra_sec) + _WALL_BUDGET_EXTENSIONS[key] = total + return total + + +def wall_budget_extension(task_id: str) -> float: + """Return the cumulative live extension granted to ``task_id`` (0 if none). + + Args: + task_id: The specialist task to look up. + + Returns: + Seconds of extension granted so far. + """ + return _WALL_BUDGET_EXTENSIONS.get(str(task_id or "").strip(), 0.0) + + +def clear_wall_budget_extension(task_id: str) -> None: + """Drop any recorded extension for ``task_id``. + + Args: + task_id: The specialist task whose entry should be removed. + """ + _WALL_BUDGET_EXTENSIONS.pop(str(task_id or "").strip(), None) + + # Configuration @dataclass(frozen=True) class SpecialistSubprocessConfig: @@ -128,8 +179,8 @@ class SpecialistSubprocessConfig: framework_source_roots: tuple[str, ...] = () """Roots used to seed ``git worktree add`` and as ``--add-dir`` parents. - The first existing root becomes the worktree base; the rest are - read-only ``--add-dir`` entries (writes still need the worktree). + The first existing root becomes the worktree base; the rest are exposed + to the CLI as additional ``--add-dir`` entries. """ mcp_config_path: str | None = None @@ -379,6 +430,7 @@ async def run( gpu_ids: tuple[int, ...] = (), wall_budget_sec: float | None = None, gpu_lease: Any = None, + progress_cb: Any = None, ) -> SpecialistSubprocessResult: """Spawn a claude subprocess, reap it, return the parsed result. @@ -413,12 +465,17 @@ async def run( devices, so any GPU command the specialist issues stays within its lease). ``None`` keeps the local ``Popen`` path, with ``gpu_ids`` pinned into ``*_VISIBLE_DEVICES`` as before. + progress_cb (Any): Optional async callback invoked with each new + partial checkpoint the specialist writes while it is still + alive. Exceptions from it never affect the run. Returns: SpecialistSubprocessResult: Parsed outcome — done payload (if any), exit code, timing, timeout / stale-heartbeat flags, process log path, and discovered patches. """ + # Drop any extension left over from a prior run of this task id. + clear_wall_budget_extension(task_id) workspace.mkdir(parents=True, exist_ok=True) prompt_file = workspace / "prompt.md" process_log = workspace / "process.log" @@ -573,13 +630,17 @@ async def run( proc=proc, workspace=workspace, done_files=tuple(done_candidates), + partial_files=tuple(partial_candidates), heartbeat_file=heartbeat_file, max_seconds=max_seconds, started=proc_started, + progress_cb=progress_cb, + task_id=task_id, ) finally: if log_fh is not None: log_fh.close() + clear_wall_budget_extension(task_id) # Patches: scan worktree/patches/ (Arbor convention). patches = self._collect_patches(worktree, workspace) @@ -698,6 +759,46 @@ def _build_claude_cmd( cmd.extend(list(cfg.extra_claude_args)) return cmd + async def _publish_partial_progress( + self, + *, + partial_files: tuple[Path, ...], + since_mtime: float, + elapsed: float, + progress_cb: Any, + ) -> float: + """Forward a freshly-rewritten partial checkpoint to ``progress_cb``. + + Publishes at most one file per call: worktree first, then workspace. + + Args: + partial_files: Candidate checkpoint paths, worktree first. + since_mtime: Newest mtime already published. + elapsed: Seconds since spawn, passed to the callback. + progress_cb: Async callback receiving ``(payload, elapsed)``. + + Returns: + The newest mtime seen, so the caller can skip unchanged files. + """ + newest = since_mtime + for cand in partial_files: + try: + mtime = cand.stat().st_mtime + except OSError: + continue + if mtime <= since_mtime: + continue + payload = self._read_done(cand) + if payload is None: + continue + newest = max(newest, mtime) + try: + await progress_cb(payload, elapsed) + except Exception: # noqa: BLE001 — never let telemetry kill a run + log.exception("specialist progress callback raised") + break + return newest + async def _reap_loop( self, *, @@ -707,13 +808,17 @@ async def _reap_loop( heartbeat_file: Path, max_seconds: float, started: float, + partial_files: tuple[Path, ...] = (), + progress_cb: Any = None, + task_id: str = "", ) -> dict[str, Any]: """Poll the subprocess until it finishes, stalls, or times out. Each tick checks (in order): a done-file at any candidate path (graceful exit with a short grace window), natural process exit, heartbeat staleness, and the hard wall-clock cap. Stale / timed-out - runs are killed via :meth:`_kill`. + runs are killed via :meth:`_kill`. Partial checkpoints written along + the way are forwarded to ``progress_cb`` as they change. Args: proc (Any): The running claude subprocess — a ``subprocess.Popen`` @@ -724,6 +829,11 @@ async def _reap_loop( heartbeat_file (Path): Heartbeat file whose mtime gauges liveness. max_seconds (float): Hard wall-clock ceiling for the run. started (float): ``time.monotonic()`` value at spawn time. + partial_files (tuple[Path, ...]): Candidate partial-checkpoint + paths polled for live progress; never an exit signal. + progress_cb (Any): Optional async callback for each new checkpoint. + task_id (str): Task identifier used to pick up live + ``extend_lease`` wall-budget extensions each poll. Returns: dict[str, Any]: Outcome with ``exit_code``, ``elapsed``, @@ -741,6 +851,7 @@ async def _reap_loop( # process.log mtime is a reliable "still working" signal even when the # agent never self-writes heartbeat.json. process_log = workspace / "process.log" + last_partial_mtime: float = 0.0 while True: await asyncio.sleep(cfg.poll_interval_seconds) @@ -766,6 +877,15 @@ async def _reap_loop( outcome["elapsed"] = elapsed break + # Still running: republish any checkpoint written since the last tick. + if progress_cb is not None: + last_partial_mtime = await self._publish_partial_progress( + partial_files=partial_files, + since_mtime=last_partial_mtime, + elapsed=elapsed, + progress_cb=progress_cb, + ) + # Liveness check: alive if EITHER heartbeat.json was refreshed OR # process.log is still growing. The hard wall-clock cap below still # bounds genuinely hung subprocesses. @@ -791,10 +911,12 @@ async def _reap_loop( outcome["elapsed"] = time.monotonic() - started break - # Hard wall-clock cap. - if elapsed > max_seconds: + # Hard wall-clock cap — re-read each poll so an ``extend_lease`` + # granted mid-run actually moves this deadline. + deadline = max_seconds + wall_budget_extension(task_id) + if elapsed > deadline: outcome["timed_out"] = True - outcome["error"] = f"specialist subprocess exceeded {max_seconds:.0f}s wall-clock cap" + outcome["error"] = f"specialist subprocess exceeded {deadline:.0f}s wall-clock cap" self._kill(proc) outcome["exit_code"] = proc.poll() outcome["elapsed"] = time.monotonic() - started diff --git a/src/hyperloom/orchestrator/state/_shared_state/render.py b/src/hyperloom/orchestrator/state/_shared_state/render.py index ff4cc98a1..26ac05f80 100644 --- a/src/hyperloom/orchestrator/state/_shared_state/render.py +++ b/src/hyperloom/orchestrator/state/_shared_state/render.py @@ -313,6 +313,36 @@ def to_phase_budget_telemetry( lines.append(f" {phase}: elapsed={int(elapsed)}s {cap_line} used={used_pct:.0f}%") return "\n".join(lines) or "(no phase history yet)" + def to_resource_pools_summary(self) -> str: + """Render the GPU pool / lane capacity block. + + These are the same numbers PolicyGate admits a ``needs_gpu`` dispatch + against, so a request can be judged schedulable before it is emitted. + + Returns: + str: One line per pool / lane dimension. + """ + from ...bus.storage.schema import DEFAULT_LANE_CAPACITIES + from ...policy.gate import ( + _effective_gpu_specialist_pool_size, + _serving_tp_for_policy, + _whole_machine_pool_size, + gpu_specialist_ceiling, + ) + + lines = [ + f"serving_tp={_serving_tp_for_policy(self)}", + f"gpu_specialist_capacity={gpu_specialist_ceiling(self)}", + f"serving_disjoint_gpu_pool={_effective_gpu_specialist_pool_size(self)}" + " (non-bench needs_gpu specialists admit against this)", + f"whole_machine_gpu_pool={_whole_machine_pool_size()}" + " (bench / framework-authoring specialists admit against this)", + f"research_lane_capacity={max(0, int(self.research_lane_capacity or 0))} (concurrent specialists)", + f"gpu_research_lane_capacity={DEFAULT_LANE_CAPACITIES['gpu_research_lane']}" + " (mutually exclusive with serving / benchmark / profile)", + ] + return "\n".join(lines) + def to_warm_start_summary(self, *, max_lines: int = 12) -> str: """Render T0 warm-start snapshot for the ``=== Warm start ===`` prompt section; empty when no recipe/pitfalls. diff --git a/src/hyperloom/orchestrator/state/orchestration_memory.py b/src/hyperloom/orchestrator/state/orchestration_memory.py index 56ed9661f..eb033dfb4 100644 --- a/src/hyperloom/orchestrator/state/orchestration_memory.py +++ b/src/hyperloom/orchestrator/state/orchestration_memory.py @@ -26,9 +26,8 @@ DEFAULT_CHECKPOINT_CHAR_BUDGET: int = 400_000 # Context-token guardrail (usage = input + cache_read + cache_creation tokens). -# Soft trigger compacts proactively; hard fraction is the overflow backstop. +# Soft trigger compacts proactively. DEFAULT_CONTEXT_TOKEN_SOFT_FRACTION: float = 0.70 -DEFAULT_CONTEXT_TOKEN_HARD_FRACTION: float = 0.85 # Conservative fallback window for an unknown model id. DEFAULT_MODEL_CONTEXT_WINDOW: int = 200_000 MODEL_CONTEXT_WINDOWS: dict[str, int] = { @@ -69,9 +68,8 @@ class CheckpointPolicy: every_ticks: int = DEFAULT_CHECKPOINT_EVERY_TICKS every_minutes: float = DEFAULT_CHECKPOINT_EVERY_MINUTES char_budget: int = DEFAULT_CHECKPOINT_CHAR_BUDGET - # Context-token soft/hard budgets (absolute token counts; 0 disables). + # Context-token soft budget (absolute token count; 0 disables). context_token_soft: int = 0 - context_token_hard: int = 0 # Always checkpoint on a phase boundary. on_phase_boundary: bool = True @@ -110,20 +108,6 @@ def should_checkpoint( return True return False - def is_hard_compaction(self, context_tokens_now: int) -> bool: - """True when context is near the window and a compaction MUST happen now. - - The hard path compacts even when the LLM summary is degenerate (using the - deterministic fallback), so the conversation never overflows. - - Args: - context_tokens_now: Current context size in tokens. - - Returns: - ``True`` when the hard budget is set and reached. - """ - return self.context_token_hard > 0 and context_tokens_now >= self.context_token_hard - # Max byte length for next_cycle_directive before truncation. _DIRECTIVE_MAX_LEN: int = 1500 @@ -249,37 +233,6 @@ def is_degenerate_checkpoint(parsed: dict[str, Any]) -> bool: return not (has_plan or has_lists) -def deterministic_memory_fallback(state: Any) -> dict[str, Any]: - """Synthesise a minimal working-memory record from authoritative SharedState facts. - - Used by the hard context-token guardrail when the LLM checkpoint reply - is degenerate but the conversation must still be compacted to avoid window - overflow. Pure read of ``state``; never raises on missing attributes. - - Args: - state: The live ``SharedState`` (duck-typed; only attribute reads). - - Returns: - A parsed-reply-shaped dict suitable for :func:`build_memory_record`. - """ - cb = getattr(state, "current_best", {}) or {} - stack = getattr(state, "optimization_stack", []) or [] - try: - gain = float(getattr(state, "cumulative_gain_validated", 0.0) or 0.0) - except (TypeError, ValueError): - gain = 0.0 - phase = str(getattr(state, "phase", "") or "") - cycle = int(getattr(state, "macro_cycle", 0) or 0) - plan = f"[auto] phase={phase} cycle={cycle} best_tput={cb.get('tput')} validated_gain={gain:.2f}%" - return { - "current_plan": plan, - "hypotheses": [], - "tried_and_why": [f"stack has {len(stack)} accepted change(s)"], - "pending": ["recover plan from SharedState facts (LLM summary was degenerate)"], - "learnings": [], - } - - def _extract_json_object(text: str) -> dict[str, Any] | None: """Extract the first JSON object embedded in free-form text. @@ -459,7 +412,6 @@ def reset(self, *, tick: int, minute_mark: float, phase: str) -> None: "DEFAULT_CHECKPOINT_CHAR_BUDGET", "DEFAULT_CHECKPOINT_EVERY_MINUTES", "DEFAULT_CHECKPOINT_EVERY_TICKS", - "DEFAULT_CONTEXT_TOKEN_HARD_FRACTION", "DEFAULT_CONTEXT_TOKEN_SOFT_FRACTION", "DEFAULT_MODEL_CONTEXT_WINDOW", "MODEL_CONTEXT_WINDOWS", @@ -468,7 +420,6 @@ def reset(self, *, tick: int, minute_mark: float, phase: str) -> None: "_sanitize_cycle_directive", "build_memory_record", "context_window_for_model", - "deterministic_memory_fallback", "is_degenerate_checkpoint", "parse_checkpoint_reply", "render_memory_for_seed", diff --git a/src/hyperloom/orchestrator/state/task_registry.py b/src/hyperloom/orchestrator/state/task_registry.py index cd193209f..44e640748 100644 --- a/src/hyperloom/orchestrator/state/task_registry.py +++ b/src/hyperloom/orchestrator/state/task_registry.py @@ -6,9 +6,11 @@ Allowed transitions:: queued -> running, cancelled - running -> succeeded, failed, cancelled, needs_manual_review - failed -> running (retry) - succeeded / cancelled / needs_manual_review -> (terminal) + running -> succeeded, failed, cancelled + succeeded / failed / cancelled -> (terminal) + +A retry creates a new row under a fresh ``idempotency_key`` rather than +re-entering ``running``. ``idempotency_key`` is UNIQUE so re-creating a logical task returns the existing row. """ @@ -31,19 +33,17 @@ "succeeded", "failed", "cancelled", - "needs_manual_review", ) _TRANSITIONS: dict[str, frozenset[str]] = { "queued": frozenset({"running", "cancelled"}), - "running": frozenset({"succeeded", "failed", "cancelled", "needs_manual_review"}), - "failed": frozenset({"running"}), + "running": frozenset({"succeeded", "failed", "cancelled"}), + "failed": frozenset(), "succeeded": frozenset(), "cancelled": frozenset(), - "needs_manual_review": frozenset(), } -TERMINAL_STATES = frozenset({"succeeded", "cancelled", "needs_manual_review"}) +TERMINAL_STATES = frozenset({"succeeded", "cancelled"}) # microseconds + ``+00:00`` (canonical helper; kept importable for callers). @@ -64,7 +64,6 @@ class Task: allowed_tools (list[str]): Tool whitelist for the task. side_effects (list[str]): Declared side effects. lease_ttl_sec (int): Lease TTL in seconds. - attempts (int): Number of run attempts so far. history (list[dict]): Recorded state-transition history. created_at (str): ISO creation timestamp. updated_at (str): ISO last-update timestamp. @@ -79,7 +78,6 @@ class Task: allowed_tools: list[str] = field(default_factory=list) side_effects: list[str] = field(default_factory=list) lease_ttl_sec: int = 0 - attempts: int = 0 history: list[dict] = field(default_factory=list) created_at: str = field(default_factory=_now_iso) updated_at: str = field(default_factory=_now_iso) @@ -105,7 +103,6 @@ def from_row(cls, row) -> "Task": allowed_tools=json.loads(row["allowed_tools"]), side_effects=json.loads(row["side_effects"]), lease_ttl_sec=row["lease_ttl_sec"], - attempts=row["attempts"], history=json.loads(row["history"]), created_at=row["created_at"], updated_at=row["updated_at"], @@ -180,8 +177,8 @@ async def create_or_return_existing( cur.execute( "INSERT INTO tasks(task_id, kind, state, params, idempotency_key, " "requires_lanes, allowed_tools, side_effects, lease_ttl_sec, " - "attempts, history, created_at, updated_at) " - "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)", + "history, created_at, updated_at) " + "VALUES (?,?,?,?,?,?,?,?,?,?,?,?)", ( task_id, kind, @@ -192,7 +189,6 @@ async def create_or_return_existing( json.dumps(allowed_tools or []), json.dumps(side_effects or []), lease_ttl_sec, - 0, "[]", now, now, @@ -209,7 +205,6 @@ async def create_or_return_existing( allowed_tools=allowed_tools or [], side_effects=side_effects or [], lease_ttl_sec=lease_ttl_sec, - attempts=0, history=[], created_at=now, updated_at=now, @@ -281,9 +276,6 @@ async def transition( ) -> Task: """Transition a task to a new state, recording history. - Increments ``attempts`` when entering ``running`` from ``queued`` or - ``failed``. - Args: task_id (str): The task identifier. new_state (str): The target state (must be in :data:`TASK_STATES`). @@ -319,12 +311,9 @@ async def transition( "evidence": evidence or {}, } ) - attempts = row["attempts"] - if new_state == "running" and current_state in ("queued", "failed"): - attempts += 1 cur.execute( - "UPDATE tasks SET state=?, history=?, attempts=?, updated_at=? WHERE task_id=?", - (new_state, json.dumps(history), attempts, now, task_id), + "UPDATE tasks SET state=?, history=?, updated_at=? WHERE task_id=?", + (new_state, json.dumps(history), now, task_id), ) return await self.get(task_id) @@ -346,6 +335,38 @@ async def running(self) -> list[Task]: rows = await self.db.fetchall("SELECT * FROM tasks WHERE state='running' ORDER BY updated_at ASC") return [Task.from_row(r) for r in rows] + async def extend_lease(self, task_id: str, extra_sec: int) -> int: + """Grow a running task's ``lease_ttl_sec`` by ``extra_sec``. + + ``updated_at`` is left alone: it marks when the task started running, + and both the TTL watchdog and the elapsed-time projections measure from + it. + + Args: + task_id: The running task to extend. + extra_sec: Seconds to add; non-positive values are a no-op. + + Returns: + The task's new ``lease_ttl_sec``. + + Raises: + TaskNotFound: If no row matches ``task_id``. + IllegalTransition: If the task is not ``running``. + """ + async with self.db.transaction() as cur: + cur.execute("SELECT state, lease_ttl_sec FROM tasks WHERE task_id=?", (task_id,)) + row = cur.fetchone() + if row is None: + raise TaskNotFound(task_id) + if row["state"] != "running": + raise IllegalTransition(f"cannot extend lease of {task_id!r} in state {row['state']!r}") + new_ttl = int(row["lease_ttl_sec"] or 0) + max(0, int(extra_sec)) + cur.execute( + "UPDATE tasks SET lease_ttl_sec=? WHERE task_id=?", + (new_ttl, task_id), + ) + return new_ttl + async def by_state(self, state: str) -> list[Task]: """Return all tasks in the given state.