From c781c67b579f34f8601e67d32805ac4890fb8ec0 Mon Sep 17 00:00:00 2001 From: ZhengGong-amd Date: Sat, 25 Jul 2026 16:42:13 +0000 Subject: [PATCH 01/32] feat(enablement): canonical eval-accuracy flag/floor reader, score validator, and state fields Add the shared foundation for routing a baseline accuracy-eval failure into enablement: - _accuracy_gate: canonical readers for INFERENCE_OPTIMIZER_ENABLEMENT_ON_EVAL_FAIL (default on) and INFERENCE_OPTIMIZER_ENABLEMENT_ACCURACY_FLOOR (finite [0,1], invalid -> warn + default); a strict score validator (None/non-numeric/NaN/Inf/ <=0/ --- .../actions/executors/_accuracy_gate.py | 123 ++++++++++++++++++ .../actions/executors/integrate_patch.py | 13 +- .../orchestrator/state/shared_state.py | 13 ++ 3 files changed, 144 insertions(+), 5 deletions(-) diff --git a/src/hyperloom/orchestrator/actions/executors/_accuracy_gate.py b/src/hyperloom/orchestrator/actions/executors/_accuracy_gate.py index 5b58e89d9..d5f914acd 100644 --- a/src/hyperloom/orchestrator/actions/executors/_accuracy_gate.py +++ b/src/hyperloom/orchestrator/actions/executors/_accuracy_gate.py @@ -12,17 +12,42 @@ from __future__ import annotations import glob +import hashlib import json import logging +import math import os from pathlib import Path from typing import Any +from hyperloom.common.env import env_bool, env_float + log = logging.getLogger(__name__) ACCURACY_THRESHOLD = 0.05 # allowed deviation +# Enablement-on-eval-fail switch and shared accuracy floor. The floor is used by +# BOTH the baseline eval-failure trigger and the enablement KEEP gate so the two +# never diverge. +ENABLEMENT_ON_EVAL_FAIL_ENV = "INFERENCE_OPTIMIZER_ENABLEMENT_ON_EVAL_FAIL" +ENABLEMENT_ACCURACY_FLOOR_ENV = "INFERENCE_OPTIMIZER_ENABLEMENT_ACCURACY_FLOOR" +DEFAULT_ENABLEMENT_ACCURACY_FLOOR = 0.0 + +# Result-dict keys stamped by the baseline executor on an eval-rooted failure and +# read by writeback promotion/persistence. +BASELINE_EVAL_FAILED_KEY = "baseline_eval_failed" +BASELINE_EVAL_FAILURE_KIND_KEY = "baseline_eval_failure_kind" +BASELINE_EVAL_OBSERVED_ACCURACY_KEY = "baseline_eval_observed_accuracy" +BASELINE_EVAL_ACCURACY_FLOOR_KEY = "baseline_eval_accuracy_floor" +BASELINE_EVAL_EVIDENCE_KEY = "baseline_eval_evidence" +BASELINE_EVAL_CONTRACT_FINGERPRINT_KEY = "baseline_eval_contract_fingerprint" + +# Distinct eval-failure kinds. +EVAL_KIND_RUNTIME_FAILURE = "eval_runtime_failure" +EVAL_KIND_ACCURACY_UNAVAILABLE = "accuracy_unavailable" +EVAL_KIND_ACCURACY_BELOW_FLOOR = "accuracy_below_floor" + # stop_reason recorded when the baseline could not produce an accuracy result # even though the accuracy test was expected to run. A broken baseline accuracy # means the environment/config is fundamentally wrong, so the whole run halts @@ -72,6 +97,89 @@ def require_framework_accuracy_default() -> bool: return v not in ("0", "false", "no", "off") +def enablement_on_eval_fail_enabled() -> bool: + """Whether a baseline eval failure routes into enablement (default on).""" + return env_bool(ENABLEMENT_ON_EVAL_FAIL_ENV, True) + + +def enablement_accuracy_floor() -> float: + """Shared accuracy floor for the eval trigger and the enablement KEEP gate. + + Reads the env override, accepting only finite values in ``[0, 1]``; anything + else is logged and falls back to the default. + """ + raw = os.environ.get(ENABLEMENT_ACCURACY_FLOOR_ENV) + if raw is None or not raw.strip(): + return DEFAULT_ENABLEMENT_ACCURACY_FLOOR + val = env_float(ENABLEMENT_ACCURACY_FLOOR_ENV, DEFAULT_ENABLEMENT_ACCURACY_FLOOR) + if not math.isfinite(val) or val < 0.0 or val > 1.0: + log.warning( + "%s=%r is not a finite value in [0,1]; using default %.3f", + ENABLEMENT_ACCURACY_FLOOR_ENV, + raw, + DEFAULT_ENABLEMENT_ACCURACY_FLOOR, + ) + return DEFAULT_ENABLEMENT_ACCURACY_FLOOR + return val + + +def _finite_score(score: Any) -> float | None: + """Return ``score`` as a finite float, or ``None`` when unusable.""" + if isinstance(score, bool) or not isinstance(score, (int, float)): + return None + val = float(score) + return val if math.isfinite(val) else None + + +def accuracy_meets_floor(score: Any, floor: float) -> bool: + """True only when ``score`` is finite, strictly positive and ``>= floor``.""" + val = _finite_score(score) + if val is None or val <= 0.0: + return False + return val >= floor + + +def classify_accuracy_failure(score: Any, floor: float) -> str | None: + """Classify an accuracy verdict; ``None`` means it passes the floor. + + Missing / non-numeric / non-finite scores are ``accuracy_unavailable``; + finite scores that are non-positive or below the floor are + ``accuracy_below_floor``. + """ + val = _finite_score(score) + if val is None: + return EVAL_KIND_ACCURACY_UNAVAILABLE + if val <= 0.0 or val < floor: + return EVAL_KIND_ACCURACY_BELOW_FLOOR + return None + + +def eval_contract_fingerprint( + *, + config_path: str | Path | None, + framework: str | None, + model: str | None, + task: str | None, + metric: str | None, +) -> str: + """Short stable digest of the eval contract (workload + eval definition). + + Hashes the materialized config bytes when available plus the framework, + model, task and metric so a later enablement re-run can detect contract + drift. Never raises; unreadable inputs contribute their string form. + """ + parts = [str(framework or ""), str(model or ""), str(task or ""), str(metric or "")] + cfg = "" + if config_path: + p = Path(config_path) + try: + cfg = p.read_bytes().hex() if p.is_file() else str(config_path) + except OSError: + cfg = str(config_path) + parts.append(cfg) + return hashlib.sha256("\x1e".join(parts).encode("utf-8", "replace")).hexdigest()[:16] + + def accuracy_keep_block( accuracy_pass: bool | None, *, @@ -382,8 +490,23 @@ def accuracy_passed( __all__ = [ "ACCURACY_THRESHOLD", "BASELINE_ACCURACY_STOP_REASON", + "BASELINE_EVAL_ACCURACY_FLOOR_KEY", + "BASELINE_EVAL_CONTRACT_FINGERPRINT_KEY", + "BASELINE_EVAL_EVIDENCE_KEY", + "BASELINE_EVAL_FAILED_KEY", + "BASELINE_EVAL_FAILURE_KIND_KEY", + "BASELINE_EVAL_OBSERVED_ACCURACY_KEY", + "DEFAULT_ENABLEMENT_ACCURACY_FLOOR", + "EVAL_KIND_ACCURACY_BELOW_FLOOR", + "EVAL_KIND_ACCURACY_UNAVAILABLE", + "EVAL_KIND_RUNTIME_FAILURE", "accuracy_keep_block", + "accuracy_meets_floor", "accuracy_passed", + "classify_accuracy_failure", + "enablement_accuracy_floor", + "enablement_on_eval_fail_enabled", + "eval_contract_fingerprint", "is_high_accuracy_risk", "parse_eval_results", "request_baseline_accuracy_stop", diff --git a/src/hyperloom/orchestrator/actions/executors/integrate_patch.py b/src/hyperloom/orchestrator/actions/executors/integrate_patch.py index 72871fd57..2ed3f0999 100644 --- a/src/hyperloom/orchestrator/actions/executors/integrate_patch.py +++ b/src/hyperloom/orchestrator/actions/executors/integrate_patch.py @@ -21,7 +21,12 @@ from hyperloom.inference_optimizer.session.session_paths import runs_dir from ...framework.paths import resolve_source_file_allowlist from ...specialists.patch_safety import patch_file_targets, patch_targets_missing -from ._accuracy_gate import accuracy_keep_block, accuracy_passed, parse_eval_results +from ._accuracy_gate import ( + accuracy_keep_block, + accuracy_passed, + enablement_accuracy_floor, + parse_eval_results, +) from ._apply_feedback import ApplyFeedback, build_apply_feedback from ._git import _run_git_cp from ._nogit_patch import ( @@ -56,9 +61,6 @@ DEFAULT_KEEP_THRESHOLD_PCT = 1.0 # grid noise floor; KEEP is re-confirmed by a stack rebench DEFAULT_VARIANT_TIMEOUT_SEC = 7800 -# Minimal-correctness floor for the enablement runnable gate: accuracy strictly -# above this counts as "not garbage". -ENABLEMENT_ACCURACY_FLOOR = 0.0 _HYPERLOOM_AUTO_STASH_MSG = "hyperloom-auto-stash: preserving user changes before candidate run" @@ -2192,9 +2194,10 @@ def _gc_on_revert() -> None: probe_timed_out = bool(gate_evidence.get("timed_out")) enablement_accuracy = gate_evidence.get("enablement_accuracy") + floor = enablement_accuracy_floor() correctness_ok: bool | None if isinstance(enablement_accuracy, (int, float)) and not _math.isnan(float(enablement_accuracy)): - correctness_ok = float(enablement_accuracy) > ENABLEMENT_ACCURACY_FLOOR + correctness_ok = float(enablement_accuracy) > floor elif isinstance(enablement_accuracy, float) and _math.isnan(enablement_accuracy): correctness_ok = False else: diff --git a/src/hyperloom/orchestrator/state/shared_state.py b/src/hyperloom/orchestrator/state/shared_state.py index 903e9f954..e5c80b823 100644 --- a/src/hyperloom/orchestrator/state/shared_state.py +++ b/src/hyperloom/orchestrator/state/shared_state.py @@ -407,6 +407,19 @@ class SharedState(_RenderMixin, _ExploreStateMixin): # One-shot: a cuda-graph capture failure asks the next baseline to retry with # cuda-graph capture disabled. Set on failure, consumed by BaselineExecutor. baseline_eager_fallback: bool = False + # Eval-origin enablement carriers: set when the first baseline runs but its + # accuracy eval fails, so the enablement pump/gate can reconstruct the trigger + # and re-run the same eval contract. Empty for boot-origin enablement. + enablement_origin: str = "" + enablement_accuracy_floor: float = 0.0 + enablement_probe_config_path: str = "" + enablement_eval_contract_fingerprint: str = "" + enablement_baseline_eval_evidence: str = "" + enablement_baseline_eval_kind: str = "" + enablement_observed_accuracy: float = 0.0 + enablement_observed_task: str = "" + enablement_observed_metric: str = "" + enablement_pending: bool = False # Enablement path (framework-agent) state. # ``enablement_launch_log``: captured launch/traceback text when baseline # cannot launch. From 39e57a331307e3ff8aea4d67b1b3aeee87a156bd Mon Sep 17 00:00:00 2001 From: ZhengGong-amd Date: Sat, 25 Jul 2026 16:48:20 +0000 Subject: [PATCH 02/32] feat(enablement): capture baseline eval failures as an enablement trigger When a genuine first baseline runs single-node with the eval-on-fail flag enabled, an accuracy-eval failure now yields an explicit eval-failure contract instead of salvaging throughput or stopping the run: - __call__: an eval-rooted subprocess failure stamps the result with the contract (kind, evidence, floor, fingerprint) and returns, skipping the RUN_EVAL=false salvage retry. - _maybe_stop_on_missing_baseline_accuracy: a succeeded baseline whose accuracy is missing/non-finite/non-positive or below the floor is stamped and returns instead of calling request_baseline_accuracy_stop. - _eval_failure_evidence returns bounded evidence alongside detection; _is_eval_rooted_failure kept as a thin bool wrapper. - failure results now carry materialized_config/result_dir/run_eval_disabled so the contract can be fingerprinted and re-run. Flag-off and multi-node keep the existing salvage/stop behavior. Existing tests pin the legacy path via a flag-off fixture; new tests cover the routed path. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../tests/test_baseline_eval_fallback.py | 83 ++++++++ .../actions/executors/baseline.py | 193 ++++++++++++++---- 2 files changed, 238 insertions(+), 38 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_baseline_eval_fallback.py b/src/hyperloom/inference_optimizer/tests/test_baseline_eval_fallback.py index b34b781fc..2b15e99cf 100644 --- a/src/hyperloom/inference_optimizer/tests/test_baseline_eval_fallback.py +++ b/src/hyperloom/inference_optimizer/tests/test_baseline_eval_fallback.py @@ -27,6 +27,13 @@ def _isolate_leak_root(tmp_path_factory, monkeypatch): monkeypatch.setenv("INFERENCE_OPTIMIZER_LEAK_ROOTS", str(sandbox)) +@pytest.fixture(autouse=True) +def _legacy_salvage_default(monkeypatch): + """Exercise the legacy salvage/stop path by default; eval-origin enablement + routing is opted into per-test.""" + monkeypatch.setenv("INFERENCE_OPTIMIZER_ENABLEMENT_ON_EVAL_FAIL", "0") + + def _write_yaml(path: Path) -> None: cfg = { "benchmark": { @@ -573,3 +580,79 @@ def fake_run(cmd, *args, **kwargs): assert len(calls) == 1 # already off, no fallback assert result["status"] == "failed" + + +# --- eval-origin enablement routing (flag on) ------------------------------ +from hyperloom.orchestrator.actions.executors._accuracy_gate import ( # noqa: E402 + BASELINE_EVAL_ACCURACY_FLOOR_KEY, + BASELINE_EVAL_CONTRACT_FINGERPRINT_KEY, + BASELINE_EVAL_EVIDENCE_KEY, + BASELINE_EVAL_FAILED_KEY, + BASELINE_EVAL_FAILURE_KIND_KEY, + BASELINE_EVAL_OBSERVED_ACCURACY_KEY, + EVAL_KIND_ACCURACY_BELOW_FLOOR, + EVAL_KIND_ACCURACY_UNAVAILABLE, +) + + +def _route(monkeypatch, framework, result, *, floor=None, nodes=None): + monkeypatch.setenv("INFERENCE_OPTIMIZER_ENABLEMENT_ON_EVAL_FAIL", "1") + if floor is not None: + monkeypatch.setenv("INFERENCE_OPTIMIZER_ENABLEMENT_ACCURACY_FLOOR", str(floor)) + if nodes is not None: + monkeypatch.setenv("INFERENCE_OPTIMIZER_NODES", str(nodes)) + executor = BaselineExecutor() + rec = _StopRecorder() + executor._maybe_stop_on_missing_baseline_accuracy(_stop_ctx(framework, rec), result) + return rec.stop_reason + + +def test_eval_enablement_missing_accuracy_routes_not_stop(monkeypatch): + result = {"status": "succeeded", "run_eval_disabled": False, "materialized_config": "/tmp/c.yaml"} + reason = _route(monkeypatch, "sglang", result) + assert reason == "" + assert result[BASELINE_EVAL_FAILED_KEY] is True + assert result[BASELINE_EVAL_FAILURE_KIND_KEY] == EVAL_KIND_ACCURACY_UNAVAILABLE + assert result[BASELINE_EVAL_OBSERVED_ACCURACY_KEY] is None + assert result[BASELINE_EVAL_ACCURACY_FLOOR_KEY] == 0.0 + assert result[BASELINE_EVAL_EVIDENCE_KEY] + assert result[BASELINE_EVAL_CONTRACT_FINGERPRINT_KEY] + assert result["eval_origin"] == "eval" + + +def test_eval_enablement_zero_accuracy_below_floor(monkeypatch): + result = {"status": "succeeded", "accuracy": 0.0, "run_eval_disabled": False} + reason = _route(monkeypatch, "sglang", result) + assert reason == "" + assert result[BASELINE_EVAL_FAILURE_KIND_KEY] == EVAL_KIND_ACCURACY_BELOW_FLOOR + assert result[BASELINE_EVAL_OBSERVED_ACCURACY_KEY] == 0.0 + + +def test_eval_enablement_positive_below_configured_floor(monkeypatch): + result = {"status": "succeeded", "accuracy": 0.2, "run_eval_disabled": False} + reason = _route(monkeypatch, "sglang", result, floor=0.5) + assert reason == "" + assert result[BASELINE_EVAL_FAILURE_KIND_KEY] == EVAL_KIND_ACCURACY_BELOW_FLOOR + assert result[BASELINE_EVAL_OBSERVED_ACCURACY_KEY] == 0.2 + assert result[BASELINE_EVAL_ACCURACY_FLOOR_KEY] == 0.5 + + +def test_eval_enablement_accuracy_at_floor_passes(monkeypatch): + result = {"status": "succeeded", "accuracy": 0.5, "run_eval_disabled": False} + reason = _route(monkeypatch, "sglang", result, floor=0.5) + assert reason == "" + assert BASELINE_EVAL_FAILED_KEY not in result + + +def test_eval_enablement_multi_node_falls_back_to_stop(monkeypatch): + result = {"status": "succeeded", "run_eval_disabled": False} + reason = _route(monkeypatch, "sglang", result, nodes=2) + assert reason == "baseline_accuracy_failed" + assert BASELINE_EVAL_FAILED_KEY not in result + + +def test_eval_enablement_operator_optout_not_routed(monkeypatch): + result = {"status": "succeeded", "run_eval_disabled": True} + reason = _route(monkeypatch, "sglang", result) + assert reason == "" + assert BASELINE_EVAL_FAILED_KEY not in result diff --git a/src/hyperloom/orchestrator/actions/executors/baseline.py b/src/hyperloom/orchestrator/actions/executors/baseline.py index 724a87c24..4ee190e00 100644 --- a/src/hyperloom/orchestrator/actions/executors/baseline.py +++ b/src/hyperloom/orchestrator/actions/executors/baseline.py @@ -1101,40 +1101,41 @@ def _after_materialize_config( return None @staticmethod - def _is_eval_rooted_failure(result: dict[str, Any]) -> bool: - """Whether a failed baseline result was caused by the accuracy eval. + def _eval_failure_evidence(result: dict[str, Any]) -> tuple[bool, str]: + """Detect an eval-rooted baseline failure and capture bounded evidence. Scans the result's ``error`` tail + ``nonfatal_warnings`` and then any benchmark stdout/stderr + ``server.log`` under the run's ``output_dir`` - for ``run_eval``-failure markers. Recursive so the double-run path is - covered (the warmup round's logs carry the marker even when the measure - round failed for a downstream reason). Never raises. - - Args: - result: A ``status="failed"`` baseline result dict. + for ``run_eval``-failure markers. Climbs to the shared task root so the + double-run warmup logs are covered. Returns the first matched window + (marker plus a bounded slice) so callers need not rescan. Never raises. Returns: - ``True`` when an eval-failure marker is found, else ``False``. + ``(is_eval_rooted, evidence)``; ``evidence`` is empty when not rooted. """ - def _hit(text: str) -> bool: - return any(m in text for m in _EVAL_FAILURE_MARKERS) + def _window(text: str) -> str | None: + for m in _EVAL_FAILURE_MARKERS: + i = text.find(m) + if i != -1: + return text[max(0, i - 200) : i + len(m) + 400] + return None - if _hit(str(result.get("error") or "")): - return True - for w in result.get("nonfatal_warnings") or []: - if _hit(str(w)): - return True + w = _window(str(result.get("error") or "")) + if w is not None: + return True, w + for warn in result.get("nonfatal_warnings") or []: + w = _window(str(warn)) + if w is not None: + return True, w out_dir = result.get("output_dir") if not out_dir: - return False + return False, "" root = Path(out_dir) - # Double-run: the eval failure markers may live in the sibling warmup - # round, so climb to the shared task root to scan both rounds. if root.name in ("warmup_round", "measure_round"): root = root.parent if not root.exists(): - return False + return False, "" log_names = ("benchmark_stderr.log", "benchmark_stdout.log", "server.log") seen = 0 try: @@ -1152,11 +1153,17 @@ def _hit(text: str) -> bool: chunk = f.read().decode("utf-8", "replace") except OSError: continue - if _hit(chunk): - return True + w = _window(chunk) + if w is not None: + return True, f"{path.name}: {w}" except OSError: - return False - return False + return False, "" + return False, "" + + @staticmethod + def _is_eval_rooted_failure(result: dict[str, Any]) -> bool: + """Whether a failed baseline result was caused by the accuracy eval.""" + return BaselineExecutor._eval_failure_evidence(result)[0] async def __call__(self, ctx: RunnerContext) -> dict[str, Any]: """Run the Magpie baseline, with a one-shot eval-failure fallback. @@ -1187,22 +1194,94 @@ async def __call__(self, ctx: RunnerContext) -> dict[str, Any]: "RUN_EVAL" in _extra_envs and str(_extra_envs["RUN_EVAL"]).strip().lower() in _RUN_EVAL_FALSE_VALUES ) eval_already_off = is_truthy(params.get("disable_run_eval")) or _explicit_run_eval - if result.get("status") != "succeeded" and not eval_already_off and self._is_eval_rooted_failure(result): - log.warning( - "baseline_executor: failure looks eval-rooted (InferenceX " - "run_eval aborted the benchmark); retrying once with " - "RUN_EVAL=false to salvage the throughput baseline without " - "the accuracy gate." - ) - retry = await self._run_once(ctx, force_disable_eval=True) - retry.setdefault("nonfatal_warnings", []) - retry["nonfatal_warnings"].append("eval_failed_fallback_no_accuracy") - if retry.get("status") == "succeeded": - retry["accuracy_source"] = "eval_unavailable" - result = retry + if result.get("status") != "succeeded" and not eval_already_off: + rooted, evidence = self._eval_failure_evidence(result) + if rooted: + if self._eval_enablement_active(ctx): + from ._accuracy_gate import EVAL_KIND_RUNTIME_FAILURE + + log.warning( + "baseline_executor: eval-rooted failure; routing to " + "enablement instead of salvaging (RUN_EVAL stays on)." + ) + self._stamp_eval_failure_contract( + ctx, result, kind=EVAL_KIND_RUNTIME_FAILURE, observed_accuracy=None, evidence=evidence + ) + return result + log.warning( + "baseline_executor: failure looks eval-rooted (InferenceX " + "run_eval aborted the benchmark); retrying once with " + "RUN_EVAL=false to salvage the throughput baseline without " + "the accuracy gate." + ) + retry = await self._run_once(ctx, force_disable_eval=True) + retry.setdefault("nonfatal_warnings", []) + retry["nonfatal_warnings"].append("eval_failed_fallback_no_accuracy") + if retry.get("status") == "succeeded": + retry["accuracy_source"] = "eval_unavailable" + result = retry self._maybe_stop_on_missing_baseline_accuracy(ctx, result) return result + def _eval_enablement_active(self, ctx: RunnerContext) -> bool: + """Whether an eval failure should route into enablement this run. + + Only for a genuine baseline, single-node, with the flag enabled. + """ + from ._accuracy_gate import enablement_on_eval_fail_enabled + from ._multi_node_env import is_multi_node + + if not enablement_on_eval_fail_enabled(): + return False + if is_multi_node(): + return False + return _should_establish_quality_ref(getattr(ctx.task, "kind", "")) + + def _stamp_eval_failure_contract( + self, + ctx: RunnerContext, + result: dict[str, Any], + *, + kind: str, + observed_accuracy: float | None, + evidence: str, + ) -> dict[str, Any]: + """Mark ``result`` as an eval-rooted baseline failure for enablement. + + Records the failure kind, observed accuracy, effective floor, bounded + evidence and an eval-contract fingerprint so writeback can persist the + trigger and a later enablement round can re-run the same contract. + """ + from ._accuracy_gate import ( + BASELINE_EVAL_ACCURACY_FLOOR_KEY, + BASELINE_EVAL_CONTRACT_FINGERPRINT_KEY, + BASELINE_EVAL_EVIDENCE_KEY, + BASELINE_EVAL_FAILED_KEY, + BASELINE_EVAL_FAILURE_KIND_KEY, + BASELINE_EVAL_OBSERVED_ACCURACY_KEY, + enablement_accuracy_floor, + eval_contract_fingerprint, + ) + + params = ctx.task.params or {} + framework = str(params.get("framework") or "").strip() or os.environ.get("FRAMEWORK", "").strip() or None + model = params.get("model") or params.get("resolved_model") + floor = enablement_accuracy_floor() + result[BASELINE_EVAL_FAILED_KEY] = True + result[BASELINE_EVAL_FAILURE_KIND_KEY] = kind + result[BASELINE_EVAL_OBSERVED_ACCURACY_KEY] = observed_accuracy + result[BASELINE_EVAL_ACCURACY_FLOOR_KEY] = floor + result[BASELINE_EVAL_EVIDENCE_KEY] = (evidence or "")[:4000] + result[BASELINE_EVAL_CONTRACT_FINGERPRINT_KEY] = eval_contract_fingerprint( + config_path=result.get("materialized_config"), + framework=framework, + model=model, + task=result.get("accuracy_task"), + metric=result.get("accuracy_metric"), + ) + result["eval_origin"] = "eval" + return result + def _maybe_stop_on_missing_baseline_accuracy( self, ctx: RunnerContext, @@ -1237,7 +1316,14 @@ def _maybe_stop_on_missing_baseline_accuracy( if result.get("status") != "succeeded": return acc = result.get("accuracy") - if acc is not None and float(acc) > 0.0: + eval_enablement = self._eval_enablement_active(ctx) + from ._accuracy_gate import accuracy_meets_floor, classify_accuracy_failure, enablement_accuracy_floor + + floor = enablement_accuracy_floor() + if eval_enablement: + if accuracy_meets_floor(acc, floor): + return # a usable baseline accuracy at/above the floor exists + elif acc is not None and float(acc) > 0.0: return # a usable baseline accuracy exists params = ctx.task.params or {} framework = str(params.get("framework") or "").strip() or os.environ.get("FRAMEWORK", "").strip() or None @@ -1253,6 +1339,26 @@ def _maybe_stop_on_missing_baseline_accuracy( ) if not (scriptable or not operator_disabled_eval): return + # Route into enablement instead of stopping/salvaging: the throughput + # baseline stays for diagnostics but is blocked from anchoring. + if eval_enablement: + kind = classify_accuracy_failure(acc, floor) + observed = float(acc) if isinstance(acc, (int, float)) else None + evidence = ( + f"baseline accuracy did not meet floor: accuracy={acc} floor={floor} " + f"task={result.get('accuracy_task')} metric={result.get('accuracy_metric')} " + f"source={result.get('accuracy_source')}" + ) + self._stamp_eval_failure_contract( + ctx, result, kind=kind or "", observed_accuracy=observed, evidence=evidence + ) + log.warning( + "baseline_executor: accuracy %s below floor %.4f; routing to " + "enablement instead of stopping the run.", + acc, + floor, + ) + return extra = getattr(ctx, "extra", None) or {} shared_state = extra.get("shared_state") or self.shared_state # Session-level salvage (complements #942): the cold-start guard and the @@ -1942,6 +2048,13 @@ async def _run_single_benchmark( # that ignore it are caught by the salvage pass. result_dir = _resolve_result_dir(output_dir, override_result_dir) env["RESULT_DIR"] = str(result_dir) + # Config/eval-contract facts echoed onto every failure result so an + # eval-rooted failure can be fingerprinted and re-run by enablement. + capture_meta = { + "materialized_config": str(materialized_config_path), + "result_dir": str(result_dir), + "run_eval_disabled": bool(run_eval_disabled), + } # InferenceX ``run_lm_eval`` cleans ``$EVAL_RESULT_DIR`` after processing # lm-eval output. Keep it under the task workspace but separate from # Magpie's ``benchmark_*`` traces in ``$RESULT_DIR``. @@ -2120,6 +2233,7 @@ async def _run_single_benchmark( "output_dir": str(output_dir), "harvested_artifacts": [str(dst) for _, dst in timeout_harvested], "nonfatal_warnings": [f"harvested_leaked_artifact:{src}" for src, _ in timeout_harvested], + **capture_meta, } # Detokenizer-stall watchdog reap: the server came up healthy but went @@ -2150,6 +2264,7 @@ async def _run_single_benchmark( "output_dir": str(output_dir), "harvested_artifacts": [str(dst) for _, dst in stall_harvested], "nonfatal_warnings": [f"harvested_leaked_artifact:{src}" for src, _ in stall_harvested], + **capture_meta, } # When the server's engine/worker bootstrap dies, the root cause is in @@ -2202,6 +2317,7 @@ async def _run_single_benchmark( failure_extras = { "output_dir": str(output_dir), "harvested_artifacts": [str(dst) for _, dst in harvested], + **capture_meta, } # Magpie never created a benchmark_* workspace, so the wrapper never # wrote server.log. Persist captured stderr/stdout so the failure @@ -2313,6 +2429,7 @@ async def _run_single_benchmark( "reported_success": measurement.get("reported_success"), "subprocess_runtime_sec": round(subprocess_runtime_sec, 2), "nonfatal_warnings": warnings, + **capture_meta, } result = { From a0f4b28c1415367504b9908663be0a2c390516f8 Mon Sep 17 00:00:00 2001 From: ZhengGong-amd Date: Sat, 25 Jul 2026 16:52:58 +0000 Subject: [PATCH 03/32] feat(enablement): block promotion of eval-failed baselines and persist the trigger Route an eval-failed baseline into enablement via the existing boot-fail machinery: - _is_promotable_result: a baseline result flagged baseline_eval_failed is not promotable (throughput is kept for diagnostics but never anchors); profile is unaffected. - _handle_unpromotable_result: persist the eval origin, floor, probe config, contract fingerprint, evidence, kind and observed accuracy, and seed enablement_launch_log so the pump dispatches. On a single-node eval-pending tick the baseline_failed stop is suppressed (the streak still advances so the pump can fire); multi-node keeps the strict backstop. - _promote_baseline: a genuine anchor clears the sticky eval-origin state. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../tests/test_coordinator_runtime.py | 50 +++++++++++++++ ..._coordinator_sync_helpers_coverage_unit.py | 9 +++ src/hyperloom/orchestrator/loop/writeback.py | 64 ++++++++++++++++++- 3 files changed, 121 insertions(+), 2 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_coordinator_runtime.py b/src/hyperloom/inference_optimizer/tests/test_coordinator_runtime.py index f2b303174..684e26825 100644 --- a/src/hyperloom/inference_optimizer/tests/test_coordinator_runtime.py +++ b/src/hyperloom/inference_optimizer/tests/test_coordinator_runtime.py @@ -894,6 +894,56 @@ async def test_handle_unpromotable_baseline_third_failure_sets_stop_reason( await c.stop() +def _eval_failed_result() -> dict: + return { + "status": "failed", + "error_class": "subprocess_nonzero", + "error": "ERROR: run_eval failed with exit code 1", + "baseline_eval_failed": True, + "baseline_eval_failure_kind": "eval_runtime_failure", + "baseline_eval_accuracy_floor": 0.2, + "baseline_eval_evidence": "run_eval failed with exit code 1", + "baseline_eval_contract_fingerprint": "abc123", + "materialized_config": "/runs/baseline/materialized.yaml", + } + + +@pytest.mark.asyncio +async def test_handle_unpromotable_baseline_eval_pending_suppresses_stop_single_node( + session_dir, monkeypatch +): + monkeypatch.delenv("INFERENCE_OPTIMIZER_NODES", raising=False) + c = Coordinator(session_dir, backends=_silent_backends()) + _mute_action_scoring(c) + try: + for i in range(3): + await c._handle_unpromotable_result(_mk_task("baseline", f"t-ev-{i}"), _eval_failed_result()) + assert c.shared_state.baseline_failure_streak == 3 + assert c.shared_state.stop_reason in ("", None) + assert c.shared_state.enablement_origin == "eval" + assert c.shared_state.enablement_pending is True + assert c.shared_state.enablement_accuracy_floor == 0.2 + assert c.shared_state.enablement_probe_config_path == "/runs/baseline/materialized.yaml" + assert c.shared_state.enablement_eval_contract_fingerprint == "abc123" + assert c.shared_state.enablement_baseline_eval_kind == "eval_runtime_failure" + assert c.shared_state.enablement_launch_log + finally: + await c.stop() + + +@pytest.mark.asyncio +async def test_handle_unpromotable_baseline_eval_pending_multi_node_still_stops(session_dir, monkeypatch): + monkeypatch.setenv("INFERENCE_OPTIMIZER_NODES", "2") + c = Coordinator(session_dir, backends=_silent_backends()) + _mute_action_scoring(c) + try: + for i in range(3): + await c._handle_unpromotable_result(_mk_task("baseline", f"t-mn-{i}"), _eval_failed_result()) + assert c.shared_state.stop_reason == "baseline_failed" + finally: + await c.stop() + + @pytest.mark.asyncio async def test_handle_unpromotable_records_for_non_baseline_kinds(session_dir): c = Coordinator(session_dir, backends=_silent_backends()) diff --git a/src/hyperloom/inference_optimizer/tests/test_coordinator_sync_helpers_coverage_unit.py b/src/hyperloom/inference_optimizer/tests/test_coordinator_sync_helpers_coverage_unit.py index f895cad33..a7d7534cb 100644 --- a/src/hyperloom/inference_optimizer/tests/test_coordinator_sync_helpers_coverage_unit.py +++ b/src/hyperloom/inference_optimizer/tests/test_coordinator_sync_helpers_coverage_unit.py @@ -204,6 +204,15 @@ def test_is_promotable_result(coord: Coordinator) -> None: assert coord._is_promotable_result("explore", {"status": "failed"}) is False +def test_is_promotable_result_baseline_eval_failed(coord: Coordinator) -> None: + measured = {"output_throughput": 1000.0, "completed_requests": 10} + assert coord._is_promotable_result("baseline", measured) is True + eval_failed = {**measured, "baseline_eval_failed": True} + assert coord._is_promotable_result("baseline", eval_failed) is False + # profile with the same key still promotes (blocker is baseline-only). + assert coord._is_promotable_result("profile", eval_failed) is True + + # -- phase / id helpers ---------------------------------------------------- def test_journal_entry_phase(coord: Coordinator) -> None: coord.shared_state.phase = "" diff --git a/src/hyperloom/orchestrator/loop/writeback.py b/src/hyperloom/orchestrator/loop/writeback.py index bfc161361..ed523ca2b 100644 --- a/src/hyperloom/orchestrator/loop/writeback.py +++ b/src/hyperloom/orchestrator/loop/writeback.py @@ -42,6 +42,14 @@ ) from ..state.task_registry import Task from ..actions.executors.benchmark_result import is_valid_measurement +from ..actions.executors._accuracy_gate import ( + BASELINE_EVAL_ACCURACY_FLOOR_KEY, + BASELINE_EVAL_CONTRACT_FINGERPRINT_KEY, + BASELINE_EVAL_EVIDENCE_KEY, + BASELINE_EVAL_FAILED_KEY, + BASELINE_EVAL_FAILURE_KIND_KEY, + BASELINE_EVAL_OBSERVED_ACCURACY_KEY, +) from .coordinator import ( _AUDIT_ACTIONS, @@ -354,7 +362,13 @@ def _is_promotable_result(self, task_kind: str, result: dict[str, Any]) -> bool: """ if not isinstance(result, dict): return False - if task_kind in ("baseline", "profile"): + if task_kind == "baseline": + # A baseline whose accuracy eval failed measured throughput but must + # not anchor; route it to _handle_unpromotable_result for enablement. + if bool(result.get("baseline_eval_failed")): + return False + return is_valid_measurement(result) + if task_kind == "profile": return is_valid_measurement(result) if task_kind == "sweep": return result.get("status") == "succeeded" @@ -419,6 +433,38 @@ def _record_intervention_for_task( delta_pct=result.get("delta_pct"), ) + def _persist_eval_failure(self, result_payload: dict[str, Any]) -> None: + """Persist an eval-rooted baseline failure so enablement can re-run it. + + Records origin, floor, probe config, contract fingerprint, evidence, + kind and observed accuracy, and seeds ``enablement_launch_log`` so the + FRAMEWORK pump dispatches even when the failure carries no boot log. + """ + state = self.shared_state + state.enablement_origin = "eval" + state.enablement_pending = True + floor = to_float(result_payload.get(BASELINE_EVAL_ACCURACY_FLOOR_KEY)) + if floor is not None: + state.enablement_accuracy_floor = float(floor) + cfg = result_payload.get("materialized_config") + if isinstance(cfg, str) and cfg: + state.enablement_probe_config_path = cfg + fp = result_payload.get(BASELINE_EVAL_CONTRACT_FINGERPRINT_KEY) + if isinstance(fp, str) and fp: + state.enablement_eval_contract_fingerprint = fp + kind = result_payload.get(BASELINE_EVAL_FAILURE_KIND_KEY) + if isinstance(kind, str) and kind: + state.enablement_baseline_eval_kind = kind + observed = to_float(result_payload.get(BASELINE_EVAL_OBSERVED_ACCURACY_KEY)) + if observed is not None: + state.enablement_observed_accuracy = float(observed) + state.enablement_observed_task = str(result_payload.get("accuracy_task") or "") + state.enablement_observed_metric = str(result_payload.get("accuracy_metric") or "") + evidence = str(result_payload.get(BASELINE_EVAL_EVIDENCE_KEY) or "") + if evidence: + state.enablement_baseline_eval_evidence = evidence[:4000] + state.enablement_launch_log = evidence + async def _handle_unpromotable_result( self, task: Task, @@ -559,6 +605,15 @@ async def _handle_unpromotable_result( or getattr(self.shared_state, "enablement_dispatched", False) or int(getattr(self.shared_state, "enablement_attempts", 0) or 0) > 0 ) + eval_failed = bool(result_payload.get(BASELINE_EVAL_FAILED_KEY)) + from ..actions.executors._multi_node_env import is_multi_node # noqa: PLC0415 + + # Single-node eval-pending failure: throughput measured fine and the + # eval is expected to re-run under enablement, so do not spend the + # baseline_failed budget yet. Multi-node keeps the strict backstop. + eval_pending_suppress = eval_failed and not is_multi_node() + if eval_failed: + self._persist_eval_failure(result_payload) if err_class == "fast_exit_arg_error": self.shared_state.baseline_arg_error_streak += 1 if self.shared_state.baseline_arg_error_streak >= 2: @@ -566,7 +621,7 @@ async def _handle_unpromotable_result( else: self.shared_state.baseline_failure_streak += 1 self.shared_state.baseline_arg_error_streak = 0 - if self.shared_state.baseline_failure_streak >= 3 and not enablement_engaged: + if self.shared_state.baseline_failure_streak >= 3 and not enablement_engaged and not eval_pending_suppress: self.shared_state.set_stop_reason("baseline_failed") # Combined backstop: count ALL baseline failures so mixed # error_classes that split the per-class streaks still fast-fail. @@ -575,6 +630,7 @@ async def _handle_unpromotable_result( self.shared_state.baseline_total_failures >= _BASELINE_MAX_TOTAL_FAILURES and not self.shared_state.stop_reason and not enablement_engaged + and not eval_pending_suppress ): self.shared_state.set_stop_reason("baseline_failed") # One-shot eager fallback: a (non-OOM) cuda-graph capture failure is @@ -599,6 +655,7 @@ async def _handle_unpromotable_result( "stop_reason": self.shared_state.stop_reason, "result_status": result_payload.get("status"), "error_class": err_class, + "baseline_eval_failed": eval_failed, } any_changed = True # Mirror the promote-path roofline failure handling: bump streak, clear gate, warn. @@ -2205,6 +2262,9 @@ async def _promote_baseline( self.shared_state.baseline_tput = float(tput) self.shared_state.baseline_failure_streak = 0 self.shared_state.baseline_arg_error_streak = 0 + # A genuine anchor clears any lingering eval-failure trigger state. + self.shared_state.enablement_origin = "" + self.shared_state.enablement_pending = False changed = True acc = result.get("accuracy") if isinstance(acc, (int, float)): From 57e38af50622d9237f0753544efc4a96787e4f69 Mon Sep 17 00:00:00 2001 From: ZhengGong-amd Date: Sat, 25 Jul 2026 16:59:16 +0000 Subject: [PATCH 04/32] feat(enablement): thread eval-origin trigger context through the enablement chain Carry the eval origin, effective accuracy floor, probe config path and eval-contract fingerprint from SharedState to every downstream task so an eval-triggered enablement round keeps the original workload/eval contract: - framework: _enablement_carrier_params feeds the specialist params and the targeted-build launch probe; the launch probe prefers the eval probe config over the (unpromoted) baseline config. - explore: both autosubmit bridges forward the carriers onto the integrate task and bench against the probe config instead of the shipped default. - integrate_patch: the enablement runnable gate reads the threaded floor when present, else the canonical reader. Carriers live on resume-safe SharedState, so serial rearm, resume and the build->launch-probe path inherit them without extra plumbing. Boot-origin enablement is unchanged (carriers are empty). Co-Authored-By: Claude Opus 4.8 (1M context) --- ...test_enablement_coordinator_wiring_unit.py | 17 ++++++++++ .../tests/test_framework_local_explore.py | 31 +++++++++++++++++++ .../actions/executors/integrate_patch.py | 3 +- src/hyperloom/orchestrator/phases/explore.py | 20 ++++++++++++ .../orchestrator/phases/framework.py | 30 +++++++++++++++++- 5 files changed, 99 insertions(+), 2 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_enablement_coordinator_wiring_unit.py b/src/hyperloom/inference_optimizer/tests/test_enablement_coordinator_wiring_unit.py index 852853e68..80e678167 100644 --- a/src/hyperloom/inference_optimizer/tests/test_enablement_coordinator_wiring_unit.py +++ b/src/hyperloom/inference_optimizer/tests/test_enablement_coordinator_wiring_unit.py @@ -95,6 +95,23 @@ def test_build_params_actionable_failure_tags_enablement(monkeypatch): assert "RUNNABILITY" in params["notes"] assert "git apply --check" in params["notes"] assert "GLM-5" in params["notes"] + # Boot-origin: no eval carriers. + assert "enablement_origin" not in params + + +def test_build_params_threads_eval_origin_carriers(monkeypatch): + _stub_enumerate(monkeypatch, []) + fake = _fake_self() + fake.shared_state.enablement_origin = "eval" + fake.shared_state.enablement_accuracy_floor = 0.3 + fake.shared_state.enablement_probe_config_path = "/runs/baseline/materialized.yaml" + fake.shared_state.enablement_eval_contract_fingerprint = "abc123" + params = Coordinator._build_enablement_specialist_params(fake, _MISSING_ARCH_LOG) + assert params is not None + assert params["enablement_origin"] == "eval" + assert params["enablement_accuracy_floor"] == 0.3 + assert params["enablement_probe_config_path"] == "/runs/baseline/materialized.yaml" + assert params["enablement_eval_contract_fingerprint"] == "abc123" _TRANSFORMERS_UNRECOGNIZED_LOG = ( diff --git a/src/hyperloom/inference_optimizer/tests/test_framework_local_explore.py b/src/hyperloom/inference_optimizer/tests/test_framework_local_explore.py index 30038f06d..50b4a4c38 100644 --- a/src/hyperloom/inference_optimizer/tests/test_framework_local_explore.py +++ b/src/hyperloom/inference_optimizer/tests/test_framework_local_explore.py @@ -355,3 +355,34 @@ async def _audit(_candidate: dict[str, Any]) -> dict[str, Any]: assert audited["n"] == 0, "local-explore arm must not run the PR semantic audit" assert len(stub.tasks.created) == 1 assert stub.tasks.created[0]["params"]["framework_local_explore"] is True + + +def test_forward_enablement_carriers_eval_origin(): + from hyperloom.orchestrator.phases.explore import _forward_enablement_carriers + + src = { + "enablement_origin": "eval", + "enablement_accuracy_floor": 0.4, + "enablement_probe_config_path": "/runs/baseline/materialized.yaml", + "enablement_eval_contract_fingerprint": "fp1", + } + dst: dict[str, Any] = {} + _forward_enablement_carriers(src, dst) + assert dst["enablement_origin"] == "eval" + assert dst["enablement_accuracy_floor"] == 0.4 + assert dst["enablement_probe_config_path"] == "/runs/baseline/materialized.yaml" + assert dst["enablement_eval_contract_fingerprint"] == "fp1" + # Benches against the original workload config, not the shipped default. + assert dst["config_path"] == "/runs/baseline/materialized.yaml" + + +def test_forward_enablement_carriers_boot_origin_noop(): + from hyperloom.orchestrator.phases.explore import _forward_enablement_carriers + + dst: dict[str, Any] = {} + _forward_enablement_carriers({}, dst) + assert dst == {} + # An existing config_path is not overwritten for boot-origin. + dst2 = {"config_path": "/keep.yaml"} + _forward_enablement_carriers({"enablement_origin": ""}, dst2) + assert dst2 == {"config_path": "/keep.yaml"} diff --git a/src/hyperloom/orchestrator/actions/executors/integrate_patch.py b/src/hyperloom/orchestrator/actions/executors/integrate_patch.py index 2ed3f0999..6d325eca8 100644 --- a/src/hyperloom/orchestrator/actions/executors/integrate_patch.py +++ b/src/hyperloom/orchestrator/actions/executors/integrate_patch.py @@ -2194,7 +2194,8 @@ def _gc_on_revert() -> None: probe_timed_out = bool(gate_evidence.get("timed_out")) enablement_accuracy = gate_evidence.get("enablement_accuracy") - floor = enablement_accuracy_floor() + _param_floor = params.get("enablement_accuracy_floor") + floor = float(_param_floor) if isinstance(_param_floor, (int, float)) else enablement_accuracy_floor() correctness_ok: bool | None if isinstance(enablement_accuracy, (int, float)) and not _math.isnan(float(enablement_accuracy)): correctness_ok = float(enablement_accuracy) > floor diff --git a/src/hyperloom/orchestrator/phases/explore.py b/src/hyperloom/orchestrator/phases/explore.py index 49848925f..2b09c5cf3 100644 --- a/src/hyperloom/orchestrator/phases/explore.py +++ b/src/hyperloom/orchestrator/phases/explore.py @@ -33,6 +33,24 @@ log = _logging.getLogger(__name__) +def _forward_enablement_carriers(src: dict[str, Any], dst: dict[str, Any]) -> None: + """Copy eval-origin trigger context from specialist params to the integrate task.""" + origin = str(src.get("enablement_origin") or "") + if not origin: + return + dst["enablement_origin"] = origin + dst["enablement_accuracy_floor"] = float(src.get("enablement_accuracy_floor") or 0.0) + fp = str(src.get("enablement_eval_contract_fingerprint") or "") + if fp: + dst["enablement_eval_contract_fingerprint"] = fp + cfg = str(src.get("enablement_probe_config_path") or "") + if cfg: + dst["enablement_probe_config_path"] = cfg + # Bench the candidate against the original workload/eval contract rather + # than the shipped default config. + dst.setdefault("config_path", cfg) + + class ExplorePhase(PhaseHandler): """Extracted phase handler; delegates unknown attrs to its Coordinator.""" @@ -1471,6 +1489,7 @@ async def _maybe_autosubmit_specialist_patches( # integrate_patch applies the runnable_decision gate. if bool(spec_params.get("enablement")): integrate_params["enablement"] = True + _forward_enablement_carriers(spec_params, integrate_params) probe = str(spec_params.get("launch_probe") or "").strip() if probe: integrate_params["launch_probe"] = probe @@ -1631,6 +1650,7 @@ async def _maybe_autosubmit_framework_config( # framework.py::_maybe_rearm_enablement. if bool(spec_params.get("enablement")): integrate_params["enablement"] = True + _forward_enablement_carriers(spec_params, integrate_params) probe = str(spec_params.get("launch_probe") or "").strip() if probe: integrate_params["launch_probe"] = probe diff --git a/src/hyperloom/orchestrator/phases/framework.py b/src/hyperloom/orchestrator/phases/framework.py index 47193a614..cd7ff175b 100644 --- a/src/hyperloom/orchestrator/phases/framework.py +++ b/src/hyperloom/orchestrator/phases/framework.py @@ -117,6 +117,27 @@ def _maybe_build_runtime_candidate( return None +def _enablement_carrier_params(state: Any) -> dict[str, Any]: + """eval-origin trigger context threaded to specialist/integrate/build tasks. + + Empty for boot-origin enablement so the boot path is unchanged. + """ + origin = str(getattr(state, "enablement_origin", "") or "") + if not origin: + return {} + out: dict[str, Any] = { + "enablement_origin": origin, + "enablement_accuracy_floor": float(getattr(state, "enablement_accuracy_floor", 0.0) or 0.0), + } + cfg = str(getattr(state, "enablement_probe_config_path", "") or "") + if cfg: + out["enablement_probe_config_path"] = cfg + fp = str(getattr(state, "enablement_eval_contract_fingerprint", "") or "") + if fp: + out["enablement_eval_contract_fingerprint"] = fp + return out + + def _maybe_build_localization_candidate( capability_gap: Any, *, @@ -1208,6 +1229,8 @@ def _build_enablement_specialist_params(self, launch_log: str, *, attempt: int = "notes": notes, # Whole-machine GPU request. Empty on multi-node / no-GPU hosts. **self._framework_gpu_params(), + # eval-origin trigger context (empty for boot-origin enablement). + **_enablement_carrier_params(state), } if runtime_candidate is not None: params_out["runtime_candidate"] = runtime_candidate @@ -4484,8 +4507,13 @@ async def _enqueue_build_launch_probe(self, build_task_id: str, br: Any) -> None "enablement_before_signature": before_sig, "source": "coordinator_internal", **self._framework_gpu_params(), + **_enablement_carrier_params(state), } - cfg = str(getattr(state, "baseline_config_path", "") or "") + # Prefer the eval-origin probe config so the re-run keeps the original + # workload/eval contract; fall back to the promoted baseline config. + cfg = str(getattr(state, "enablement_probe_config_path", "") or "") or str( + getattr(state, "baseline_config_path", "") or "" + ) if cfg: params["config_path"] = cfg idem = f"build_launch_probe:{build_task_id}" From 25f8d6e88ada695f54182c7ed94c3a5c3ae9c249 Mon Sep 17 00:00:00 2001 From: ZhengGong-amd Date: Sun, 26 Jul 2026 01:57:51 +0000 Subject: [PATCH 05/32] feat(enablement): strict accuracy KEEP gate for eval-origin patches The enablement runnable gate now enforces accuracy for eval-origin rounds: - eval-origin: a missing accuracy fails closed (REVERT) instead of a provisional KEEP; a finite score must be >0 and >= the effective floor to KEEP. - eval-contract fingerprint drift (dataset/task/metric/config changed vs the captured baseline contract) forces a REVERT so a score run under a different contract can never earn KEEP. - _bench_patch surfaces the observed task/metric and computes the candidate's contract fingerprint; the gate records origin, observed accuracy, floor, task/metric, fingerprint, drift and failure kind on every outcome. Boot-origin enablement is unchanged (missing accuracy stays provisional). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../tests/test_integrate_patch_executor.py | 75 +++++++++++++++++++ .../actions/executors/integrate_patch.py | 68 +++++++++++++++-- 2 files changed, 136 insertions(+), 7 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_integrate_patch_executor.py b/src/hyperloom/inference_optimizer/tests/test_integrate_patch_executor.py index 48b79fa82..463b89fe0 100644 --- a/src/hyperloom/inference_optimizer/tests/test_integrate_patch_executor.py +++ b/src/hyperloom/inference_optimizer/tests/test_integrate_patch_executor.py @@ -572,6 +572,10 @@ async def _run_enablement_integrate( enablement_accuracy=None, bench_error: str = "", before_signature=None, + enablement_origin: str = "", + accuracy_floor=None, + expected_fingerprint: str = "", + candidate_fingerprint: str = "", ): session_dir = tmp_path / "session" session_dir.mkdir() @@ -589,6 +593,9 @@ async def _fake_bench(**_kwargs): return bench_result, { "accuracy_pass": None, "enablement_accuracy": enablement_accuracy, + "enablement_accuracy_task": "gsm8k", + "enablement_accuracy_metric": "exact_match", + "enablement_eval_contract_fingerprint": candidate_fingerprint, "timed_out": False, } @@ -606,6 +613,12 @@ async def _noop_kb(**_kwargs): } if before_signature is not None: params["enablement_before_signature"] = before_signature + if enablement_origin: + params["enablement_origin"] = enablement_origin + if accuracy_floor is not None: + params["enablement_accuracy_floor"] = accuracy_floor + if expected_fingerprint: + params["enablement_eval_contract_fingerprint"] = expected_fingerprint ctx = _make_ctx("t-int-en", params) return await executor(ctx), repo @@ -654,6 +667,68 @@ async def test_enablement_reverts_when_accuracy_zero(tmp_path: Path, monkeypatch assert (repo / "src.py").read_text().endswith("return 1\n") +@pytest.mark.asyncio +async def test_enablement_eval_origin_reverts_when_accuracy_missing(tmp_path: Path, monkeypatch): + """eval-origin: booted but no accuracy -> fail closed (REVERT), not provisional KEEP.""" + result, repo = await _run_enablement_integrate( + tmp_path, monkeypatch, booted=True, enablement_accuracy=None, enablement_origin="eval" + ) + assert result["status"] == "reverted" + assert result["runnable"] is False + assert result["correctness_verified"] is False + assert result["enablement_origin"] == "eval" + + +@pytest.mark.asyncio +async def test_enablement_eval_origin_reverts_below_configured_floor(tmp_path: Path, monkeypatch): + result, _ = await _run_enablement_integrate( + tmp_path, monkeypatch, booted=True, enablement_accuracy=0.2, enablement_origin="eval", accuracy_floor=0.5 + ) + assert result["status"] == "reverted" + assert result["enablement_eval_failure_kind"] == "accuracy_below_floor" + assert result["enablement_observed_accuracy"] == 0.2 + + +@pytest.mark.asyncio +async def test_enablement_eval_origin_keeps_at_or_above_floor(tmp_path: Path, monkeypatch): + result, _ = await _run_enablement_integrate( + tmp_path, monkeypatch, booted=True, enablement_accuracy=0.5, enablement_origin="eval", accuracy_floor=0.5 + ) + assert result["status"] == "kept" + assert result["correctness_verified"] is True + assert result["provisional"] is False + + +@pytest.mark.asyncio +async def test_enablement_eval_origin_reverts_on_fingerprint_drift(tmp_path: Path, monkeypatch): + result, _ = await _run_enablement_integrate( + tmp_path, + monkeypatch, + booted=True, + enablement_accuracy=0.9, + enablement_origin="eval", + expected_fingerprint="fp_baseline", + candidate_fingerprint="fp_changed", + ) + assert result["status"] == "reverted" + assert result["enablement_eval_contract_drift"] is True + + +@pytest.mark.asyncio +async def test_enablement_eval_origin_keeps_when_fingerprint_matches(tmp_path: Path, monkeypatch): + result, _ = await _run_enablement_integrate( + tmp_path, + monkeypatch, + booted=True, + enablement_accuracy=0.9, + enablement_origin="eval", + expected_fingerprint="fp_same", + candidate_fingerprint="fp_same", + ) + assert result["status"] == "kept" + assert result["enablement_eval_contract_drift"] is False + + @pytest.mark.asyncio async def test_enablement_reverts_when_accuracy_nan(tmp_path: Path, monkeypatch): """Booted but NaN accuracy -> REVERT (treated as garbage, not provisional).""" diff --git a/src/hyperloom/orchestrator/actions/executors/integrate_patch.py b/src/hyperloom/orchestrator/actions/executors/integrate_patch.py index 6d325eca8..ed2867aa2 100644 --- a/src/hyperloom/orchestrator/actions/executors/integrate_patch.py +++ b/src/hyperloom/orchestrator/actions/executors/integrate_patch.py @@ -23,8 +23,11 @@ from ...specialists.patch_safety import patch_file_targets, patch_targets_missing from ._accuracy_gate import ( accuracy_keep_block, + accuracy_meets_floor, accuracy_passed, + classify_accuracy_failure, enablement_accuracy_floor, + eval_contract_fingerprint, parse_eval_results, ) from ._apply_feedback import ApplyFeedback, build_apply_feedback @@ -2196,13 +2199,39 @@ def _gc_on_revert() -> None: enablement_accuracy = gate_evidence.get("enablement_accuracy") _param_floor = params.get("enablement_accuracy_floor") floor = float(_param_floor) if isinstance(_param_floor, (int, float)) else enablement_accuracy_floor() + eval_origin = str(params.get("enablement_origin") or "") == "eval" + accuracy_kind = classify_accuracy_failure(enablement_accuracy, floor) correctness_ok: bool | None - if isinstance(enablement_accuracy, (int, float)) and not _math.isnan(float(enablement_accuracy)): - correctness_ok = float(enablement_accuracy) > floor - elif isinstance(enablement_accuracy, float) and _math.isnan(enablement_accuracy): - correctness_ok = False + if enablement_accuracy is None: + # Truly absent: eval-origin fails closed; boot-origin stays provisional. + correctness_ok = False if eval_origin else None + elif accuracy_meets_floor(enablement_accuracy, floor): + correctness_ok = True else: - correctness_ok = None + # Present but below floor / non-positive / non-finite. + correctness_ok = False + # eval-origin only: a changed eval contract (dataset/task/metric/config) + # means the score is not comparable — fail closed. + expected_fp = str(params.get("enablement_eval_contract_fingerprint") or "") + candidate_fp = str(gate_evidence.get("enablement_eval_contract_fingerprint") or "") + fp_drift = eval_origin and bool(expected_fp) and bool(candidate_fp) and expected_fp != candidate_fp + if fp_drift: + correctness_ok = False + log.warning( + "integrate_patch: eval-contract fingerprint drift (expected %s, got %s); reverting", + expected_fp, + candidate_fp, + ) + eval_provenance = { + "enablement_origin": str(params.get("enablement_origin") or ""), + "enablement_observed_accuracy": enablement_accuracy, + "enablement_accuracy_floor": floor, + "accuracy_task": gate_evidence.get("enablement_accuracy_task") or "", + "accuracy_metric": gate_evidence.get("enablement_accuracy_metric") or "", + "enablement_eval_contract_fingerprint": candidate_fp, + "enablement_eval_contract_drift": fp_drift, + "enablement_eval_failure_kind": accuracy_kind or "", + } after_signature = classify_failure(str(bench_result.get("error") or "")) before_signature: FailureSignature | None = None @@ -2259,6 +2288,7 @@ def _gc_on_revert() -> None: "setup_commands_applied": list(setup_result.get("applied") or []), "bench_result": bench_result, "workspace": str(output_root), + **eval_provenance, }, ) artifacts_reverted = self._revert_artifacts(applied_artifacts) @@ -2285,9 +2315,14 @@ def _gc_on_revert() -> None: "enablement": True, "runnable": False, "correctness_verified": correctness_ok is True, - "reason": f"enablement not runnable: {run_reason}", + "reason": ( + "enablement eval-contract drift; reverted" + if fp_drift + else f"enablement not runnable: {run_reason}" + ), "bench_result": bench_result, "workspace": str(output_root), + **eval_provenance, }, ) @@ -2319,6 +2354,7 @@ def _gc_on_revert() -> None: "setup_commands_applied": list(setup_result.get("applied") or []), "bench_result": bench_result, "workspace": str(output_root), + **eval_provenance, } # Record the KEEP'd attempt runtime so it survives rearm and every later # bench in this session re-activates it. @@ -3022,6 +3058,9 @@ async def _bench_patch( # Enablement path: surface the raw accuracy so the branch can apply a floor. enablement_accuracy: float | None = None + enablement_accuracy_task = "" + enablement_accuracy_metric = "" + candidate_fingerprint = "" if bool(params.get("enablement")) and bench.get("status") == "succeeded": try: eval_results = parse_eval_results( @@ -3031,10 +3070,25 @@ async def _bench_patch( acc = eval_results.get("accuracy") if isinstance(acc, (int, float)): enablement_accuracy = float(acc) + enablement_accuracy_task = str(eval_results.get("task") or "") + enablement_accuracy_metric = str(eval_results.get("metric") or "") + candidate_fingerprint = eval_contract_fingerprint( + config_path=config_path, + framework=params.get("framework") or os.environ.get("FRAMEWORK") or None, + model=resolved_model, + task=enablement_accuracy_task, + metric=enablement_accuracy_metric, + ) except Exception: # noqa: BLE001 — eval may not produce a result log.debug("integrate_patch: enablement eval parse failed", exc_info=True) - return bench, {"accuracy_pass": accuracy_pass, "enablement_accuracy": enablement_accuracy} + return bench, { + "accuracy_pass": accuracy_pass, + "enablement_accuracy": enablement_accuracy, + "enablement_accuracy_task": enablement_accuracy_task, + "enablement_accuracy_metric": enablement_accuracy_metric, + "enablement_eval_contract_fingerprint": candidate_fingerprint, + } @staticmethod def _framework_run_eval_envs(params: dict[str, Any]) -> dict[str, Any] | None: From 6b012a5290c26abc684fd6558f9a01dad9b634e6 Mon Sep 17 00:00:00 2001 From: ZhengGong-amd Date: Sun, 26 Jul 2026 02:06:08 +0000 Subject: [PATCH 06/32] feat(enablement): revalidate eval-origin KEEP with a genuine baseline before success An eval-origin enablement KEEP is no longer terminal: the patch passed the gate, but tput/accuracy only become official once a real baseline re-runs the original eval contract. - _maybe_rearm_enablement: eval-origin KEEP sets enablement_validation_pending (not enablement_succeeded) and keeps the stall streak so repeated KEEP->revalidation-fail cycles still reach the stall cap. - new _maybe_enqueue_enablement_baseline_revalidation pump enqueues exactly one genuine baseline with the probe config and RUN_EVAL on; the specialist pump is blocked while validation is pending. - _promote_baseline finalizes enablement_succeeded only when the revalidation baseline promotes with accuracy at/above the floor; a still-failing revalidation clears pending, reopens authoring, and counts a stall (capped by enablement_stalled). - close guard stays active while validation is pending, even with a stray tput. - new SharedState.enablement_validation_pending (resume-safe). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../tests/test_coordinator_runtime.py | 35 ++++++++ ...test_enablement_coordinator_wiring_unit.py | 80 ++++++++++++++++++- .../orchestrator/loop/coordinator.py | 1 + src/hyperloom/orchestrator/loop/writeback.py | 20 ++++- .../orchestrator/phases/framework.py | 57 ++++++++++++- .../orchestrator/state/shared_state.py | 5 +- 6 files changed, 193 insertions(+), 5 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_coordinator_runtime.py b/src/hyperloom/inference_optimizer/tests/test_coordinator_runtime.py index 684e26825..2fd0aaa24 100644 --- a/src/hyperloom/inference_optimizer/tests/test_coordinator_runtime.py +++ b/src/hyperloom/inference_optimizer/tests/test_coordinator_runtime.py @@ -944,6 +944,41 @@ async def test_handle_unpromotable_baseline_eval_pending_multi_node_still_stops( await c.stop() +@pytest.mark.asyncio +async def test_promote_baseline_finalizes_eval_origin_when_accuracy_meets_floor(session_dir): + c = Coordinator(session_dir, backends=_silent_backends()) + _mute_action_scoring(c) + try: + c.shared_state.enablement_origin = "eval" + c.shared_state.enablement_validation_pending = True + c.shared_state.enablement_accuracy_floor = 0.3 + await c._promote_to_shared_state( + "baseline", + {"output_throughput": 1000.0, "completed_requests": 10, "accuracy": 0.42}, + task=_mk_task("baseline", "t-reval-ok"), + ) + assert c.shared_state.baseline_tput == 1000.0 + assert c.shared_state.enablement_succeeded is True + assert c.shared_state.enablement_validation_pending is False + assert c.shared_state.enablement_origin == "" + finally: + await c.stop() + + +@pytest.mark.asyncio +async def test_persist_eval_failure_clears_pending_and_counts_stall(session_dir, monkeypatch): + monkeypatch.delenv("INFERENCE_OPTIMIZER_NODES", raising=False) + c = Coordinator(session_dir, backends=_silent_backends()) + _mute_action_scoring(c) + try: + c.shared_state.enablement_validation_pending = True + await c._handle_unpromotable_result(_mk_task("baseline", "t-reval-fail"), _eval_failed_result()) + assert c.shared_state.enablement_validation_pending is False + assert c.shared_state.enablement_stall_streak == 1 + finally: + await c.stop() + + @pytest.mark.asyncio async def test_handle_unpromotable_records_for_non_baseline_kinds(session_dir): c = Coordinator(session_dir, backends=_silent_backends()) diff --git a/src/hyperloom/inference_optimizer/tests/test_enablement_coordinator_wiring_unit.py b/src/hyperloom/inference_optimizer/tests/test_enablement_coordinator_wiring_unit.py index 80e678167..c7972146a 100644 --- a/src/hyperloom/inference_optimizer/tests/test_enablement_coordinator_wiring_unit.py +++ b/src/hyperloom/inference_optimizer/tests/test_enablement_coordinator_wiring_unit.py @@ -194,14 +194,22 @@ def test_build_params_none_for_blank_log(): class _FakeTasks: - def __init__(self): + def __init__(self, queued=None, running=None): self.created = [] + self._queued = list(queued or []) + self._running = list(running or []) async def create_or_return_existing(self, **kwargs): self.created.append(kwargs) task = types.SimpleNamespace(task_id=f"spec-{len(self.created)}", state="queued") return task, False + async def queued(self): + return list(self._queued) + + async def running(self): + return list(self._running) + def _enqueue_self(**state_kw): state = types.SimpleNamespace( @@ -219,6 +227,11 @@ def _enqueue_self(**state_kw): enablement_launch_log=state_kw.get("enablement_launch_log", _MISSING_ARCH_LOG), enablement_inflight_task_id=state_kw.get("enablement_inflight_task_id", ""), enablement_dispatch_tick=state_kw.get("enablement_dispatch_tick", -1), + enablement_origin=state_kw.get("enablement_origin", ""), + enablement_validation_pending=state_kw.get("enablement_validation_pending", False), + enablement_probe_config_path=state_kw.get("enablement_probe_config_path", ""), + enablement_accuracy_floor=state_kw.get("enablement_accuracy_floor", 0.0), + enablement_eval_contract_fingerprint=state_kw.get("enablement_eval_contract_fingerprint", ""), tick=state_kw.get("tick", 0), stop_reason=state_kw.get("stop_reason", ""), save=lambda *a, **k: None, @@ -258,6 +271,9 @@ async def _record_obs(_source, _topic, payload): Coordinator._maybe_record_enablement_human_review, fake ) fake._maybe_rearm_enablement = types.MethodType(Coordinator._maybe_rearm_enablement, fake) + fake._maybe_enqueue_enablement_baseline_revalidation = types.MethodType( + Coordinator._maybe_enqueue_enablement_baseline_revalidation, fake + ) fake._enablement_round_silently_finished = types.MethodType( Coordinator._enablement_round_silently_finished, fake ) @@ -405,6 +421,53 @@ async def test_rearm_kept_is_terminal(monkeypatch): assert await Coordinator._maybe_enqueue_enablement_specialist(fake) == "" +@pytest.mark.asyncio +async def test_rearm_kept_eval_origin_holds_for_revalidation(monkeypatch): + fake = _enqueue_self(enablement_dispatched=True, enablement_origin="eval") + fake._maybe_rearm_enablement({"enablement": True, "status": "kept"}) + # eval-origin KEEP is NOT terminal: hold succeeded, open validation window. + assert fake.shared_state.enablement_validation_pending is True + assert fake.shared_state.enablement_succeeded is False + assert fake.shared_state.enablement_dispatched is False + # The specialist pump is blocked while validation is pending. + from hyperloom.orchestrator.actions.executors import _multi_node_env as mne + + monkeypatch.setattr(mne, "is_multi_node", lambda: False) + assert await Coordinator._maybe_enqueue_enablement_specialist(fake) == "" + + +@pytest.mark.asyncio +async def test_revalidation_enqueues_genuine_baseline(): + fake = _enqueue_self( + enablement_validation_pending=True, + enablement_origin="eval", + enablement_probe_config_path="/runs/baseline/materialized.yaml", + enablement_accuracy_floor=0.3, + ) + tid = await fake._maybe_enqueue_enablement_baseline_revalidation() + assert tid + created = fake.tasks.created[-1] + assert created["kind"] == "baseline" + assert created["params"]["config_path"] == "/runs/baseline/materialized.yaml" + assert created["params"]["disable_run_eval"] is False + assert created["params"]["enablement_origin"] == "eval" + + +@pytest.mark.asyncio +async def test_revalidation_skips_when_tput_positive(): + fake = _enqueue_self(enablement_validation_pending=True, enablement_origin="eval", baseline_tput=123.0) + assert await fake._maybe_enqueue_enablement_baseline_revalidation() == "" + assert fake.tasks.created == [] + + +@pytest.mark.asyncio +async def test_revalidation_skips_when_baseline_in_flight(): + fake = _enqueue_self(enablement_validation_pending=True, enablement_origin="eval") + fake.tasks = _FakeTasks(running=[types.SimpleNamespace(kind="baseline")]) + assert await fake._maybe_enqueue_enablement_baseline_revalidation() == "" + assert fake.tasks.created == [] + + @pytest.mark.asyncio async def test_rearm_ignores_non_enablement(monkeypatch): fake = _enqueue_self(enablement_dispatched=True) @@ -563,6 +626,21 @@ def test_enablement_close_guard_blocks_premature_skip_to_close(): assert s.enablement_close_guard_active() is False +def test_enablement_close_guard_active_during_validation_pending(): + """An eval-origin KEEP awaiting revalidation must keep the guard active even + if a stale positive tput is present.""" + from hyperloom.orchestrator.state.shared_state import SharedState + + s = SharedState() + s.phase = "SWEEP" + s.baseline_tput = 100.0 + s.enablement_succeeded = False + s.enablement_validation_pending = True + assert s.enablement_close_guard_active() is True + s.enablement_validation_pending = False + assert s.enablement_close_guard_active() is False + + def test_build_params_threads_base_setup_commands_when_stacked(monkeypatch): """Q3: stacked base setup commands are passed to the next round + noted.""" _stub_enumerate(monkeypatch, []) diff --git a/src/hyperloom/orchestrator/loop/coordinator.py b/src/hyperloom/orchestrator/loop/coordinator.py index 8809859c9..ed75561c6 100644 --- a/src/hyperloom/orchestrator/loop/coordinator.py +++ b/src/hyperloom/orchestrator/loop/coordinator.py @@ -1047,6 +1047,7 @@ def router(self) -> IntentRouter: "_maybe_reauthor_from_critic_feedback": "phase_framework", "_pump_framework_agent_phase_safely": "phase_framework", "_pump_enablement_safely": "phase_framework", + "_maybe_enqueue_enablement_baseline_revalidation": "phase_framework", "_record_framework_agent_authored_outcome": "phase_framework", "_recover_framework_agent_authoring_outcome": "phase_framework", "_record_framework_agent_authoring_empty_outcome": "phase_framework", diff --git a/src/hyperloom/orchestrator/loop/writeback.py b/src/hyperloom/orchestrator/loop/writeback.py index ed523ca2b..33526558d 100644 --- a/src/hyperloom/orchestrator/loop/writeback.py +++ b/src/hyperloom/orchestrator/loop/writeback.py @@ -49,12 +49,14 @@ BASELINE_EVAL_FAILED_KEY, BASELINE_EVAL_FAILURE_KIND_KEY, BASELINE_EVAL_OBSERVED_ACCURACY_KEY, + accuracy_meets_floor, ) from .coordinator import ( _AUDIT_ACTIONS, _BASELINE_MAX_TOTAL_FAILURES, _DEFAULT_RESUME_DRIFT_FLOOR_PCT, + _ENABLEMENT_MAX_STALL, _SEVERITY_CRASH, _SEVERITY_REGRESS, PendingProposal, @@ -441,8 +443,16 @@ def _persist_eval_failure(self, result_payload: dict[str, Any]) -> None: FRAMEWORK pump dispatches even when the failure carries no boot log. """ state = self.shared_state + was_validation_pending = bool(getattr(state, "enablement_validation_pending", False)) state.enablement_origin = "eval" state.enablement_pending = True + # A failed revalidation reopens the authoring loop and counts as a + # no-progress round so the enablement_stalled cap can still terminate. + if was_validation_pending: + state.enablement_validation_pending = False + state.enablement_stall_streak = int(getattr(state, "enablement_stall_streak", 0) or 0) + 1 + if state.enablement_stall_streak >= _ENABLEMENT_MAX_STALL and not state.stop_reason: + state.set_stop_reason("enablement_stalled") floor = to_float(result_payload.get(BASELINE_EVAL_ACCURACY_FLOOR_KEY)) if floor is not None: state.enablement_accuracy_floor = float(floor) @@ -2262,7 +2272,15 @@ async def _promote_baseline( self.shared_state.baseline_tput = float(tput) self.shared_state.baseline_failure_streak = 0 self.shared_state.baseline_arg_error_streak = 0 - # A genuine anchor clears any lingering eval-failure trigger state. + # A genuine anchor revalidates an eval-origin enablement: reaching here + # means the result was promotable (not baseline_eval_failed), i.e. the + # accuracy met the floor, so finalize the enablement success. + if bool(getattr(self.shared_state, "enablement_validation_pending", False)): + acc = result.get("accuracy") + floor = float(getattr(self.shared_state, "enablement_accuracy_floor", 0.0) or 0.0) + if accuracy_meets_floor(acc, floor): + self.shared_state.enablement_succeeded = True + self.shared_state.enablement_validation_pending = False self.shared_state.enablement_origin = "" self.shared_state.enablement_pending = False changed = True diff --git a/src/hyperloom/orchestrator/phases/framework.py b/src/hyperloom/orchestrator/phases/framework.py index cd7ff175b..e8f9452c9 100644 --- a/src/hyperloom/orchestrator/phases/framework.py +++ b/src/hyperloom/orchestrator/phases/framework.py @@ -1519,6 +1519,10 @@ async def _maybe_enqueue_enablement_specialist(self) -> str: state = self.shared_state if bool(getattr(state, "enablement_succeeded", False)): return "" + if bool(getattr(state, "enablement_validation_pending", False)): + # A KEEP'd eval-origin patch is awaiting genuine-baseline revalidation; + # do not dispatch another authoring round until that resolves. + return "" if bool(getattr(state, "enablement_dispatched", False)): # Watchdog: an enablement round is marked in-flight. If it has # silently finished without ever calling _maybe_rearm_enablement @@ -1819,11 +1823,20 @@ def _stack_kept_runtime() -> None: state.enablement_localization_manifest = existing if status == "kept": - state.enablement_succeeded = True - state.enablement_stall_streak = 0 _reset_baseline_failure_backstop() _stack_setup_commands() _stack_kept_runtime() + if str(getattr(state, "enablement_origin", "") or "") == "eval": + # eval-origin: the patch boots and re-passed accuracy in the gate, + # but tput/accuracy only become official once a GENUINE baseline + # promotes. Hold succeeded; open the revalidation window. Keep the + # stall streak so repeated KEEP->revalidation-fail cycles still + # reach the stall cap. + state.enablement_validation_pending = True + state.enablement_dispatched = False + else: + state.enablement_succeeded = True + state.enablement_stall_streak = 0 elif status == "advanced" or bool(res.get("advanced")): # Forward progress on a serial enablement: stack the progressing # patches + setup commands and pivot to the newly-revealed gap. @@ -4533,6 +4546,42 @@ async def _enqueue_build_launch_probe(self, build_task_id: str, br: Any) -> None existing, ) + async def _maybe_enqueue_enablement_baseline_revalidation(self) -> str: + """Enqueue one genuine baseline to revalidate a KEEP'd eval-origin patch. + + Reproduces the original workload/eval contract (probe config, RUN_EVAL on) + so the accuracy is measured for real; the KEEP is finalized in + ``_promote_baseline`` only when that baseline promotes with accuracy at or + above the floor. Idempotent and one-at-a-time. + """ + state = self.shared_state + if not bool(getattr(state, "enablement_validation_pending", False)): + return "" + if float(getattr(state, "baseline_tput", 0.0) or 0.0) > 0: + return "" + try: + for t in (*await self.tasks.queued(), *await self.tasks.running()): + if getattr(t, "kind", "") == "baseline": + return "" + except Exception: # noqa: BLE001 — defensive + return "" + params: dict[str, Any] = { + "source": "coordinator_internal", + "reason": "enablement_eval_revalidation", + "disable_run_eval": False, + **_enablement_carrier_params(state), + } + cfg = str(getattr(state, "enablement_probe_config_path", "") or "") + if cfg: + params["config_path"] = cfg + attempt = int(getattr(state, "enablement_attempts", 0) or 0) + task, _existing = await self.tasks.create_or_return_existing( + kind="baseline", + params=params, + idempotency_key=f"enablement_revalidation:{attempt}", + ) + return str(getattr(task, "task_id", "") or "") + async def _pump_enablement_safely(self, *, caller: str) -> None: """Phase-independent enablement pump — runs every tick. @@ -4558,6 +4607,10 @@ async def _pump_enablement_safely(self, *, caller: str) -> None: await self._maybe_route_build_outcomes() except Exception: # noqa: BLE001 — never wedge the tick log.exception("ENABLEMENT route_build_outcomes (%s) failed", caller) + try: + await self._maybe_enqueue_enablement_baseline_revalidation() + except Exception: # noqa: BLE001 — never wedge the tick + log.exception("ENABLEMENT revalidation (%s) failed", caller) try: await self._maybe_enqueue_enablement_specialist() except Exception: # noqa: BLE001 — never wedge the tick diff --git a/src/hyperloom/orchestrator/state/shared_state.py b/src/hyperloom/orchestrator/state/shared_state.py index e5c80b823..04875fc55 100644 --- a/src/hyperloom/orchestrator/state/shared_state.py +++ b/src/hyperloom/orchestrator/state/shared_state.py @@ -420,6 +420,9 @@ class SharedState(_RenderMixin, _ExploreStateMixin): enablement_observed_task: str = "" enablement_observed_metric: str = "" enablement_pending: bool = False + # Set on an eval-origin KEEP: the patch passed the gate but a genuine baseline + # must revalidate accuracy before the run is considered enabled. + enablement_validation_pending: bool = False # Enablement path (framework-agent) state. # ``enablement_launch_log``: captured launch/traceback text when baseline # cannot launch. @@ -1497,7 +1500,7 @@ def enablement_close_guard_active(self) -> bool: phase in ("PRELUDE", "FRAMEWORK_AGENT") and float(getattr(self, "baseline_tput", 0.0) or 0.0) <= 0.0 and not bool(getattr(self, "enablement_succeeded", False)) - ) + ) or bool(getattr(self, "enablement_validation_pending", False)) # phase machine writer (Coordinator-only, single writer) def record_phase_transition( From f2b477df3f3c9e17dce83e1d84e20bf5579fede1 Mon Sep 17 00:00:00 2001 From: ZhengGong-amd Date: Sun, 26 Jul 2026 02:15:18 +0000 Subject: [PATCH 07/32] feat(enablement): classify eval failures, generalize prompts/Critic, surface in breakdown/docs Round out eval-origin enablement awareness across classifier, prompts, Critic, observability and docs: - classifier: add accuracy_below_floor (high priority) and eval_runtime_failure (lowest priority so a co-occurring import/serve-flag/tokenizer root cause still wins) kinds, seed keywords and ladder rows. - prompts: generalize the boot-only enablement framing to "boot or accuracy" in the specialist identity, mandate GOAL/invariants, domain description and the stale UNKNOWN-dispatch docstring; add an anti-gaming rule against altering the eval contract. - Critic: eval-origin enablement patches boot but may miss the accuracy floor; the downstream gate re-runs the accuracy eval, so approve on the structural bar without requiring a throughput before/after. - breakdown: EnablementBreakdown gains origin/trigger_kind/observed accuracy/ floor/task/metric/probe config/fingerprint/validation_pending, collected for eval-origin sessions. - docs: document INFERENCE_OPTIMIZER_ENABLEMENT_ON_EVAL_FAIL and INFERENCE_OPTIMIZER_ENABLEMENT_ACCURACY_FLOOR (env reference + .env.template) and the eval trigger in the optimization-loop conceptual doc. Co-Authored-By: Claude Opus 4.8 (1M context) --- .env.template | 7 ++++ docs/conceptual/optimization-loop.md | 12 ++++-- docs/reference/environment-variables.md | 14 +++++++ src/hyperloom/agents/critic/SKILL.md | 37 +++++++++++-------- .../actions/review_coordinator_inbox.md | 10 +++-- src/hyperloom/agents/framework/AGENTS.md | 9 +++-- src/hyperloom/agents/framework/README.md | 24 +++++++----- src/hyperloom/agents/framework/SKILL.md | 5 ++- src/hyperloom/agents/framework/enablement.py | 32 ++++++++++++++++ .../agents/framework/enablement_ops.py | 21 +++++++---- .../agents/framework/tests/test_enablement.py | 22 +++++++++++ .../breakdown/collectors/sessions.py | 16 +++++++- .../inference_optimizer/breakdown/schema.py | 20 ++++++++++ .../tests/test_enablement_breakdown.py | 25 +++++++++++++ .../orchestrator/phases/framework.py | 15 +++++--- .../prompts/specialist_prompt_builder.py | 14 ++++--- .../orchestrator/specialists/domains.py | 11 ++++-- 17 files changed, 230 insertions(+), 64 deletions(-) diff --git a/.env.template b/.env.template index 952bb982d..db474010a 100644 --- a/.env.template +++ b/.env.template @@ -66,6 +66,13 @@ OPENAI_BASE_URL=https:///api/v1/llm-proxy/v1 # Populated automatically by src/hyperloom/inference_optimizer/assets/install.sh probe. # INFERENCE_OPTIMIZER_FRAMEWORK_SOURCE_ROOTS= +# (Optional) When a first baseline boots but fails its accuracy eval, route it +# into enablement instead of halting the run. Default on (single-node only). +# INFERENCE_OPTIMIZER_ENABLEMENT_ON_EVAL_FAIL=1 +# (Optional) Shared accuracy floor for the eval-failure trigger AND the +# enablement KEEP gate (finite, in [0,1]). Default 0.0. +# INFERENCE_OPTIMIZER_ENABLEMENT_ACCURACY_FLOOR=0.0 + # Writable artifact root: hosts every session dir, optimizer_runs/, and the # runtime/ tree generated by install.sh (GEAK e2e checkout, # kernel-agent.env.sh, etc.). The default /workspace/hyperloom matches the in-code diff --git a/docs/conceptual/optimization-loop.md b/docs/conceptual/optimization-loop.md index 10a3426a9..1c19a4fb2 100644 --- a/docs/conceptual/optimization-loop.md +++ b/docs/conceptual/optimization-loop.md @@ -120,8 +120,9 @@ The LLM doesn't own a separate framework role in the current runtime. ### Enablement escalation ladder -When a `(model, backend)` combination cannot launch, enablement repairs it -along two axes. **Diagnosis (once):** work out which capability layer is +When a `(model, backend)` combination cannot launch — or it launches but fails +its accuracy eval (`accuracy_below_floor` / `eval_runtime_failure`) — enablement +repairs it along two axes. **Diagnosis (once):** work out which capability layer is missing — read the failure signature, the model's `config.json` architecture, the framework's supported-architecture registry and installed version, and upstream (whether the capability already exists and in which version/PR). That @@ -171,8 +172,11 @@ A verified build does not KEEP on artifact verification alone. After a build's artifacts verify, the Coordinator runs a launch probe: it boots the actual model with the built runtime through the same runnable-decision gate the authored-patch lane uses. Only a runtime that actually launches — and -passes minimal correctness — earns KEEP. Otherwise the build reverts, or, if -the boot advanced past the original failure to a new or deeper gap, the loop +passes minimal correctness — earns KEEP. For an eval-origin trigger the gate +additionally re-runs the accuracy eval against the captured contract and a +KEEP requires the accuracy to meet the floor; the KEEP is finalized only after +a genuine baseline revalidates it. Otherwise the build reverts, or, if the +boot advanced past the original failure to a new or deeper gap, the loop advances to the next round to repair that gap. ### Discovery-driven build refs diff --git a/docs/reference/environment-variables.md b/docs/reference/environment-variables.md index e47e46e68..655e58425 100644 --- a/docs/reference/environment-variables.md +++ b/docs/reference/environment-variables.md @@ -200,6 +200,20 @@ do not publish it unchanged in support bundles. --- +## Enablement accuracy trigger + +When a first baseline boots and measures throughput but its accuracy eval fails +(crashes, produces no result, or scores below the floor), enablement can repair +the model instead of halting the run. Single-node only; multi-node keeps the +existing behavior. + +| Variable | Default | Description | +|----------|---------|-------------| +| `INFERENCE_OPTIMIZER_ENABLEMENT_ON_EVAL_FAIL` | `1` (on) | Route a baseline accuracy-eval failure into enablement instead of halting. Set `0`/`false`/`no`/`off` to keep the legacy salvage/stop behavior. Reader: `_accuracy_gate.enablement_on_eval_fail_enabled`. | +| `INFERENCE_OPTIMIZER_ENABLEMENT_ACCURACY_FLOOR` | `0.0` | Shared accuracy floor used by BOTH the baseline eval-failure trigger and the enablement KEEP gate. Accepts a finite value in `[0, 1]`; out-of-range values are ignored with a warning. Reader: `_accuracy_gate.enablement_accuracy_floor`. | + +--- + ## Framework / source-tree discovery The following variables configure framework source discovery and path overrides. diff --git a/src/hyperloom/agents/critic/SKILL.md b/src/hyperloom/agents/critic/SKILL.md index 2d87fa857..9d876ae39 100644 --- a/src/hyperloom/agents/critic/SKILL.md +++ b/src/hyperloom/agents/critic/SKILL.md @@ -122,11 +122,13 @@ the absence of priors as *unknown*, not as *no contradicting prior*: - For **`enablement_landing`** proposals (enablement / framework-agent authoring `integrate_patch`): treat like `evidence_producer` — an absent KB prior is the default cold-start state, not a blocker. Do **not** block - on a missing before/after benchmark, accuracy gate, or a restated - rollback plan: the model cannot boot yet (so that evidence cannot - exist), and rollback is guaranteed by the enablement integrate executor - + runnable-decision gate. Approve unless a *contradicting* KB prior or a - packet-local defect (e.g. a malformed patch) is present. + on a missing throughput before/after or a restated rollback plan: rollback + is guaranteed by the enablement integrate executor + runnable-decision gate. + Boot-origin has no baseline yet; eval-origin booted but missed the accuracy + floor and the downstream gate re-runs the accuracy eval, so the KEEP evidence + is that re-run, not a throughput before/after. Approve unless a + *contradicting* KB prior or a packet-local defect (e.g. a malformed patch) is + present. - For **`evidence_producer`** proposals (`explore` / `specialist` / `profile` / `kernel_opt` / ...): an absent KB prior is the **default cold-start state**, not a blocker. Approve unless a *contradicting* @@ -148,7 +150,7 @@ Every proposal in `judge_bundle.proposals` is classified into one of: | Class | Actions | Approve bar | |---|---|---| | `patch_landing` | `integrate`, `integrate_patch`, `apply_patch` (production promotion) | Strict — comparable before/after benchmark + accuracy gate + active-path proof + rollback. Critic is the last gate before `optimization_stack` / `framework_source_roots` mutates. | -| `enablement_landing` | `integrate` / `integrate_patch` / `apply_patch` tagged `params.enablement` or `params.framework_agent_authoring` | Structural — same bar as `evidence_producer` (provenance + in-phase + no contradicting KB prior). The patch only makes the model **boot at all** (runnability, not throughput): it is dispatched *before* any usable baseline, so a before/after benchmark + accuracy gate are impossible by construction, and rollback is guaranteed by the enablement integrate executor (`git apply` + `git reset --hard` on REVERT) plus the downstream runnable-decision gate. **Default approve when KB priors are silent.** | +| `enablement_landing` | `integrate` / `integrate_patch` / `apply_patch` tagged `params.enablement` or `params.framework_agent_authoring` | Structural — same bar as `evidence_producer` (provenance + in-phase + no contradicting KB prior). The patch makes the model **run correctly** (runnability, or the accuracy floor for eval-origin — not throughput): boot-origin is dispatched *before* any usable baseline, and eval-origin booted but missed the accuracy floor. A throughput before/after is impossible/irrelevant by construction; rollback is guaranteed by the enablement integrate executor (`git apply` + `git reset --hard` on REVERT) plus the downstream runnable-decision gate (which additionally re-runs the accuracy eval for eval-origin). **Default approve when KB priors are silent.** | | `evidence_producer` | `explore`, `specialist`, `sweep`, `profile`, `roofline`, `kernel_opt`, `deep_kernel_analysis`, `operator_tuning`, `vendor_kernel_config` | Structural — provenance non-empty (specialist or default_grid), action in current phase's allowed set, no contradicting KB prior. **Default approve when KB priors are silent.** | | `framework_op` | `baseline`, `target_analysis`, `recover`, `report`, `session_breakdown`, `framework_pr` | None — approve by default; Critic is not a useful gatekeeper here. (`framework_pr` = FRAMEWORK_PR candidate pre-screen; landing is re-reviewed strictly as `integrate_patch`.) | @@ -204,10 +206,11 @@ Return `approve` only when all blocker risks are cleared: - Robustness findings and known failure patterns do not contradict the decision. -### `enablement_landing` proposals — structural-only (pre-boot patch) +### `enablement_landing` proposals — structural-only Enablement / framework-agent-authoring `integrate_patch` proposals whose -purpose is to make the model **boot at all**. Review them with the +purpose is to make the model **run correctly** — boot-origin (boot at all) +or eval-origin (boot but meet the accuracy floor). Review them with the `evidence_producer` structural bar, **not** the strict `patch_landing` bar. Return `approve` when: @@ -216,14 +219,16 @@ bar. Return `approve` when: - No KB prior actively contradicts the patch, and the packet shows no self-evident defect (e.g. a patch that fails `git apply --check`). -Do **not** require a comparable before/after benchmark or accuracy gate — -there is no bootable baseline yet, so that evidence cannot exist. Do -**not** block solely because the proposal does not restate a rollback -plan: the enablement integrate executor reverts with `git reset --hard` -and the runnable-decision gate REVERTs any patch that does not boot. The -runnable gate — not the Critic — is the real filter for these patches. -Use `needs_review` only when the packet shows an actual defect that the -runnable gate would not catch. +Do **not** require a comparable throughput before/after: boot-origin has no +bootable baseline yet, and eval-origin's KEEP evidence is the downstream +accuracy re-run, not a throughput delta. Do **not** block solely because the +proposal does not restate a rollback plan: the enablement integrate executor +reverts with `git reset --hard`, the runnable-decision gate REVERTs any patch +that does not boot, and for eval-origin it additionally re-runs the accuracy +eval and REVERTs a patch that still misses the floor. The runnable/accuracy +gate — not the Critic — is the real filter for these patches. Use +`needs_review` only when the packet shows an actual defect that the gate would +not catch. ### `evidence_producer` proposals — structural-only diff --git a/src/hyperloom/agents/critic/actions/review_coordinator_inbox.md b/src/hyperloom/agents/critic/actions/review_coordinator_inbox.md index b8e5e5d0f..3abcfbe73 100644 --- a/src/hyperloom/agents/critic/actions/review_coordinator_inbox.md +++ b/src/hyperloom/agents/critic/actions/review_coordinator_inbox.md @@ -67,11 +67,13 @@ Default behavior summary: - `patch_landing` proposal without comparable benchmark + accuracy gate → `needs_review` (or `reject` if the packet itself shows a regression). -- `enablement_landing` proposal (pre-boot enablement / framework-agent +- `enablement_landing` proposal (enablement / framework-agent `integrate_patch`) → **approve** on the structural bar; do NOT block on - a missing before/after benchmark, accuracy gate, or restated rollback - plan (the model cannot boot yet and rollback is automatic). The - downstream runnable-decision gate REVERTs any patch that fails to boot. + a missing throughput before/after or a restated rollback plan (rollback is + automatic). Boot-origin has no baseline yet; eval-origin booted but missed the + accuracy floor. Either way the downstream runnable-decision gate REVERTs any + patch that fails to boot, and for eval-origin additionally re-runs the accuracy + eval and REVERTs a patch that still misses the floor. - `framework_op` proposal → approve by default; only block when structurally malformed. diff --git a/src/hyperloom/agents/framework/AGENTS.md b/src/hyperloom/agents/framework/AGENTS.md index 235ff7153..848c90740 100644 --- a/src/hyperloom/agents/framework/AGENTS.md +++ b/src/hyperloom/agents/framework/AGENTS.md @@ -67,12 +67,13 @@ The package serves two distinct objectives, gated differently: only if throughput improves. Patch authoring for perf is owned by the Hyperloom `specialist → integrate_patch` path; this package does the discovery + static audit + git-apply/bench execution. -- **Enablement** (opt-in) — make a currently **non-runnable** - `(model, backend)` combo *run at all*. This path DOES author bridging +- **Enablement** (opt-in) — make a `(model, backend)` combo that is + **non-runnable**, or that boots but **fails its accuracy eval**, *run + correctly*. This path DOES author bridging patches (via the `enablement_specialist` domain / `SpecialistRunner` worktree authoring) and is gated on **runnability** (server boots + minimal - correctness), not throughput. Pure, GPU-free building blocks live in this - package: + correctness) or, for eval-origin, the accuracy floor; not throughput. Pure, + GPU-free building blocks live in this package: - `hyperloom.agents.framework.enablement` — failure-signature classifier (`classify_failure`) + `EnablementRequest` + the `runnable_decision` gate. - `hyperloom.agents.framework.enablement_ops` — discovery + authoring: diff --git a/src/hyperloom/agents/framework/README.md b/src/hyperloom/agents/framework/README.md index 7e35d8035..089d23236 100644 --- a/src/hyperloom/agents/framework/README.md +++ b/src/hyperloom/agents/framework/README.md @@ -10,25 +10,27 @@ integration uses the FRAMEWORK discovery path: - **Standalone PR exploration** (`fa candidates` / `fa explore`) — ad-hoc tooling outside the `inference_optimizer` runtime path. -- **Enablement** (opt-in) — make a currently **non-runnable** - `(model, backend)` combo *run at all* by authoring a bridging patch. Unlike - the perf path (gated on throughput), enablement is gated on **runnability** - (server boots + minimal correctness). See +- **Enablement** (opt-in) — make a `(model, backend)` combo that is + **non-runnable**, or that boots but **fails its accuracy eval**, *run + correctly* by authoring a bridging patch. Unlike the perf path (gated on + throughput), enablement is gated on **runnability** (server boots + minimal + correctness) or, for an eval-origin trigger, meeting the accuracy floor. See [Enablement path](#enablement-path-non-runnable-model--backend-combos). See [`SKILL.md`](./SKILL.md) for the full architectural overview. ## Enablement path (non-runnable model + backend combos) -When a `(model, backend)` combo will not start, the enablement building -blocks turn the failure into an authored bridging patch, gated on *does it -run* rather than *is it faster*: +When a `(model, backend)` combo will not start, or it starts but fails its +accuracy eval, the enablement building blocks turn the failure into an authored +bridging patch, gated on *does it run correctly* rather than *is it faster*: 1. **Classify** — `hyperloom.agents.framework.enablement.classify_failure(log)` parses a - launch/import/build log into a `FailureSignature` + launch/import/build/eval log into a `FailureSignature` (`missing_model_arch` / `unsupported_dtype` / `hip_kernel_missing` / `import_error` / `shape_mismatch` / `not_implemented` / - `capability_disabled`) with the offending file/symbol and a `bridge_layer`. + `capability_disabled` / `accuracy_below_floor` / `eval_runtime_failure`) + with the offending file/symbol and a `bridge_layer`. 2. **Discover** — `hyperloom.agents.framework.enablement_ops.build_search_plan(...)` picks the repos to scout (the framework repo, plus ROCm/HIP/aiter via `repo_map.bridge_repo_urls` for the failure's bridge layer) and ranks @@ -40,7 +42,9 @@ run* rather than *is it faster*: `SpecialistRunner`, which writes the patch into an isolated worktree. 4. **Verify** — `hyperloom.agents.framework.enablement.runnable_decision(...)` is the KEEP/REVERT gate: the launch probe must exit 0 (no timeout) and any minimal - correctness check must pass; the same failure re-appearing is a reject. + correctness check must pass; the same failure re-appearing is a reject. For an + eval-origin trigger the gate additionally re-runs the accuracy eval and + REVERTs a patch that boots but still misses the accuracy floor. Editing ROCm/HIP source (`/opt/rocm`) is a **default-on** part of the enablement path — the IO-side allowlist always surfaces those roots (alongside diff --git a/src/hyperloom/agents/framework/SKILL.md b/src/hyperloom/agents/framework/SKILL.md index d470f44c0..c10a0db28 100644 --- a/src/hyperloom/agents/framework/SKILL.md +++ b/src/hyperloom/agents/framework/SKILL.md @@ -104,8 +104,9 @@ A discovered candidate reference now drives the enablement targeted build ## Enablement ladder (methodology) -When a candidate is for enablement (making a non-runnable `(model, backend)` -combo boot at all, not perf), the repair follows a tiered ladder — diagnose the +When a candidate is for enablement (making a `(model, backend)` combo that is +non-runnable, or boots but fails its accuracy eval, run correctly — not perf), +the repair follows a tiered ladder — diagnose the missing capability layer once, then climb only as far as needed: Rung 0 diagnose, 1 serve-flag/config wire-up, 2 in-tree source patch, 3 attempt-scoped runtime, 4 source localization, 5 off-loop compiled build. A supported-but-un-wired diff --git a/src/hyperloom/agents/framework/enablement.py b/src/hyperloom/agents/framework/enablement.py index a97a992dc..dd13b46f5 100644 --- a/src/hyperloom/agents/framework/enablement.py +++ b/src/hyperloom/agents/framework/enablement.py @@ -32,11 +32,16 @@ SERVE_FLAG = "serve_flag" # Resource constraints (OOM, TP/GPU count) are NOT code acquisition targets. RESOURCE_CONSTRAINT = "resource_constraint" +# Accuracy-eval triggers (values match _accuracy_gate EVAL_KIND_*): a booting +# baseline whose accuracy is below the floor, and a crashed eval run. +ACCURACY_BELOW_FLOOR = "accuracy_below_floor" +EVAL_RUNTIME_FAILURE = "eval_runtime_failure" UNKNOWN = "unknown" # Ordered most-specific to least-specific. FAILURE_KINDS: tuple[str, ...] = ( MISSING_MODEL_ARCH, + ACCURACY_BELOW_FLOOR, RESOURCE_CONSTRAINT, HIP_KERNEL_MISSING, UNSUPPORTED_DTYPE, @@ -47,6 +52,7 @@ TOKENIZER_ERROR, SERVE_FLAG, IMPORT_ERROR, + EVAL_RUNTIME_FAILURE, UNKNOWN, ) @@ -186,6 +192,17 @@ def _grp(match: re.Match[str]) -> str: confidence=0.95, symbol_from=_grp, ), + _Rule( + # A booting baseline whose accuracy is below the floor. Not a bridge-repo + # target — the fix is correctness of the model's real output. + kind=ACCURACY_BELOW_FLOOR, + bridge_layer="", + patterns=( + re.compile(r"accuracy\s+.*?(?:did not meet|below)\s+.*?floor"), + re.compile(r"baseline[_ ]accuracy[_ ]below[_ ]floor"), + ), + confidence=0.9, + ), _Rule( kind=HIP_KERNEL_MISSING, bridge_layer="rocm_hip", @@ -316,6 +333,19 @@ def _grp(match: re.Match[str]) -> str: confidence=0.7, symbol_from=_grp, ), + _Rule( + # LAST rule: a generic eval-run crash. Kept lowest-priority so a + # co-occurring import/serve-flag/tokenizer/HIP signature in the same log + # wins as the primary root cause and this only surfaces as secondary. + kind=EVAL_RUNTIME_FAILURE, + bridge_layer="", + patterns=( + re.compile(r"run_eval failed with exit code"), + re.compile(r"ERROR: run_eval failed"), + re.compile(r"accuracy eval (?:failed|crashed|did not run)"), + ), + confidence=0.6, + ), ) @@ -682,7 +712,9 @@ def is_targeted_build_candidate( __all__ = [ + "ACCURACY_BELOW_FLOOR", "CAPABILITY_DISABLED", + "EVAL_RUNTIME_FAILURE", "FAILURE_KINDS", "HIP_KERNEL_MISSING", "IMPORT_ERROR", diff --git a/src/hyperloom/agents/framework/enablement_ops.py b/src/hyperloom/agents/framework/enablement_ops.py index 8877372ab..04490311c 100644 --- a/src/hyperloom/agents/framework/enablement_ops.py +++ b/src/hyperloom/agents/framework/enablement_ops.py @@ -67,6 +67,8 @@ "shape_mismatch": ("shape", "reshape", "layout"), "not_implemented": ("implement", "support", "rocm"), "capability_disabled": ("enable", "rocm", "supported"), + "accuracy_below_floor": (), + "eval_runtime_failure": ("eval", "accuracy", "harness"), "unknown": (), } @@ -253,12 +255,14 @@ def _resolve_actual_root_hints(framework: str) -> list[str]: "Only *source edits* must stay under the allowed source roots listed below; " "touching any other path with a code patch is a hard reject. (Environment " "setup via ENVIRONMENT SETUP below is separate and allowed.)", - "Do NOT fabricate throughput/latency/accuracy numbers — the gate here is " - "RUNNABILITY (does the server boot + pass a minimal inference), not perf.", - "Prefer the smallest bridging change that makes the combo run — or, when full " - "runnability is out of reach this round, the smallest change that ADVANCES the " - "boot past the current failure (see PROGRESS DELIVERABLE below); do not " - "refactor unrelated code.", + "Do NOT fabricate throughput/latency/accuracy numbers, and do NOT alter the " + "eval dataset/task/metric/limit or the result parsing to inflate a score — the " + "gate here is RUNNABILITY (server boots + minimal inference) or, for an " + "eval-origin round, the real model output meeting the accuracy floor; not perf.", + "Prefer the smallest bridging change that makes the combo run correctly — or, " + "when that is out of reach this round, the smallest change that ADVANCES past " + "the current failure (a deeper boot gap, or a real accuracy gain toward the " + "floor) (see PROGRESS DELIVERABLE below); do not refactor unrelated code.", "If a discovered PR already implements the fix, adapt/backport it rather than authoring from scratch.", ) @@ -385,6 +389,8 @@ def _resolve_actual_root_hints(framework: str) -> list[str]: "import_error / merged-PR closure -> Rung 4", "hip_kernel_missing / native unsupported_dtype / missing compiled symbol -> Rung 5", "resource_constraint (OOM / GPU count) -> NOT a code gap; cannot be patched", + "accuracy_below_floor / eval_runtime_failure -> re-diagnose against the failing " + "eval contract (accuracy target), then enter at the rung the underlying gap implies", ) @@ -472,7 +478,8 @@ def _render_task_description( """ lines: list[str] = [] lines.append( - f"GOAL: make model `{req.model}` run under the `{req.framework}` backend. It currently fails to start." + f"GOAL: make model `{req.model}` run correctly under the `{req.framework}` backend. " + "It currently fails to start, or it starts but fails its accuracy eval." ) lines.append("") lines.append(f"FAILURE CLASS: {sig.kind} (confidence {sig.confidence:.2f}).") diff --git a/src/hyperloom/agents/framework/tests/test_enablement.py b/src/hyperloom/agents/framework/tests/test_enablement.py index f148f5a96..fc6dc4946 100644 --- a/src/hyperloom/agents/framework/tests/test_enablement.py +++ b/src/hyperloom/agents/framework/tests/test_enablement.py @@ -12,7 +12,9 @@ import pytest from hyperloom.agents.framework.enablement import ( + ACCURACY_BELOW_FLOOR, CAPABILITY_DISABLED, + EVAL_RUNTIME_FAILURE, HIP_KERNEL_MISSING, IMPORT_ERROR, MISSING_MODEL_ARCH, @@ -39,6 +41,26 @@ # --- classify_failure: kind detection -------------------------------------- +def test_accuracy_below_floor_kind() -> None: + sig = classify_failure("baseline accuracy did not meet floor: accuracy=0.12 floor=0.30 task=gsm8k") + assert sig.kind == ACCURACY_BELOW_FLOOR + assert sig.bridge_layer == "" + + +def test_eval_runtime_failure_kind() -> None: + sig = classify_failure("benchmark_stderr.log: ERROR: run_eval failed with exit code 1") + assert sig.kind == EVAL_RUNTIME_FAILURE + + +def test_eval_crash_with_import_error_classifies_as_import_error() -> None: + """The generic eval rule is lowest priority: a real root cause in the same + log (import/serve-flag) must win over eval_runtime_failure.""" + log = "run_eval failed with exit code 1\nModuleNotFoundError: No module named 'lm_eval'" + assert classify_failure(log).kind == IMPORT_ERROR + log2 = "run_eval failed with exit code 1\nvllm: error: unrecognized arguments: --bad" + assert classify_failure(log2).kind == SERVE_FLAG + + def test_missing_model_arch() -> None: """Unsupported architecture message -> missing_model_arch + arch symbol.""" log = "ValueError: Model architecture 'Glm5ForCausalLM' is not supported for now." diff --git a/src/hyperloom/inference_optimizer/breakdown/collectors/sessions.py b/src/hyperloom/inference_optimizer/breakdown/collectors/sessions.py index 909881324..280f25100 100644 --- a/src/hyperloom/inference_optimizer/breakdown/collectors/sessions.py +++ b/src/hyperloom/inference_optimizer/breakdown/collectors/sessions.py @@ -1353,15 +1353,29 @@ def collect_enablement( build_manifest_raw = state.get("enablement_build_manifest") last_build_failure_raw = state.get("enablement_last_build_failure") + origin = str(state.get("enablement_origin") or "") have_active = isinstance(active_runtime_raw, dict) and bool(active_runtime_raw) have_attempts = isinstance(attempt_runtimes_raw, list) and bool(attempt_runtimes_raw) have_actions = isinstance(stack_actions_raw, list) and bool(stack_actions_raw) have_build_manifest = isinstance(build_manifest_raw, list) and bool(build_manifest_raw) have_last_failure = isinstance(last_build_failure_raw, dict) and bool(last_build_failure_raw) - if not (have_active or have_attempts or have_actions or have_build_manifest or have_last_failure): + have_eval = origin == "eval" + if not (have_active or have_attempts or have_actions or have_build_manifest or have_last_failure or have_eval): return {} out: dict[str, Any] = {} + if have_eval: + out["origin"] = origin + out["trigger_kind"] = str(state.get("enablement_baseline_eval_kind") or "") + out["observed_accuracy"] = float(state.get("enablement_observed_accuracy") or 0.0) + out["accuracy_floor"] = float(state.get("enablement_accuracy_floor") or 0.0) + out["observed_task"] = str(state.get("enablement_observed_task") or "") + out["observed_metric"] = str(state.get("enablement_observed_metric") or "") + out["eval_contract_fingerprint"] = str(state.get("enablement_eval_contract_fingerprint") or "") + out["validation_pending"] = bool(state.get("enablement_validation_pending")) + probe_cfg = str(state.get("enablement_probe_config_path") or "") + if probe_cfg: + out["probe_config_path"] = _rel(Path(probe_cfg), session_dir) or probe_cfg if have_actions: summaries: list[dict[str, Any]] = [] for a in stack_actions_raw: diff --git a/src/hyperloom/inference_optimizer/breakdown/schema.py b/src/hyperloom/inference_optimizer/breakdown/schema.py index fd7ba8f85..464340742 100644 --- a/src/hyperloom/inference_optimizer/breakdown/schema.py +++ b/src/hyperloom/inference_optimizer/breakdown/schema.py @@ -2122,6 +2122,17 @@ class EnablementBreakdown(TypedDict, total=False): last_build_failure: ``{failure_class, failure_summary}`` from the most recent failed build attempt (framework-channel decision input). build_attempt_count: Total number of targeted-build rows attempted. + origin: Enablement trigger origin: "" (boot) or "eval". + trigger_kind: Eval trigger kind (eval_runtime_failure / + accuracy_below_floor / accuracy_unavailable) when origin is "eval". + observed_accuracy: Baseline accuracy observed at the eval trigger. + accuracy_floor: Effective accuracy floor for the trigger + KEEP gate. + observed_task: Eval task name observed at the trigger. + observed_metric: Eval metric observed at the trigger. + probe_config_path: Materialized config re-run to reproduce the contract. + eval_contract_fingerprint: Fingerprint of the captured eval contract. + validation_pending: True while an eval-origin KEEP awaits baseline + revalidation. """ stack_actions: list[EnablementStackActionSummary] @@ -2131,6 +2142,15 @@ class EnablementBreakdown(TypedDict, total=False): build_attempts: list[TargetedBuildAttemptSummary] last_build_failure: dict[str, str] build_attempt_count: int + origin: str + trigger_kind: str + observed_accuracy: float + accuracy_floor: float + observed_task: str + observed_metric: str + probe_config_path: str + eval_contract_fingerprint: str + validation_pending: bool # --------------------------------------------------------------------------- diff --git a/src/hyperloom/inference_optimizer/tests/test_enablement_breakdown.py b/src/hyperloom/inference_optimizer/tests/test_enablement_breakdown.py index 8fef1b0be..f16b5c380 100644 --- a/src/hyperloom/inference_optimizer/tests/test_enablement_breakdown.py +++ b/src/hyperloom/inference_optimizer/tests/test_enablement_breakdown.py @@ -18,6 +18,31 @@ def test_collect_enablement_returns_empty_when_nothing(): assert collect_enablement(Path("/tmp"), _state(), []) == {} +def test_collect_enablement_eval_origin_surfaced(): + out = collect_enablement( + Path("/tmp"), + _state( + enablement_origin="eval", + enablement_baseline_eval_kind="accuracy_below_floor", + enablement_observed_accuracy=0.12, + enablement_accuracy_floor=0.3, + enablement_observed_task="gsm8k", + enablement_observed_metric="exact_match", + enablement_eval_contract_fingerprint="fp1", + enablement_validation_pending=True, + enablement_probe_config_path="/tmp/runs/materialized.yaml", + ), + [], + ) + assert out["origin"] == "eval" + assert out["trigger_kind"] == "accuracy_below_floor" + assert out["observed_accuracy"] == 0.12 + assert out["accuracy_floor"] == 0.3 + assert out["eval_contract_fingerprint"] == "fp1" + assert out["validation_pending"] is True + assert out["probe_config_path"] + + def test_collect_enablement_build_manifest_surfaced(): manifest = [ { diff --git a/src/hyperloom/orchestrator/phases/framework.py b/src/hyperloom/orchestrator/phases/framework.py index e8f9452c9..c21fdab12 100644 --- a/src/hyperloom/orchestrator/phases/framework.py +++ b/src/hyperloom/orchestrator/phases/framework.py @@ -1498,20 +1498,23 @@ def _discover_enablement_candidate_refs(self, req: Any, plan: Any) -> tuple[str, return tuple(refs) async def _maybe_enqueue_enablement_specialist(self) -> str: - """Dispatch an enablement_specialist when baseline cannot launch. + """Dispatch an enablement_specialist when a baseline cannot launch or its + accuracy eval fails. - Retries until the combo runs or the run wall-clock deadline passes (no - attempt-count cap). Guards: + Retries until the combo runs correctly or the run wall-clock deadline + passes (no attempt-count cap). Guards: * ``enablement_succeeded`` — terminal: a prior attempt was KEPT. + * ``enablement_validation_pending`` — an eval-origin KEEP is awaiting + genuine-baseline revalidation; authoring is paused until it resolves. * ``enablement_dispatched`` — an authoring attempt is in flight; cleared on REVERT by :meth:`_maybe_rearm_enablement` so the next tick retries with the next bridging candidate (``enablement_attempts`` rotates it). * run deadline passed — stop dispatching new work near the close. - When the captured log classifies to ``UNKNOWN``, no authoring is - dispatched; a one-shot ``needs_human_review`` record is emitted (deduped - per distinct log). No-op on multi-node. + A non-blank log is always dispatched (the specialist repairs from the raw + log even when it classifies to ``UNKNOWN``); only a blank log is a no-op, + recorded once as ``needs_human_review``. No-op on multi-node. Returns: str: The dispatched specialist ``task_id`` (empty when skipped). diff --git a/src/hyperloom/orchestrator/prompts/specialist_prompt_builder.py b/src/hyperloom/orchestrator/prompts/specialist_prompt_builder.py index 11ca2bf58..62bc6fef9 100644 --- a/src/hyperloom/orchestrator/prompts/specialist_prompt_builder.py +++ b/src/hyperloom/orchestrator/prompts/specialist_prompt_builder.py @@ -655,11 +655,13 @@ def _focus_enablement_specialist( """ return [ "You are the **enablement specialist** — an AUTHORING sub-agent for a", - "currently non-runnable (model, backend) combo. The gate is RUNNABILITY", - "(the server boots and passes a minimal inference), not throughput.", + "(model, backend) combo that is non-runnable OR that boots but fails its", + "accuracy eval. The gate is RUNNABILITY (the server boots and passes a", + "minimal inference) or, for an eval-origin dispatch, the real model output", + "meeting the accuracy floor — not throughput.", "", "Your deliverable is the smallest **runnable delta** that advances the", - "boot — which may be a serve flag, an in-tree source patch, an", + "boot (or the accuracy) — which may be a serve flag, an in-tree source patch, an", "attempt-scoped runtime, or a ``needs_targeted_build`` request. Do NOT", "stop at a token registration / two-line alias when the diagnosis says the", "architecture is genuinely new: advancing one boot step counts, and a", @@ -2062,9 +2064,9 @@ def build_specialist_prompts(inp: SpecialistPromptInputs) -> tuple[str, str]: ] if inp.domain.key == "enablement_specialist": # Pre-baseline enablement: the perf context (roofline / recipe / lessons / - # pitfalls / KG knobs / KB subgraph) is noise when the server cannot even - # boot. Carry only the failure, the tiered playbook, and the tools to - # discover + navigate a fix. + # pitfalls / KG knobs / KB subgraph) is noise when the server cannot boot + # or the baseline fails its accuracy eval. Carry only the failure, the + # tiered playbook, and the tools to discover + navigate a fix. user_sections = [ _section_hardware(inp), _section_pd_disaggregation(inp), # § 1a (omitted unless disaggregated) diff --git a/src/hyperloom/orchestrator/specialists/domains.py b/src/hyperloom/orchestrator/specialists/domains.py index f1c740d28..bff5907ce 100644 --- a/src/hyperloom/orchestrator/specialists/domains.py +++ b/src/hyperloom/orchestrator/specialists/domains.py @@ -154,19 +154,22 @@ class SpecialistDomain: ), SpecialistDomain( key="enablement_specialist", - layer="non-runnable (model, backend) enablement / framework + ROCm/HIP bridging", + layer="non-runnable or eval-failing (model, backend) enablement / framework + ROCm/HIP bridging", kb_anchor="framework", available_in="M6", description=( "Authoring specialist for the ENABLEMENT objective: makes a " - "currently non-runnable (model, backend) combo *run at all*. Given a " + "(model, backend) combo that is non-runnable, or that boots but fails " + "its accuracy eval, *run correctly*. Given a " "structured failure signature (missing model arch, unsupported " "dtype, missing HIP kernel, import/build error, shape mismatch, " - "not-implemented) and ranked bridging PRs, it authors a bridging " + "not-implemented, accuracy below floor, eval runtime failure) and " + "ranked bridging PRs, it authors a bridging " "patch into an isolated worktree — editing the framework source, " "and, when the framework layer cannot bridge it, /opt/rocm / HIP / " "aiter source. Gated on RUNNABILITY (server boots + minimal " - "correctness), NOT throughput. Distinct from static_recon " + "correctness) or, for eval-origin, meeting the accuracy floor; NOT " + "throughput. Distinct from static_recon " "(which only finds already-runnable-but-disabled fast paths)." ), ), From 137c4f246deff626e663887b0aa0e4ee3e8d7d8b Mon Sep 17 00:00:00 2001 From: ZhengGong-amd Date: Sun, 26 Jul 2026 02:23:31 +0000 Subject: [PATCH 08/32] test(enablement): cover the eval-crash routing path end-to-end Add a __call__-level test that a single-node eval crash with the flag on is stamped as an eval-failure contract (kind, origin, materialized config, fingerprint) with no RUN_EVAL=false salvage retry, closing the last gap in the eval-origin capture matrix. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../tests/test_baseline_eval_fallback.py | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/src/hyperloom/inference_optimizer/tests/test_baseline_eval_fallback.py b/src/hyperloom/inference_optimizer/tests/test_baseline_eval_fallback.py index 2b15e99cf..799384382 100644 --- a/src/hyperloom/inference_optimizer/tests/test_baseline_eval_fallback.py +++ b/src/hyperloom/inference_optimizer/tests/test_baseline_eval_fallback.py @@ -237,6 +237,52 @@ def fake_run(cmd, *args, **kwargs): assert "eval_failed_fallback_no_accuracy" in result.get("nonfatal_warnings", []) +def test_eval_crash_routes_to_enablement_no_salvage(tmp_path, monkeypatch): + """flag on + single-node: an eval crash is stamped as an eval-failure + contract with no RUN_EVAL=false salvage retry.""" + monkeypatch.setenv("INFERENCE_OPTIMIZER_ENABLEMENT_ON_EVAL_FAIL", "1") + monkeypatch.delenv("INFERENCE_OPTIMIZER_NODES", raising=False) + base = tmp_path / "base.yaml" + _write_yaml(base) + calls: list[dict] = [] + + def fake_run(cmd, *args, **kwargs): + cfg_idx = cmd.index("--benchmark-config") + cfg = yaml.safe_load(Path(cmd[cfg_idx + 1]).read_text()) + run_eval = str(cfg["benchmark"]["envs"].get("RUN_EVAL", "true")).lower() + calls.append({"run_eval": run_eval}) + return subprocess.CompletedProcess(cmd, 1, "", "ERROR: run_eval failed with exit code 1\n") + + executor = BaselineExecutor( + magpie_python="/opt/venv/bin/python", + default_config_path=base, + session_dir=tmp_path, + ) + ctx = _make_ctx( + { + "output_dir": str(tmp_path / "ws"), + "timeout_sec": 10, + "model_path": "/path/models/Qwen-Qwen3-8B", + "gpu_type": "mi300x", + } + ) + ctx.task.kind = "baseline" + with patch( + "hyperloom.orchestrator.actions.executors.baseline.run_with_session_kill", + side_effect=fake_run, + ): + result = _run(executor(ctx)) + + # No RUN_EVAL=false salvage retry: eval stays on. + assert all(c["run_eval"] != "false" for c in calls) + assert result["status"] == "failed" + assert result["baseline_eval_failed"] is True + assert result["baseline_eval_failure_kind"] == "eval_runtime_failure" + assert result["eval_origin"] == "eval" + assert result["materialized_config"] + assert result["baseline_eval_contract_fingerprint"] + + def test_non_eval_failure_does_not_retry(tmp_path): base = tmp_path / "base.yaml" _write_yaml(base) From 865365d59d04f3f92fcead7c2fe242426e1754b5 Mon Sep 17 00:00:00 2001 From: ZhengGong-amd Date: Sun, 26 Jul 2026 02:25:29 +0000 Subject: [PATCH 09/32] test(enablement): unit-cover canonical flag/floor readers, validator and fingerprint Direct coverage for the eval-accuracy foundation: the on/off flag, the floor reader's finite-[0,1] validation and fallback, the accuracy validator across None/bool/str/NaN/Inf/negative/zero/at-floor, the failure-kind classifier, and the eval-contract fingerprint's stability, config-byte sensitivity and missing-file safety. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../tests/test_accuracy_gate_units.py | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/src/hyperloom/inference_optimizer/tests/test_accuracy_gate_units.py b/src/hyperloom/inference_optimizer/tests/test_accuracy_gate_units.py index 93f1bd271..0f0c39181 100644 --- a/src/hyperloom/inference_optimizer/tests/test_accuracy_gate_units.py +++ b/src/hyperloom/inference_optimizer/tests/test_accuracy_gate_units.py @@ -112,3 +112,73 @@ def test_outside_default_threshold(self): def test_custom_threshold(self): assert ag.accuracy_passed(0.80, 0.70, threshold=0.11) is True assert ag.accuracy_passed(0.80, 0.68, threshold=0.10) is False + + +class TestEnablementReaders: + def test_on_eval_fail_default_on(self, monkeypatch): + monkeypatch.delenv("INFERENCE_OPTIMIZER_ENABLEMENT_ON_EVAL_FAIL", raising=False) + assert ag.enablement_on_eval_fail_enabled() is True + + def test_on_eval_fail_disabled(self, monkeypatch): + monkeypatch.setenv("INFERENCE_OPTIMIZER_ENABLEMENT_ON_EVAL_FAIL", "off") + assert ag.enablement_on_eval_fail_enabled() is False + + def test_floor_default(self, monkeypatch): + monkeypatch.delenv("INFERENCE_OPTIMIZER_ENABLEMENT_ACCURACY_FLOOR", raising=False) + assert ag.enablement_accuracy_floor() == 0.0 + + def test_floor_valid(self, monkeypatch): + monkeypatch.setenv("INFERENCE_OPTIMIZER_ENABLEMENT_ACCURACY_FLOOR", "0.7") + assert ag.enablement_accuracy_floor() == pytest.approx(0.7) + + @pytest.mark.parametrize("bad", ["1.5", "-0.1", "nonsense", "nan", "inf"]) + def test_floor_invalid_or_out_of_range_falls_back(self, monkeypatch, bad): + monkeypatch.setenv("INFERENCE_OPTIMIZER_ENABLEMENT_ACCURACY_FLOOR", bad) + assert ag.enablement_accuracy_floor() == 0.0 + + +class TestAccuracyValidator: + @pytest.mark.parametrize( + "score,floor,expected", + [ + (0.5, 0.2, True), + (0.2, 0.2, True), + (0.19, 0.2, False), + (0.0, 0.0, False), + (-0.1, 0.0, False), + (None, 0.0, False), + (True, 0.0, False), + (float("nan"), 0.0, False), + (float("inf"), 0.0, False), + ("0.5", 0.2, False), + ], + ) + def test_meets_floor(self, score, floor, expected): + assert ag.accuracy_meets_floor(score, floor) is expected + + def test_classify(self): + assert ag.classify_accuracy_failure(0.5, 0.2) is None + assert ag.classify_accuracy_failure(None, 0.2) == ag.EVAL_KIND_ACCURACY_UNAVAILABLE + assert ag.classify_accuracy_failure(float("nan"), 0.2) == ag.EVAL_KIND_ACCURACY_UNAVAILABLE + assert ag.classify_accuracy_failure(0.1, 0.2) == ag.EVAL_KIND_ACCURACY_BELOW_FLOOR + assert ag.classify_accuracy_failure(0.0, 0.0) == ag.EVAL_KIND_ACCURACY_BELOW_FLOOR + + +class TestEvalContractFingerprint: + def test_stable_and_sensitive(self): + a = ag.eval_contract_fingerprint(config_path=None, framework="sglang", model="m", task="gsm8k", metric="em") + b = ag.eval_contract_fingerprint(config_path=None, framework="sglang", model="m", task="gsm8k", metric="em") + c = ag.eval_contract_fingerprint(config_path=None, framework="sglang", model="m", task="mmlu", metric="em") + assert a == b and a != c and len(a) == 16 + + def test_hashes_config_bytes(self, tmp_path): + cfg = tmp_path / "c.yaml" + cfg.write_text("a: 1\n") + fp1 = ag.eval_contract_fingerprint(config_path=cfg, framework="f", model="m", task="t", metric="x") + cfg.write_text("a: 2\n") + fp2 = ag.eval_contract_fingerprint(config_path=cfg, framework="f", model="m", task="t", metric="x") + assert fp1 != fp2 + # A missing config path contributes its string form and never raises. + assert ag.eval_contract_fingerprint( + config_path="/no/such/file.yaml", framework="f", model="m", task="t", metric="x" + ) From eb00e16a00388ad78e0db2905e69f06b1edd0ec4 Mon Sep 17 00:00:00 2001 From: ZhengGong-amd Date: Sun, 26 Jul 2026 13:47:33 +0000 Subject: [PATCH 10/32] fix(enablement): canonicalize and protect evaluation contracts Replace the config-bytes+task/metric fingerprint with one derived from stable YAML contract fields (framework, model, script, precision, workload shape, RUN_EVAL, MAGPIE_EVAL_TASKS/LIMIT). Result-level task/metric are excluded so an eval crash and a successful eval on the same workload produce identical fingerprints. Missing or unreadable configs return an empty string (invalid) that the gate treats as fail-closed. Persist MAGPIE_EVAL_TASKS and MAGPIE_EVAL_LIMIT into the materialized YAML so bypass_runner reads them from the YAML-layer authority rather than os.environ, making them trackable by the fingerprint. For enablement runs in integrate_patch, force RUN_EVAL=true onto the variant envs after all overlays are applied so no candidate can suppress it. Preserve list/dict structure in runtime_override (stop coercing nested values to str before apply_runtime_override sees them). Compute the candidate fingerprint unconditionally at bench time (not only on success) so an eval-crash round still produces a comparable digest. Update tests to verify server-arg changes do not drift the fingerprint, workload/eval-control changes do, crash and success produce identical digests, and missing configs return the empty invalid sentinel. Co-authored-by: Cursor --- .../tests/test_accuracy_gate_units.py | 101 +++++++++++++++--- .../tests/test_baseline_eval_fallback.py | 27 ++++- .../actions/executors/_accuracy_gate.py | 97 ++++++++++++++--- .../actions/executors/_workload_envs.py | 8 ++ .../actions/executors/baseline.py | 5 +- .../actions/executors/bypass_runner.py | 4 +- .../actions/executors/integrate_patch.py | 50 +++++---- 7 files changed, 230 insertions(+), 62 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_accuracy_gate_units.py b/src/hyperloom/inference_optimizer/tests/test_accuracy_gate_units.py index 0f0c39181..373636920 100644 --- a/src/hyperloom/inference_optimizer/tests/test_accuracy_gate_units.py +++ b/src/hyperloom/inference_optimizer/tests/test_accuracy_gate_units.py @@ -164,21 +164,92 @@ def test_classify(self): assert ag.classify_accuracy_failure(0.0, 0.0) == ag.EVAL_KIND_ACCURACY_BELOW_FLOOR +def _write_minimal_bench_yaml(path, *, framework="sglang", model="/m", conc=64, isl=1024, osl=1024, run_eval="true"): + """Write a minimal Magpie YAML for fingerprint tests.""" + import yaml as _yaml + + cfg = { + "benchmark": { + "framework": framework, + "model": model, + "benchmark_script": f"{framework}_mi300x.sh", + "precision": "fp8", + "envs": { + "CONC": conc, + "ISL": isl, + "OSL": osl, + "TP": 8, + "RUN_EVAL": run_eval, + }, + } + } + path.write_text(_yaml.safe_dump(cfg), encoding="utf-8") + + class TestEvalContractFingerprint: - def test_stable_and_sensitive(self): - a = ag.eval_contract_fingerprint(config_path=None, framework="sglang", model="m", task="gsm8k", metric="em") - b = ag.eval_contract_fingerprint(config_path=None, framework="sglang", model="m", task="gsm8k", metric="em") - c = ag.eval_contract_fingerprint(config_path=None, framework="sglang", model="m", task="mmlu", metric="em") - assert a == b and a != c and len(a) == 16 + def test_stable_across_calls(self, tmp_path): + """Same config produces the same fingerprint on repeated calls.""" + cfg = tmp_path / "c.yaml" + _write_minimal_bench_yaml(cfg) + a = ag.eval_contract_fingerprint(config_path=cfg) + b = ag.eval_contract_fingerprint(config_path=cfg) + assert a == b and len(a) == 16 - def test_hashes_config_bytes(self, tmp_path): + def test_task_metric_do_not_affect_fingerprint(self, tmp_path): + """Result-level task/metric do not change the fingerprint.""" cfg = tmp_path / "c.yaml" - cfg.write_text("a: 1\n") - fp1 = ag.eval_contract_fingerprint(config_path=cfg, framework="f", model="m", task="t", metric="x") - cfg.write_text("a: 2\n") - fp2 = ag.eval_contract_fingerprint(config_path=cfg, framework="f", model="m", task="t", metric="x") - assert fp1 != fp2 - # A missing config path contributes its string form and never raises. - assert ag.eval_contract_fingerprint( - config_path="/no/such/file.yaml", framework="f", model="m", task="t", metric="x" - ) + _write_minimal_bench_yaml(cfg) + fp_no_task = ag.eval_contract_fingerprint(config_path=cfg) + fp_with_task = ag.eval_contract_fingerprint(config_path=cfg, task="gsm8k", metric="exact_match") + fp_other_task = ag.eval_contract_fingerprint(config_path=cfg, task="mmlu", metric="acc") + assert fp_no_task == fp_with_task == fp_other_task + + def test_crash_and_success_produce_same_fingerprint(self, tmp_path): + """eval crash (no task/metric) and successful eval produce identical fingerprint.""" + cfg = tmp_path / "c.yaml" + _write_minimal_bench_yaml(cfg) + fp_crash = ag.eval_contract_fingerprint(config_path=cfg, task=None, metric=None) + fp_success = ag.eval_contract_fingerprint(config_path=cfg, task="gsm8k", metric="exact_match,strict-match") + assert fp_crash == fp_success + + def test_workload_change_changes_fingerprint(self, tmp_path): + """A change to workload shape (ISL) changes the fingerprint.""" + import yaml as _yaml + + cfg1 = tmp_path / "c1.yaml" + cfg2 = tmp_path / "c2.yaml" + _write_minimal_bench_yaml(cfg1, isl=1024) + _write_minimal_bench_yaml(cfg2, isl=2048) + assert ag.eval_contract_fingerprint(config_path=cfg1) != ag.eval_contract_fingerprint(config_path=cfg2) + + def test_eval_control_change_changes_fingerprint(self, tmp_path): + """Changing RUN_EVAL changes the fingerprint.""" + cfg_on = tmp_path / "on.yaml" + cfg_off = tmp_path / "off.yaml" + _write_minimal_bench_yaml(cfg_on, run_eval="true") + _write_minimal_bench_yaml(cfg_off, run_eval="false") + assert ag.eval_contract_fingerprint(config_path=cfg_on) != ag.eval_contract_fingerprint(config_path=cfg_off) + + def test_server_args_do_not_affect_fingerprint(self, tmp_path): + """Server arg changes (allowed tuning) do not change the fingerprint.""" + import yaml as _yaml + + cfg = tmp_path / "c.yaml" + _write_minimal_bench_yaml(cfg) + fp_base = ag.eval_contract_fingerprint(config_path=cfg) + # Add server args to YAML (different from contract fields). + data = _yaml.safe_load(cfg.read_text()) + data["benchmark"]["envs"]["EXTRA_SGLANG_ARGS"] = "--some-tuning-flag" + cfg.write_text(_yaml.safe_dump(data)) + fp_with_args = ag.eval_contract_fingerprint(config_path=cfg) + assert fp_base == fp_with_args + + def test_missing_config_returns_empty_string(self): + """Missing config returns empty string (invalid contract sentinel).""" + fp = ag.eval_contract_fingerprint(config_path="/no/such/file.yaml") + assert fp == "" + + def test_none_config_returns_empty_string(self): + """None config returns empty string.""" + fp = ag.eval_contract_fingerprint(config_path=None) + assert fp == "" diff --git a/src/hyperloom/inference_optimizer/tests/test_baseline_eval_fallback.py b/src/hyperloom/inference_optimizer/tests/test_baseline_eval_fallback.py index 799384382..cddc0d287 100644 --- a/src/hyperloom/inference_optimizer/tests/test_baseline_eval_fallback.py +++ b/src/hyperloom/inference_optimizer/tests/test_baseline_eval_fallback.py @@ -641,21 +641,40 @@ def fake_run(cmd, *args, **kwargs): ) -def _route(monkeypatch, framework, result, *, floor=None, nodes=None): +def _write_minimal_route_yaml(tmp_path: Path, framework: str = "sglang") -> Path: + """Write a minimal materialized YAML for _route tests.""" + p = tmp_path / "route_config.yaml" + cfg = { + "benchmark": { + "framework": framework, + "model": "/path/models/test", + "benchmark_script": f"{framework}_mi300x.sh", + "precision": "bf16", + "envs": {"CONC": 64, "ISL": 1024, "OSL": 1024, "TP": 8, "RUN_EVAL": "true"}, + } + } + p.write_text(yaml.safe_dump(cfg), encoding="utf-8") + return p + + +def _route(monkeypatch, framework, result, *, floor=None, nodes=None, tmp_path=None): monkeypatch.setenv("INFERENCE_OPTIMIZER_ENABLEMENT_ON_EVAL_FAIL", "1") if floor is not None: monkeypatch.setenv("INFERENCE_OPTIMIZER_ENABLEMENT_ACCURACY_FLOOR", str(floor)) if nodes is not None: monkeypatch.setenv("INFERENCE_OPTIMIZER_NODES", str(nodes)) + if tmp_path is not None and "materialized_config" not in result: + result["materialized_config"] = str(_write_minimal_route_yaml(tmp_path, framework)) executor = BaselineExecutor() rec = _StopRecorder() executor._maybe_stop_on_missing_baseline_accuracy(_stop_ctx(framework, rec), result) return rec.stop_reason -def test_eval_enablement_missing_accuracy_routes_not_stop(monkeypatch): - result = {"status": "succeeded", "run_eval_disabled": False, "materialized_config": "/tmp/c.yaml"} - reason = _route(monkeypatch, "sglang", result) +def test_eval_enablement_missing_accuracy_routes_not_stop(monkeypatch, tmp_path): + cfg_path = str(_write_minimal_route_yaml(tmp_path)) + result = {"status": "succeeded", "run_eval_disabled": False, "materialized_config": cfg_path} + reason = _route(monkeypatch, "sglang", result, tmp_path=tmp_path) assert reason == "" assert result[BASELINE_EVAL_FAILED_KEY] is True assert result[BASELINE_EVAL_FAILURE_KIND_KEY] == EVAL_KIND_ACCURACY_UNAVAILABLE diff --git a/src/hyperloom/orchestrator/actions/executors/_accuracy_gate.py b/src/hyperloom/orchestrator/actions/executors/_accuracy_gate.py index d5f914acd..ea55eccef 100644 --- a/src/hyperloom/orchestrator/actions/executors/_accuracy_gate.py +++ b/src/hyperloom/orchestrator/actions/executors/_accuracy_gate.py @@ -154,30 +154,92 @@ def classify_accuracy_failure(score: Any, floor: float) -> str | None: return None +def _extract_eval_contract_fields(config_path: str | Path | None) -> dict[str, str]: + """Extract stable eval-contract fields from a materialized Magpie YAML. + + Reads the fields that define what workload is evaluated and how eval is + controlled. Server args, runtime paths, lifecycle envs and any field + that a server-arg tuning candidate is allowed to change are excluded so + the fingerprint stays stable across valid enablement patches. + + Returns an empty dict when the config is absent or unreadable. + """ + if not config_path: + return {} + try: + import yaml as _yaml + + p = Path(config_path) + if not p.is_file(): + return {} + data = _yaml.safe_load(p.read_text(encoding="utf-8")) or {} + except Exception: # noqa: BLE001 — best-effort + return {} + + bench = data.get("benchmark") or {} + envs: dict = bench.get("envs") or {} + + # Eval-contract keys in benchmark.envs; all others are excluded. + _EVAL_CONTRACT_ENV_KEYS = ( + "RUN_EVAL", + "MAGPIE_EVAL_TASKS", + "MAGPIE_EVAL_LIMIT", + ) + # Workload-shape keys that define what is being measured. + _WORKLOAD_SHAPE_ENV_KEYS = ( + "CONC", + "ISL", + "OSL", + "MAX_MODEL_LEN", + "TP", + "RANDOM_RANGE_RATIO", + ) + contract: dict[str, str] = { + "framework": str(bench.get("framework") or ""), + "model": str(bench.get("model") or ""), + "benchmark_script": str(bench.get("benchmark_script") or ""), + "precision": str(bench.get("precision") or ""), + } + for k in _EVAL_CONTRACT_ENV_KEYS + _WORKLOAD_SHAPE_ENV_KEYS: + v = envs.get(k) + if v is not None: + contract[k] = str(v) + return contract + + def eval_contract_fingerprint( *, config_path: str | Path | None, - framework: str | None, - model: str | None, - task: str | None, - metric: str | None, + framework: str | None = None, + model: str | None = None, + task: str | None = None, + metric: str | None = None, ) -> str: """Short stable digest of the eval contract (workload + eval definition). - Hashes the materialized config bytes when available plus the framework, - model, task and metric so a later enablement re-run can detect contract - drift. Never raises; unreadable inputs contribute their string form. + Derives the digest from stable eval-contract inputs extracted from the + materialized YAML (framework, model, script, precision, workload shape, + eval controls). Result-level outputs such as task/metric names are NOT + included so an eval crash (where those are absent) produces the same + fingerprint as a successful eval on the identical contract. + + ``task`` and ``metric`` parameters are accepted for call-site compatibility + but are not included in the hash. + + Returns an empty string when the config cannot be read, signalling to + callers that the contract is invalid and drift checking should fail closed. """ - parts = [str(framework or ""), str(model or ""), str(task or ""), str(metric or "")] - cfg = "" - if config_path: - p = Path(config_path) - try: - cfg = p.read_bytes().hex() if p.is_file() else str(config_path) - except OSError: - cfg = str(config_path) - parts.append(cfg) - return hashlib.sha256("\x1e".join(parts).encode("utf-8", "replace")).hexdigest()[:16] + contract = _extract_eval_contract_fields(config_path) + if not contract: + # Unreadable or missing config — return invalid sentinel. + return "" + # Supplement with caller-supplied framework/model when the YAML lacks them. + if framework and not contract.get("framework"): + contract["framework"] = str(framework) + if model and not contract.get("model"): + contract["model"] = str(model) + payload = json.dumps(contract, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + return hashlib.sha256(payload.encode("utf-8", "replace")).hexdigest()[:16] def accuracy_keep_block( @@ -500,6 +562,7 @@ def accuracy_passed( "EVAL_KIND_ACCURACY_BELOW_FLOOR", "EVAL_KIND_ACCURACY_UNAVAILABLE", "EVAL_KIND_RUNTIME_FAILURE", + "_extract_eval_contract_fields", "accuracy_keep_block", "accuracy_meets_floor", "accuracy_passed", diff --git a/src/hyperloom/orchestrator/actions/executors/_workload_envs.py b/src/hyperloom/orchestrator/actions/executors/_workload_envs.py index 33d95d5fb..7a7c5c4b9 100644 --- a/src/hyperloom/orchestrator/actions/executors/_workload_envs.py +++ b/src/hyperloom/orchestrator/actions/executors/_workload_envs.py @@ -960,6 +960,14 @@ def materialize_config_with_envs( if "RUN_EVAL" not in envs: env_run_eval = os.environ.get("RUN_EVAL") envs["RUN_EVAL"] = env_run_eval if env_run_eval is not None else "true" + # Persist eval task/limit into the YAML so they become part of the + # fingerprint-able contract and bypass_runner reads them from a stable source. + _eval_tasks_env = os.environ.get("MAGPIE_EVAL_TASKS", "").strip() + if _eval_tasks_env and "MAGPIE_EVAL_TASKS" not in envs: + envs["MAGPIE_EVAL_TASKS"] = _eval_tasks_env + _eval_limit_env = os.environ.get("MAGPIE_EVAL_LIMIT", "").strip() + if _eval_limit_env and "MAGPIE_EVAL_LIMIT" not in envs: + envs["MAGPIE_EVAL_LIMIT"] = _eval_limit_env if str(envs.get("RUN_EVAL", "")).strip().lower() in _RUN_EVAL_FALSE_VALUES: global _RUN_EVAL_DISABLED_WARN_EMITTED if not _RUN_EVAL_DISABLED_WARN_EMITTED: diff --git a/src/hyperloom/orchestrator/actions/executors/baseline.py b/src/hyperloom/orchestrator/actions/executors/baseline.py index 4ee190e00..97d206cf2 100644 --- a/src/hyperloom/orchestrator/actions/executors/baseline.py +++ b/src/hyperloom/orchestrator/actions/executors/baseline.py @@ -1272,12 +1272,13 @@ def _stamp_eval_failure_contract( result[BASELINE_EVAL_OBSERVED_ACCURACY_KEY] = observed_accuracy result[BASELINE_EVAL_ACCURACY_FLOOR_KEY] = floor result[BASELINE_EVAL_EVIDENCE_KEY] = (evidence or "")[:4000] + # Fingerprint derives from the materialized YAML contract fields only — + # task/metric are result outputs and may be absent on eval crash, so they + # must not participate in the stable identity. result[BASELINE_EVAL_CONTRACT_FINGERPRINT_KEY] = eval_contract_fingerprint( config_path=result.get("materialized_config"), framework=framework, model=model, - task=result.get("accuracy_task"), - metric=result.get("accuracy_metric"), ) result["eval_origin"] = "eval" return result diff --git a/src/hyperloom/orchestrator/actions/executors/bypass_runner.py b/src/hyperloom/orchestrator/actions/executors/bypass_runner.py index 4666dac8b..2e1549cda 100644 --- a/src/hyperloom/orchestrator/actions/executors/bypass_runner.py +++ b/src/hyperloom/orchestrator/actions/executors/bypass_runner.py @@ -775,8 +775,8 @@ def _run_client_and_eval( base_url=base_url, conc=conc, out_dir=str(workspace / "lm_eval"), - tasks=os.environ.get("MAGPIE_EVAL_TASKS", "gsm8k").strip() or "gsm8k", - limit=(os.environ.get("MAGPIE_EVAL_LIMIT", "").strip() or None), + tasks=str(bench_envs.get("MAGPIE_EVAL_TASKS") or os.environ.get("MAGPIE_EVAL_TASKS", "")).strip() or "gsm8k", + limit=(str(bench_envs.get("MAGPIE_EVAL_LIMIT") or os.environ.get("MAGPIE_EVAL_LIMIT", "")).strip() or None), ) eval_rc = _run_subprocess(eval_cmd, timeout_s, workspace, "eval") # Magpie's ``run_eval ... || exit $?`` aborts the benchmark when the diff --git a/src/hyperloom/orchestrator/actions/executors/integrate_patch.py b/src/hyperloom/orchestrator/actions/executors/integrate_patch.py index ed2867aa2..aa73e2ebd 100644 --- a/src/hyperloom/orchestrator/actions/executors/integrate_patch.py +++ b/src/hyperloom/orchestrator/actions/executors/integrate_patch.py @@ -2981,6 +2981,9 @@ async def _bench_patch( _base_envs = dict(params.get("base_extra_envs") or {}) _variant_envs = dict(_base_envs) _variant_envs.update(extra_envs_applied) + # For enablement runs, ensure RUN_EVAL=true survives any variant overlay. + if bool(params.get("enablement")): + _variant_envs["RUN_EVAL"] = "true" variant = GridVariant( name=f"integrate-patch-{specialist_task_id[:8]}", extra_server_args=extra_server_args_applied, @@ -2992,7 +2995,8 @@ async def _bench_patch( ) _rt = params.get("runtime_override") if isinstance(_rt, dict) and _rt: - variant.runtime_override = {str(k): str(v) for k, v in _rt.items()} + # Preserve list/dict values; apply_runtime_override expects them. + variant.runtime_override = dict(_rt) # Ray-managed GPU execution: hold a serving lease # (num_gpus=TP + serving_slot) for the whole run_grid so @@ -3061,26 +3065,28 @@ async def _bench_patch( enablement_accuracy_task = "" enablement_accuracy_metric = "" candidate_fingerprint = "" - if bool(params.get("enablement")) and bench.get("status") == "succeeded": - try: - eval_results = parse_eval_results( - eval_search_root, - framework=params.get("framework") or os.environ.get("FRAMEWORK") or None, - ) - acc = eval_results.get("accuracy") - if isinstance(acc, (int, float)): - enablement_accuracy = float(acc) - enablement_accuracy_task = str(eval_results.get("task") or "") - enablement_accuracy_metric = str(eval_results.get("metric") or "") - candidate_fingerprint = eval_contract_fingerprint( - config_path=config_path, - framework=params.get("framework") or os.environ.get("FRAMEWORK") or None, - model=resolved_model, - task=enablement_accuracy_task, - metric=enablement_accuracy_metric, - ) - except Exception: # noqa: BLE001 — eval may not produce a result - log.debug("integrate_patch: enablement eval parse failed", exc_info=True) + if bool(params.get("enablement")): + # Compute the candidate fingerprint from the materialized config + # regardless of bench outcome — eval crash must still produce a + # comparable fingerprint. + candidate_fingerprint = eval_contract_fingerprint( + config_path=config_path, + framework=params.get("framework") or os.environ.get("FRAMEWORK") or None, + model=resolved_model, + ) + if bench.get("status") == "succeeded": + try: + eval_results = parse_eval_results( + eval_search_root, + framework=params.get("framework") or os.environ.get("FRAMEWORK") or None, + ) + acc = eval_results.get("accuracy") + if isinstance(acc, (int, float)): + enablement_accuracy = float(acc) + enablement_accuracy_task = str(eval_results.get("task") or "") + enablement_accuracy_metric = str(eval_results.get("metric") or "") + except Exception: # noqa: BLE001 — eval may not produce a result + log.debug("integrate_patch: enablement eval parse failed", exc_info=True) return bench, { "accuracy_pass": accuracy_pass, @@ -3204,7 +3210,7 @@ async def _confirm_stack_rebench( ) _rt_rb = params.get("runtime_override") if isinstance(_rt_rb, dict) and _rt_rb: - variant.runtime_override = {str(k): str(v) for k, v in _rt_rb.items()} + variant.runtime_override = dict(_rt_rb) rebench = await measure_stack_rebench( config_path=config_path, base_extra_args=base_extra_args, From 0439f0cd8e8b8ea638cdede36338e7ffb5cfca04 Mon Sep 17 00:00:00 2001 From: ZhengGong-amd Date: Sun, 26 Jul 2026 13:55:29 +0000 Subject: [PATCH 11/32] fix(enablement): replay accepted launch context for validation Add enablement_accepted_config_path to SharedState to record the actual materialized config from the KEEP'd candidate bench. This is the config with server fixes already baked in, distinct from the original trigger probe config. integrate_patch now carries this path in the KEEP result so _maybe_rearm_ enablement can persist it. _maybe_enqueue_enablement_baseline_revalidation prefers the accepted config over the probe config when launching the genuine baseline, and also picks up the KEEP'd FrameworkRuntime to set runtime_override on the revalidation task. baseline._run_once applies runtime_override from params into the materialized YAML so the revalidation baseline boots under the same framework runtime (PATH/PYTHONPATH/framework_bin) as the KEEP'd candidate. Also carry the materialized config path in _bench_patch's bench result so the gate can surface it in the KEEP result. Tests verify that accepted_config_path takes precedence over probe_config, that active runtime override is propagated, and that both fields survive save/load roundtrip. Co-authored-by: Cursor --- ...test_enablement_coordinator_wiring_unit.py | 39 +++++++++++++++++++ .../tests/test_shared_state_evolution.py | 13 +++++++ .../actions/executors/baseline.py | 17 ++++++++ .../actions/executors/integrate_patch.py | 5 +++ .../orchestrator/phases/framework.py | 30 +++++++++++--- .../orchestrator/state/shared_state.py | 6 +++ 6 files changed, 105 insertions(+), 5 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_enablement_coordinator_wiring_unit.py b/src/hyperloom/inference_optimizer/tests/test_enablement_coordinator_wiring_unit.py index c7972146a..2c975607e 100644 --- a/src/hyperloom/inference_optimizer/tests/test_enablement_coordinator_wiring_unit.py +++ b/src/hyperloom/inference_optimizer/tests/test_enablement_coordinator_wiring_unit.py @@ -230,8 +230,10 @@ def _enqueue_self(**state_kw): enablement_origin=state_kw.get("enablement_origin", ""), enablement_validation_pending=state_kw.get("enablement_validation_pending", False), enablement_probe_config_path=state_kw.get("enablement_probe_config_path", ""), + enablement_accepted_config_path=state_kw.get("enablement_accepted_config_path", ""), enablement_accuracy_floor=state_kw.get("enablement_accuracy_floor", 0.0), enablement_eval_contract_fingerprint=state_kw.get("enablement_eval_contract_fingerprint", ""), + enablement_active_runtime=state_kw.get("enablement_active_runtime", {}), tick=state_kw.get("tick", 0), stop_reason=state_kw.get("stop_reason", ""), save=lambda *a, **k: None, @@ -453,6 +455,43 @@ async def test_revalidation_enqueues_genuine_baseline(): assert created["params"]["enablement_origin"] == "eval" +@pytest.mark.asyncio +async def test_revalidation_prefers_accepted_config_over_probe(): + """accepted_config_path (from KEEP'd bench) takes precedence over probe config.""" + fake = _enqueue_self( + enablement_validation_pending=True, + enablement_origin="eval", + enablement_probe_config_path="/runs/baseline/probe.yaml", + enablement_accepted_config_path="/runs/specialist/accepted.yaml", + enablement_accuracy_floor=0.3, + ) + tid = await fake._maybe_enqueue_enablement_baseline_revalidation() + assert tid + created = fake.tasks.created[-1] + assert created["params"]["config_path"] == "/runs/specialist/accepted.yaml" + + +@pytest.mark.asyncio +async def test_revalidation_carries_active_runtime(): + """When an active runtime is recorded, its override is included in params.""" + from hyperloom.orchestrator.framework.stack_actions import FrameworkRuntime + + rt = FrameworkRuntime(bin_path="/attempt/bin", python_path="/attempt/bin/python", venv_root="/attempt/venv") + fake = _enqueue_self( + enablement_validation_pending=True, + enablement_origin="eval", + enablement_probe_config_path="/runs/baseline/probe.yaml", + enablement_accepted_config_path="/runs/specialist/accepted.yaml", + enablement_active_runtime=rt.to_state(), + ) + tid = await fake._maybe_enqueue_enablement_baseline_revalidation() + assert tid + created = fake.tasks.created[-1] + rt_override = created["params"].get("runtime_override") + assert isinstance(rt_override, dict) and rt_override + assert rt_override.get("framework_bin") == "/attempt/bin" + + @pytest.mark.asyncio async def test_revalidation_skips_when_tput_positive(): fake = _enqueue_self(enablement_validation_pending=True, enablement_origin="eval", baseline_tput=123.0) diff --git a/src/hyperloom/inference_optimizer/tests/test_shared_state_evolution.py b/src/hyperloom/inference_optimizer/tests/test_shared_state_evolution.py index 51ecc20dd..2b80fa7f7 100644 --- a/src/hyperloom/inference_optimizer/tests/test_shared_state_evolution.py +++ b/src/hyperloom/inference_optimizer/tests/test_shared_state_evolution.py @@ -445,6 +445,19 @@ def test_search_ledgers_in_core_state_fields(): ) +def test_enablement_accepted_config_path_roundtrips(tmp_path): + """enablement_accepted_config_path is persisted and reloaded correctly.""" + sd = tmp_path / "session" + sd.mkdir() + s = SharedState() + s.enablement_accepted_config_path = "/runs/specialist/t-spec-1/integrate_patch.with_envs.yaml" + s.enablement_active_runtime = {"bin_path": "/attempt/bin", "venv_root": "/attempt/venv"} + s.save(sd) + loaded = SharedState.load_or_init(sd) + assert loaded.enablement_accepted_config_path == "/runs/specialist/t-spec-1/integrate_patch.with_envs.yaml" + assert loaded.enablement_active_runtime == {"bin_path": "/attempt/bin", "venv_root": "/attempt/venv"} + + @pytest.mark.parametrize("field_name", ["explore_search"]) def test_policy_blocks_llm_search_ledger_write(field_name): """LLM ``update_state`` of a search ledger surfaces a ``state_field`` denial.""" diff --git a/src/hyperloom/orchestrator/actions/executors/baseline.py b/src/hyperloom/orchestrator/actions/executors/baseline.py index 97d206cf2..a77224ab9 100644 --- a/src/hyperloom/orchestrator/actions/executors/baseline.py +++ b/src/hyperloom/orchestrator/actions/executors/baseline.py @@ -1595,6 +1595,23 @@ async def _run_once( } # Stash for the result so Coordinator can reuse it downstream. materialized_config_path = config_path + # Apply runtime_override from params into the materialized YAML so the + # revalidation baseline boots under the same framework runtime as the + # KEEP'd candidate (PATH/PYTHONPATH/framework_bin etc.). + _rt_from_params = params.get("runtime_override") + if isinstance(_rt_from_params, dict) and _rt_from_params: + try: + import yaml as _yaml + + from ._grid_runner import apply_runtime_override + + _cfg_data = _yaml.safe_load(config_path.read_text(encoding="utf-8")) or {} + _cfg_bench = _cfg_data.setdefault("benchmark", {}) + _cfg_envs = _cfg_bench.setdefault("envs", {}) + apply_runtime_override(_cfg_envs, _rt_from_params) + config_path.write_text(_yaml.safe_dump(_cfg_data), encoding="utf-8") + except Exception: # noqa: BLE001 — runtime overlay is best-effort + log.debug("baseline_executor: runtime_override application failed", exc_info=True) # Whether THIS run actually executes lm-eval, read back from the # materialized config the subprocess consumes -- the single source of # truth. ``materialize_config_with_envs`` folds RUN_EVAL from the base diff --git a/src/hyperloom/orchestrator/actions/executors/integrate_patch.py b/src/hyperloom/orchestrator/actions/executors/integrate_patch.py index aa73e2ebd..f65bbbfd1 100644 --- a/src/hyperloom/orchestrator/actions/executors/integrate_patch.py +++ b/src/hyperloom/orchestrator/actions/executors/integrate_patch.py @@ -2354,6 +2354,9 @@ def _gc_on_revert() -> None: "setup_commands_applied": list(setup_result.get("applied") or []), "bench_result": bench_result, "workspace": str(output_root), + # The actual materialized config from the KEEP'd bench, used for + # revalidation baseline so the same effective config is re-run. + "enablement_accepted_config_path": str(bench_result.get("materialized_config") or ""), **eval_provenance, } # Record the KEEP'd attempt runtime so it survives rearm and every later @@ -3043,6 +3046,8 @@ async def _bench_patch( "workspace": str(getattr(r, "workspace", "") or ""), "error": getattr(r, "error", "") or "", "nonfatal_warnings": list(getattr(r, "nonfatal_warnings", []) or []), + # Materialized config used for this bench; needed by revalidation. + "materialized_config": str(config_path), } accuracy_pass: bool | None = None diff --git a/src/hyperloom/orchestrator/phases/framework.py b/src/hyperloom/orchestrator/phases/framework.py index c21fdab12..74ec60131 100644 --- a/src/hyperloom/orchestrator/phases/framework.py +++ b/src/hyperloom/orchestrator/phases/framework.py @@ -1829,6 +1829,11 @@ def _stack_kept_runtime() -> None: _reset_baseline_failure_backstop() _stack_setup_commands() _stack_kept_runtime() + # Persist the actual materialized config used for the KEEP'd bench so + # the revalidation baseline re-runs the identical effective config. + accepted_cfg = str(res.get("enablement_accepted_config_path") or "").strip() + if accepted_cfg: + state.enablement_accepted_config_path = accepted_cfg if str(getattr(state, "enablement_origin", "") or "") == "eval": # eval-origin: the patch boots and re-passed accuracy in the gate, # but tput/accuracy only become official once a GENUINE baseline @@ -4552,10 +4557,10 @@ async def _enqueue_build_launch_probe(self, build_task_id: str, br: Any) -> None async def _maybe_enqueue_enablement_baseline_revalidation(self) -> str: """Enqueue one genuine baseline to revalidate a KEEP'd eval-origin patch. - Reproduces the original workload/eval contract (probe config, RUN_EVAL on) - so the accuracy is measured for real; the KEEP is finalized in - ``_promote_baseline`` only when that baseline promotes with accuracy at or - above the floor. Idempotent and one-at-a-time. + Uses the accepted config from the KEEP'd candidate bench (preferred) or + falls back to the original probe config. The frozen eval controls from + the carrier params ensure RUN_EVAL and eval task/limit match the trigger + contract. Idempotent and one-at-a-time. """ state = self.shared_state if not bool(getattr(state, "enablement_validation_pending", False)): @@ -4574,9 +4579,24 @@ async def _maybe_enqueue_enablement_baseline_revalidation(self) -> str: "disable_run_eval": False, **_enablement_carrier_params(state), } - cfg = str(getattr(state, "enablement_probe_config_path", "") or "") + # Prefer the accepted (post-fix) config so revalidation uses the same + # effective config the KEEP'd candidate ran; fall back to the trigger + # probe config only when no accepted config was recorded. + accepted_cfg = str(getattr(state, "enablement_accepted_config_path", "") or "").strip() + probe_cfg = str(getattr(state, "enablement_probe_config_path", "") or "").strip() + cfg = accepted_cfg or probe_cfg if cfg: params["config_path"] = cfg + # Carry the active runtime override so the revalidation baseline runs + # under the same framework runtime as the KEEP'd candidate. + active_rt = getattr(state, "enablement_active_runtime", None) or {} + if isinstance(active_rt, dict) and active_rt: + from ..framework.stack_actions import FrameworkRuntime + + rt_obj = FrameworkRuntime.from_state(active_rt) + rt_override = rt_obj.to_runtime_override() + if rt_override: + params["runtime_override"] = rt_override attempt = int(getattr(state, "enablement_attempts", 0) or 0) task, _existing = await self.tasks.create_or_return_existing( kind="baseline", diff --git a/src/hyperloom/orchestrator/state/shared_state.py b/src/hyperloom/orchestrator/state/shared_state.py index 04875fc55..7b493d42d 100644 --- a/src/hyperloom/orchestrator/state/shared_state.py +++ b/src/hyperloom/orchestrator/state/shared_state.py @@ -464,6 +464,12 @@ class SharedState(_RenderMixin, _ExploreStateMixin): enablement_stall_streak: int = 0 # Launch-log hashes already recorded as needs_human_review; one record per log. enablement_human_review_logged: list = field(default_factory=list) + # Path to the materialized config produced by the KEEP'd candidate bench. + # This is the effective config (with server fixes applied) used for + # revalidation; distinct from enablement_probe_config_path (the original + # trigger config before any enablement patches). Optional; defaults via + # from_dict, no schema bump. + enablement_accepted_config_path: str = "" # Attempt-scoped runtime acquisition state. All optional; NOT in # fact_layer_keys and do NOT bump schema_version (default via from_dict). # ``enablement_stack_actions``: candidate EnablementStackAction dicts considered. From 0c012d68e8674fad10a2b155563b17dc4b098ead Mon Sep 17 00:00:00 2001 From: ZhengGong-amd Date: Sun, 26 Jul 2026 15:42:10 +0000 Subject: [PATCH 12/32] fix(enablement): make baseline revalidation task-aware Add enablement_revalidation_task_id and enablement_revalidation_generation to SharedState. Each eval-origin KEEP increments the generation so the next revalidation window gets a fresh idempotency key, preventing silent reuse of terminal TaskRegistry rows. _maybe_enqueue_enablement_baseline_revalidation now checks if the tracked task is still alive before creating a new one, and persists the new task_id immediately after creation. The old baseline_tput > 0 short-circuit is removed; a positive baseline tput is not a valid reason to skip enqueuing a revalidation for a pending KEEP. _promote_baseline is now task-aware: it only finalizes enablement when the promoting baseline matches the tracked revalidation task_id or carries the "enablement_eval_revalidation" reason. An unrelated baseline only anchors tput; it does not consume the pending state. Sub-floor accuracy on the tracked task increments the stall streak and reopens the specialist loop without clearing the frozen trigger identity. All non-success outcomes of the tracked revalidation task (boot failure, OOM, timeout, sub-floor accuracy) now share a unified recovery path in _handle_unpromotable_result that clears pending, clears the tracked task_id, increments stall, and rearns the specialist loop, bounded by the existing _ENABLEMENT_MAX_STALL cap. _resume_consistency_pass gains a new _resume_recover_pending_revalidation step: on restart, if the tracked revalidation task is already terminal or missing, the pending flag is cleared and stall is incremented so a fresh revalidation can be dispatched. Tests cover: tracked-task-only finalization, unrelated baseline isolation, sub-floor rearm, boot failure recovery, generation increment, resume cleanup, and the aligned stale-tput semantics. Co-authored-by: Cursor --- .../tests/test_coordinator_runtime.py | 77 ++++++++++ ...test_enablement_coordinator_wiring_unit.py | 40 ++++- src/hyperloom/orchestrator/loop/writeback.py | 140 ++++++++++++++++-- .../orchestrator/phases/framework.py | 40 +++-- .../orchestrator/state/shared_state.py | 7 + 5 files changed, 275 insertions(+), 29 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_coordinator_runtime.py b/src/hyperloom/inference_optimizer/tests/test_coordinator_runtime.py index 2fd0aaa24..0427bc0a8 100644 --- a/src/hyperloom/inference_optimizer/tests/test_coordinator_runtime.py +++ b/src/hyperloom/inference_optimizer/tests/test_coordinator_runtime.py @@ -952,6 +952,8 @@ async def test_promote_baseline_finalizes_eval_origin_when_accuracy_meets_floor( c.shared_state.enablement_origin = "eval" c.shared_state.enablement_validation_pending = True c.shared_state.enablement_accuracy_floor = 0.3 + # Set tracked task_id so the gate recognizes this as the revalidation task. + c.shared_state.enablement_revalidation_task_id = "t-reval-ok" await c._promote_to_shared_state( "baseline", {"output_throughput": 1000.0, "completed_requests": 10, "accuracy": 0.42}, @@ -965,6 +967,55 @@ async def test_promote_baseline_finalizes_eval_origin_when_accuracy_meets_floor( await c.stop() +@pytest.mark.asyncio +async def test_promote_baseline_unrelated_baseline_does_not_consume_pending(session_dir): + """A baseline that is NOT the tracked revalidation task must not consume pending state.""" + c = Coordinator(session_dir, backends=_silent_backends()) + _mute_action_scoring(c) + try: + c.shared_state.enablement_origin = "eval" + c.shared_state.enablement_validation_pending = True + c.shared_state.enablement_accuracy_floor = 0.3 + c.shared_state.enablement_revalidation_task_id = "t-reval-tracked" + await c._promote_to_shared_state( + "baseline", + {"output_throughput": 1000.0, "completed_requests": 10, "accuracy": 0.42}, + task=_mk_task("baseline", "t-unrelated"), + ) + assert c.shared_state.baseline_tput == 1000.0 + # Pending state must NOT be consumed by the unrelated baseline. + assert c.shared_state.enablement_validation_pending is True + assert c.shared_state.enablement_succeeded is False + assert c.shared_state.enablement_revalidation_task_id == "t-reval-tracked" + finally: + await c.stop() + + +@pytest.mark.asyncio +async def test_promote_baseline_sub_floor_accuracy_rearmes_stall(session_dir): + """Tracked revalidation baseline with sub-floor accuracy should rearm, not succeed.""" + c = Coordinator(session_dir, backends=_silent_backends()) + _mute_action_scoring(c) + try: + c.shared_state.enablement_origin = "eval" + c.shared_state.enablement_validation_pending = True + c.shared_state.enablement_accuracy_floor = 0.8 + c.shared_state.enablement_revalidation_task_id = "t-reval-subflo" + await c._promote_to_shared_state( + "baseline", + {"output_throughput": 1000.0, "completed_requests": 10, "accuracy": 0.5}, + task=_mk_task("baseline", "t-reval-subflo"), + ) + # Baseline tput anchors normally, but enablement is NOT succeeded. + assert c.shared_state.baseline_tput == 1000.0 + assert c.shared_state.enablement_succeeded is False + assert c.shared_state.enablement_validation_pending is False + assert c.shared_state.enablement_stall_streak == 1 + assert c.shared_state.enablement_revalidation_task_id == "" + finally: + await c.stop() + + @pytest.mark.asyncio async def test_persist_eval_failure_clears_pending_and_counts_stall(session_dir, monkeypatch): monkeypatch.delenv("INFERENCE_OPTIMIZER_NODES", raising=False) @@ -972,6 +1023,7 @@ async def test_persist_eval_failure_clears_pending_and_counts_stall(session_dir, _mute_action_scoring(c) try: c.shared_state.enablement_validation_pending = True + c.shared_state.enablement_revalidation_task_id = "t-reval-fail" await c._handle_unpromotable_result(_mk_task("baseline", "t-reval-fail"), _eval_failed_result()) assert c.shared_state.enablement_validation_pending is False assert c.shared_state.enablement_stall_streak == 1 @@ -979,6 +1031,31 @@ async def test_persist_eval_failure_clears_pending_and_counts_stall(session_dir, await c.stop() +@pytest.mark.asyncio +async def test_revalidation_boot_failure_clears_pending_and_rearmes(session_dir, monkeypatch): + """Any revalidation failure (including plain boot failures) clears pending and increments stall.""" + monkeypatch.delenv("INFERENCE_OPTIMIZER_NODES", raising=False) + c = Coordinator(session_dir, backends=_silent_backends()) + _mute_action_scoring(c) + try: + c.shared_state.enablement_validation_pending = True + c.shared_state.enablement_revalidation_task_id = "t-reval-boot" + c.shared_state.enablement_eval_contract_fingerprint = "frozen-fp" + c.shared_state.enablement_accuracy_floor = 0.5 + await c._handle_unpromotable_result( + _mk_task("baseline", "t-reval-boot"), + {"status": "failed", "error_class": "oom"}, + ) + assert c.shared_state.enablement_validation_pending is False + assert c.shared_state.enablement_stall_streak == 1 + assert c.shared_state.enablement_revalidation_task_id == "" + # Frozen trigger identity must be preserved. + assert c.shared_state.enablement_eval_contract_fingerprint == "frozen-fp" + assert c.shared_state.enablement_accuracy_floor == 0.5 + finally: + await c.stop() + + @pytest.mark.asyncio async def test_handle_unpromotable_records_for_non_baseline_kinds(session_dir): c = Coordinator(session_dir, backends=_silent_backends()) diff --git a/src/hyperloom/inference_optimizer/tests/test_enablement_coordinator_wiring_unit.py b/src/hyperloom/inference_optimizer/tests/test_enablement_coordinator_wiring_unit.py index 2c975607e..f14985439 100644 --- a/src/hyperloom/inference_optimizer/tests/test_enablement_coordinator_wiring_unit.py +++ b/src/hyperloom/inference_optimizer/tests/test_enablement_coordinator_wiring_unit.py @@ -234,6 +234,8 @@ def _enqueue_self(**state_kw): enablement_accuracy_floor=state_kw.get("enablement_accuracy_floor", 0.0), enablement_eval_contract_fingerprint=state_kw.get("enablement_eval_contract_fingerprint", ""), enablement_active_runtime=state_kw.get("enablement_active_runtime", {}), + enablement_revalidation_task_id=state_kw.get("enablement_revalidation_task_id", ""), + enablement_revalidation_generation=state_kw.get("enablement_revalidation_generation", 0), tick=state_kw.get("tick", 0), stop_reason=state_kw.get("stop_reason", ""), save=lambda *a, **k: None, @@ -425,12 +427,16 @@ async def test_rearm_kept_is_terminal(monkeypatch): @pytest.mark.asyncio async def test_rearm_kept_eval_origin_holds_for_revalidation(monkeypatch): - fake = _enqueue_self(enablement_dispatched=True, enablement_origin="eval") + fake = _enqueue_self(enablement_dispatched=True, enablement_origin="eval", enablement_revalidation_generation=1) fake._maybe_rearm_enablement({"enablement": True, "status": "kept"}) # eval-origin KEEP is NOT terminal: hold succeeded, open validation window. assert fake.shared_state.enablement_validation_pending is True assert fake.shared_state.enablement_succeeded is False assert fake.shared_state.enablement_dispatched is False + # Generation is incremented to get a fresh idempotency key. + assert fake.shared_state.enablement_revalidation_generation == 2 + # Tracked task_id is cleared for the new window. + assert fake.shared_state.enablement_revalidation_task_id == "" # The specialist pump is blocked while validation is pending. from hyperloom.orchestrator.actions.executors import _multi_node_env as mne @@ -493,17 +499,35 @@ async def test_revalidation_carries_active_runtime(): @pytest.mark.asyncio -async def test_revalidation_skips_when_tput_positive(): - fake = _enqueue_self(enablement_validation_pending=True, enablement_origin="eval", baseline_tput=123.0) - assert await fake._maybe_enqueue_enablement_baseline_revalidation() == "" +async def test_revalidation_skips_when_already_tracked(): + """If a tracked revalidation task is already alive, no new task is created.""" + fake = _enqueue_self( + enablement_validation_pending=True, + enablement_origin="eval", + enablement_revalidation_task_id="existing-spec-1", + ) + # Put the tracked task in the running list so it appears alive. + import types as _types + + fake.tasks = _FakeTasks(running=[_types.SimpleNamespace(kind="baseline", task_id="existing-spec-1")]) + result = await fake._maybe_enqueue_enablement_baseline_revalidation() + assert result == "existing-spec-1" assert fake.tasks.created == [] @pytest.mark.asyncio -async def test_revalidation_skips_when_baseline_in_flight(): - fake = _enqueue_self(enablement_validation_pending=True, enablement_origin="eval") - fake.tasks = _FakeTasks(running=[types.SimpleNamespace(kind="baseline")]) - assert await fake._maybe_enqueue_enablement_baseline_revalidation() == "" +async def test_revalidation_skips_when_tracked_task_in_flight(): + """Skip enqueue when the tracked revalidation task is still running.""" + import types as _types + + fake = _enqueue_self( + enablement_validation_pending=True, + enablement_origin="eval", + enablement_revalidation_task_id="reval-in-flight", + ) + fake.tasks = _FakeTasks(running=[_types.SimpleNamespace(kind="baseline", task_id="reval-in-flight")]) + result = await fake._maybe_enqueue_enablement_baseline_revalidation() + assert result == "reval-in-flight" assert fake.tasks.created == [] diff --git a/src/hyperloom/orchestrator/loop/writeback.py b/src/hyperloom/orchestrator/loop/writeback.py index 33526558d..897f63391 100644 --- a/src/hyperloom/orchestrator/loop/writeback.py +++ b/src/hyperloom/orchestrator/loop/writeback.py @@ -616,6 +616,39 @@ async def _handle_unpromotable_result( or int(getattr(self.shared_state, "enablement_attempts", 0) or 0) > 0 ) eval_failed = bool(result_payload.get(BASELINE_EVAL_FAILED_KEY)) + # Revalidation task failed for any reason (boot/OOM/timeout/eval): clear + # pending state, preserve the frozen trigger identity, increment stall. + reval_tid = str(getattr(self.shared_state, "enablement_revalidation_task_id", "") or "").strip() + is_revalidation = bool( + (task.params or {}).get("reason") == "enablement_eval_revalidation" + or (reval_tid and reval_tid == str(task.task_id or "")) + ) + if is_revalidation and bool(getattr(self.shared_state, "enablement_validation_pending", False)): + self.shared_state.enablement_validation_pending = False + self.shared_state.enablement_revalidation_task_id = "" + self.shared_state.enablement_stall_streak = ( + int(getattr(self.shared_state, "enablement_stall_streak", 0) or 0) + 1 + ) + try: + from ..phases.framework import _ENABLEMENT_MAX_STALL as _max_stall + except ImportError: + _max_stall = 5 + if self.shared_state.enablement_stall_streak >= _max_stall and not self.shared_state.stop_reason: + self.shared_state.set_stop_reason("enablement_stalled") + else: + self.shared_state.enablement_dispatched = False + launch_log = _extract_enablement_launch_log(result_payload) + if launch_log: + self.shared_state.enablement_launch_log = launch_log + log.warning( + "enablement revalidation task %s failed (error_class=%s); " + "stall_streak=%d rearm=%s", + task.task_id, + err_class, + self.shared_state.enablement_stall_streak, + not bool(self.shared_state.stop_reason), + ) + any_changed = True from ..actions.executors._multi_node_env import is_multi_node # noqa: PLC0415 # Single-node eval-pending failure: throughput measured fine and the @@ -2272,17 +2305,61 @@ async def _promote_baseline( self.shared_state.baseline_tput = float(tput) self.shared_state.baseline_failure_streak = 0 self.shared_state.baseline_arg_error_streak = 0 - # A genuine anchor revalidates an eval-origin enablement: reaching here - # means the result was promotable (not baseline_eval_failed), i.e. the - # accuracy met the floor, so finalize the enablement success. + # A genuine baseline may revalidate an eval-origin enablement. if bool(getattr(self.shared_state, "enablement_validation_pending", False)): - acc = result.get("accuracy") - floor = float(getattr(self.shared_state, "enablement_accuracy_floor", 0.0) or 0.0) - if accuracy_meets_floor(acc, floor): - self.shared_state.enablement_succeeded = True - self.shared_state.enablement_validation_pending = False - self.shared_state.enablement_origin = "" - self.shared_state.enablement_pending = False + tracked_tid = str(getattr(self.shared_state, "enablement_revalidation_task_id", "") or "").strip() + promoting_tid = str(getattr(task, "task_id", "") or "").strip() if task is not None else "" + task_params = getattr(task, "params", None) or {} if task is not None else {} + is_revalidation = ( + task_params.get("reason") == "enablement_eval_revalidation" + or (tracked_tid and tracked_tid == promoting_tid) + ) + if is_revalidation: + acc = result.get("accuracy") + floor = float(getattr(self.shared_state, "enablement_accuracy_floor", 0.0) or 0.0) + if accuracy_meets_floor(acc, floor): + self.shared_state.enablement_succeeded = True + self.shared_state.enablement_validation_pending = False + self.shared_state.enablement_revalidation_task_id = "" + self.shared_state.enablement_origin = "" + self.shared_state.enablement_pending = False + else: + # Sub-floor accuracy on the tracked revalidation: rearm the + # specialist loop without clearing the frozen trigger identity. + log.warning( + "enablement revalidation: accuracy %.4f below floor %.4f; rearming", + acc if isinstance(acc, (int, float)) else float("nan"), + floor, + ) + self.shared_state.enablement_validation_pending = False + self.shared_state.enablement_revalidation_task_id = "" + self.shared_state.enablement_stall_streak = ( + int(getattr(self.shared_state, "enablement_stall_streak", 0) or 0) + 1 + ) + _max_stall = getattr(self, "_ENABLEMENT_MAX_STALL", None) + if _max_stall is None: + try: + from ..phases.framework import _ENABLEMENT_MAX_STALL as _max_stall + except ImportError: + _max_stall = 5 + if self.shared_state.enablement_stall_streak >= _max_stall and not self.shared_state.stop_reason: + self.shared_state.set_stop_reason("enablement_stalled") + else: + self.shared_state.enablement_dispatched = False + else: + # An unrelated baseline promoted while revalidation is pending. + # Only anchor tput; do not consume or clear the pending state. + log.info( + "enablement revalidation pending: unrelated baseline promoted " + "(task=%s tracked=%s); not consuming pending state", + promoting_tid, + tracked_tid, + ) + self.shared_state.enablement_origin = "" + self.shared_state.enablement_pending = False + else: + self.shared_state.enablement_origin = "" + self.shared_state.enablement_pending = False changed = True acc = result.get("accuracy") if isinstance(acc, (int, float)): @@ -3305,6 +3382,10 @@ async def _resume_consistency_pass(self) -> dict[str, Any]: # coordinator restart, so kill the orphan group, GC its attempt dir, # sweep its jit locks, fail the row, and clear the sentinel. await self._resume_recover_pending_targeted_build(report) + # (1c) Orphaned revalidation tasks: if enablement_validation_pending is set + # but the tracked revalidation task is already terminal, clear the pending + # flag and rearm the stall counter so a fresh revalidation can be enqueued. + await self._resume_recover_pending_revalidation(report) # (2) Orphaned KEEPs: replay integrate_patch KEEPs # that crashed before the append landed; surface ambiguous ones loudly. await self._resume_recover_orphaned_keeps(report) @@ -3617,6 +3698,45 @@ async def _resume_recover_pending_targeted_build(self, report: dict[str, Any]) - state.pending_targeted_build = {} report["fixes"].append(summary) + async def _resume_recover_pending_revalidation(self, report: dict[str, Any]) -> None: + """Clear stale enablement_validation_pending when the tracked revalidation task is terminal. + + If the coordinator died while a revalidation baseline was running, the + task row may already be in a terminal state on resume. Without this + recovery the pending flag stays set indefinitely and the next + revalidation cannot be enqueued (tracked_tid is still the old row). + """ + state = self.shared_state + if not bool(getattr(state, "enablement_validation_pending", False)): + return + tracked_tid = str(getattr(state, "enablement_revalidation_task_id", "") or "").strip() + if not tracked_tid: + return + try: + from ..state.task_registry import TERMINAL_STATES, TaskNotFound + + try: + row = await self.tasks.get(tracked_tid) + is_terminal = row.state in TERMINAL_STATES + except TaskNotFound: + is_terminal = True + if is_terminal: + state.enablement_validation_pending = False + state.enablement_revalidation_task_id = "" + state.enablement_stall_streak = ( + int(getattr(state, "enablement_stall_streak", 0) or 0) + 1 + ) + state.enablement_dispatched = False + report["fixes"].append( + {"kind": "cleared_orphaned_revalidation_pending", "task_id": tracked_tid} + ) + log.info( + "resume: cleared stale enablement_validation_pending for terminal revalidation task %s", + tracked_tid, + ) + except Exception: # noqa: BLE001 — best-effort + log.debug("resume: revalidation pending recovery check failed", exc_info=True) + async def _resume_recover_orphaned_keeps(self, report: dict[str, Any]) -> None: """Recover / surface KEEPs present in the event log but absent from the stack (Gap B). diff --git a/src/hyperloom/orchestrator/phases/framework.py b/src/hyperloom/orchestrator/phases/framework.py index 74ec60131..5e67546b0 100644 --- a/src/hyperloom/orchestrator/phases/framework.py +++ b/src/hyperloom/orchestrator/phases/framework.py @@ -1842,6 +1842,12 @@ def _stack_kept_runtime() -> None: # reach the stall cap. state.enablement_validation_pending = True state.enablement_dispatched = False + # Increment generation so the new window gets a fresh idempotency + # key and cannot reuse a prior terminal TaskRegistry row. + state.enablement_revalidation_generation = ( + int(getattr(state, "enablement_revalidation_generation", 0) or 0) + 1 + ) + state.enablement_revalidation_task_id = "" else: state.enablement_succeeded = True state.enablement_stall_streak = 0 @@ -4565,14 +4571,16 @@ async def _maybe_enqueue_enablement_baseline_revalidation(self) -> str: state = self.shared_state if not bool(getattr(state, "enablement_validation_pending", False)): return "" - if float(getattr(state, "baseline_tput", 0.0) or 0.0) > 0: - return "" - try: - for t in (*await self.tasks.queued(), *await self.tasks.running()): - if getattr(t, "kind", "") == "baseline": - return "" - except Exception: # noqa: BLE001 — defensive - return "" + # If we already have a tracked revalidation task that is still alive, do + # not create another one. + tracked_tid = str(getattr(state, "enablement_revalidation_task_id", "") or "").strip() + if tracked_tid: + try: + for t in (*await self.tasks.queued(), *await self.tasks.running()): + if str(getattr(t, "task_id", "") or "") == tracked_tid: + return tracked_tid + except Exception: # noqa: BLE001 — defensive + pass params: dict[str, Any] = { "source": "coordinator_internal", "reason": "enablement_eval_revalidation", @@ -4597,13 +4605,23 @@ async def _maybe_enqueue_enablement_baseline_revalidation(self) -> str: rt_override = rt_obj.to_runtime_override() if rt_override: params["runtime_override"] = rt_override - attempt = int(getattr(state, "enablement_attempts", 0) or 0) + # Use generation in the idempotency key so each revalidation window gets + # a fresh row even when a prior window's row is in a terminal state. + gen = int(getattr(state, "enablement_revalidation_generation", 0) or 0) task, _existing = await self.tasks.create_or_return_existing( kind="baseline", params=params, - idempotency_key=f"enablement_revalidation:{attempt}", + idempotency_key=f"enablement_revalidation:gen{gen}", ) - return str(getattr(task, "task_id", "") or "") + task_id = str(getattr(task, "task_id", "") or "") + # Persist the task_id so _promote_baseline can verify identity. + if task_id and task_id != str(getattr(state, "enablement_revalidation_task_id", "") or ""): + state.enablement_revalidation_task_id = task_id + try: + state.save(self.session_dir) + except Exception: # noqa: BLE001 — defensive + log.debug("enablement revalidation: save of task_id failed", exc_info=True) + return task_id async def _pump_enablement_safely(self, *, caller: str) -> None: """Phase-independent enablement pump — runs every tick. diff --git a/src/hyperloom/orchestrator/state/shared_state.py b/src/hyperloom/orchestrator/state/shared_state.py index 7b493d42d..712aa82d0 100644 --- a/src/hyperloom/orchestrator/state/shared_state.py +++ b/src/hyperloom/orchestrator/state/shared_state.py @@ -470,6 +470,13 @@ class SharedState(_RenderMixin, _ExploreStateMixin): # trigger config before any enablement patches). Optional; defaults via # from_dict, no schema bump. enablement_accepted_config_path: str = "" + # Task identity for the current revalidation baseline task. Cleared when + # the task finishes (success or failure). Optional; defaults via from_dict. + enablement_revalidation_task_id: str = "" + # Monotonically increasing counter: incremented each time an eval-origin + # KEEP opens a new revalidation window so each window gets a fresh idempotency + # key and cannot reuse a prior terminal TaskRegistry row. + enablement_revalidation_generation: int = 0 # Attempt-scoped runtime acquisition state. All optional; NOT in # fact_layer_keys and do NOT bump schema_version (default via from_dict). # ``enablement_stack_actions``: candidate EnablementStackAction dicts considered. From b62649491861b51dcbb3e80314f10cdeee17f144 Mon Sep 17 00:00:00 2001 From: ZhengGong-amd Date: Sun, 26 Jul 2026 15:54:05 +0000 Subject: [PATCH 13/32] fix(breakdown): preserve evaluation enablement history after success collect_enablement now detects eval-origin sessions via enablement_baseline_eval_kind (which is not cleared on success) rather than relying solely on the active enablement_origin (which is reset to "" after successful promotion). This means the trigger_kind, observed accuracy, floor, contract fingerprint and probe config remain visible in session_breakdown.json even after the revalidation baseline promoted. Add succeeded and revalidation_task_id to the EnablementBreakdown schema and to the collector, exposing the final validation outcome and the tracked revalidation task identity. Add accepted_config_path to the schema so the accepted (post-fix) config used for revalidation is surfaced alongside the original probe config. Record richer eval-verdict fields in _promote_integrate_patch audit_extras (correctness_verified, eval_kind, observed_accuracy, drift flag, provisional) so the integrate_patch attempt history carries per-round accuracy verdicts. Record revalidation context in _promote_baseline audit_extras (is_revalidation, enablement_succeeded, floor) so the baseline attempt history tracks which promotions are revalidations and what their outcome was. Update .env.template comment to clarify score==floor pass semantics, strict-positive-required invariant and single-node scope. Tests verify: history is preserved after origin is cleared; succeeded flag is exposed correctly; combined origin/succeeded state works end to end. Co-authored-by: Cursor --- .env.template | 2 + .../breakdown/collectors/sessions.py | 17 +++++- .../inference_optimizer/breakdown/schema.py | 9 +++ .../tests/test_enablement_breakdown.py | 55 +++++++++++++++++++ src/hyperloom/orchestrator/loop/writeback.py | 13 ++++- 5 files changed, 92 insertions(+), 4 deletions(-) diff --git a/.env.template b/.env.template index db474010a..d97b3d38c 100644 --- a/.env.template +++ b/.env.template @@ -71,6 +71,8 @@ OPENAI_BASE_URL=https:///api/v1/llm-proxy/v1 # INFERENCE_OPTIMIZER_ENABLEMENT_ON_EVAL_FAIL=1 # (Optional) Shared accuracy floor for the eval-failure trigger AND the # enablement KEEP gate (finite, in [0,1]). Default 0.0. +# Note: score=0.0 always fails regardless of floor (strictly positive required). +# score>=floor passes (score==floor is a pass). Single-node only. # INFERENCE_OPTIMIZER_ENABLEMENT_ACCURACY_FLOOR=0.0 # Writable artifact root: hosts every session dir, optimizer_runs/, and the diff --git a/src/hyperloom/inference_optimizer/breakdown/collectors/sessions.py b/src/hyperloom/inference_optimizer/breakdown/collectors/sessions.py index 280f25100..517fa5013 100644 --- a/src/hyperloom/inference_optimizer/breakdown/collectors/sessions.py +++ b/src/hyperloom/inference_optimizer/breakdown/collectors/sessions.py @@ -1354,28 +1354,39 @@ def collect_enablement( last_build_failure_raw = state.get("enablement_last_build_failure") origin = str(state.get("enablement_origin") or "") + # eval_kind is NOT cleared on success, so it can identify an eval-origin + # enablement even after the run succeeds and origin is reset to "". + eval_kind = str(state.get("enablement_baseline_eval_kind") or "") have_active = isinstance(active_runtime_raw, dict) and bool(active_runtime_raw) have_attempts = isinstance(attempt_runtimes_raw, list) and bool(attempt_runtimes_raw) have_actions = isinstance(stack_actions_raw, list) and bool(stack_actions_raw) have_build_manifest = isinstance(build_manifest_raw, list) and bool(build_manifest_raw) have_last_failure = isinstance(last_build_failure_raw, dict) and bool(last_build_failure_raw) - have_eval = origin == "eval" + # Detect eval-origin by active origin OR persisted kind from a completed run. + have_eval = origin == "eval" or bool(eval_kind) if not (have_active or have_attempts or have_actions or have_build_manifest or have_last_failure or have_eval): return {} out: dict[str, Any] = {} if have_eval: - out["origin"] = origin - out["trigger_kind"] = str(state.get("enablement_baseline_eval_kind") or "") + out["origin"] = origin if origin else "eval" + out["trigger_kind"] = eval_kind out["observed_accuracy"] = float(state.get("enablement_observed_accuracy") or 0.0) out["accuracy_floor"] = float(state.get("enablement_accuracy_floor") or 0.0) out["observed_task"] = str(state.get("enablement_observed_task") or "") out["observed_metric"] = str(state.get("enablement_observed_metric") or "") out["eval_contract_fingerprint"] = str(state.get("enablement_eval_contract_fingerprint") or "") out["validation_pending"] = bool(state.get("enablement_validation_pending")) + out["succeeded"] = bool(state.get("enablement_succeeded")) probe_cfg = str(state.get("enablement_probe_config_path") or "") if probe_cfg: out["probe_config_path"] = _rel(Path(probe_cfg), session_dir) or probe_cfg + accepted_cfg = str(state.get("enablement_accepted_config_path") or "") + if accepted_cfg: + out["accepted_config_path"] = _rel(Path(accepted_cfg), session_dir) or accepted_cfg + reval_tid = str(state.get("enablement_revalidation_task_id") or "") + if reval_tid: + out["revalidation_task_id"] = reval_tid if have_actions: summaries: list[dict[str, Any]] = [] for a in stack_actions_raw: diff --git a/src/hyperloom/inference_optimizer/breakdown/schema.py b/src/hyperloom/inference_optimizer/breakdown/schema.py index 464340742..9b563dd0c 100644 --- a/src/hyperloom/inference_optimizer/breakdown/schema.py +++ b/src/hyperloom/inference_optimizer/breakdown/schema.py @@ -2130,9 +2130,15 @@ class EnablementBreakdown(TypedDict, total=False): observed_task: Eval task name observed at the trigger. observed_metric: Eval metric observed at the trigger. probe_config_path: Materialized config re-run to reproduce the contract. + accepted_config_path: Effective config from the KEEP'd candidate bench + used as the revalidation baseline config. eval_contract_fingerprint: Fingerprint of the captured eval contract. validation_pending: True while an eval-origin KEEP awaits baseline revalidation. + succeeded: True once the revalidation baseline promoted with accuracy + at or above the floor. + revalidation_task_id: TaskRegistry id of the tracked revalidation task, + or "" when no revalidation is in progress. """ stack_actions: list[EnablementStackActionSummary] @@ -2149,8 +2155,11 @@ class EnablementBreakdown(TypedDict, total=False): observed_task: str observed_metric: str probe_config_path: str + accepted_config_path: str eval_contract_fingerprint: str validation_pending: bool + succeeded: bool + revalidation_task_id: str # --------------------------------------------------------------------------- diff --git a/src/hyperloom/inference_optimizer/tests/test_enablement_breakdown.py b/src/hyperloom/inference_optimizer/tests/test_enablement_breakdown.py index f16b5c380..a2efda303 100644 --- a/src/hyperloom/inference_optimizer/tests/test_enablement_breakdown.py +++ b/src/hyperloom/inference_optimizer/tests/test_enablement_breakdown.py @@ -143,3 +143,58 @@ def test_collect_enablement_combined_rung3_and_rung5(): assert "build_attempts" in out assert "last_build_failure" in out assert out["last_build_failure"]["failure_class"] == "symbol_missing" + + +def test_collect_enablement_history_preserved_after_success(): + """After success, enablement_origin is cleared to '' but trigger history must still be surfaced.""" + out = collect_enablement( + Path("/tmp"), + _state( + # After success, origin is cleared but eval_kind is preserved. + enablement_origin="", + enablement_succeeded=True, + enablement_baseline_eval_kind="accuracy_below_floor", + enablement_observed_accuracy=0.12, + enablement_accuracy_floor=0.3, + enablement_eval_contract_fingerprint="fp-done", + enablement_validation_pending=False, + enablement_probe_config_path="/tmp/runs/probe.yaml", + enablement_accepted_config_path="/tmp/runs/accepted.yaml", + ), + [], + ) + # The block should still be non-empty despite origin being cleared. + assert out + assert out.get("trigger_kind") == "accuracy_below_floor" + assert out.get("observed_accuracy") == 0.12 + assert out.get("eval_contract_fingerprint") == "fp-done" + assert out.get("succeeded") is True + assert out.get("validation_pending") is False + + +def test_collect_enablement_succeeded_flag(): + """succeeded field is exposed correctly in the breakdown.""" + pending_out = collect_enablement( + Path("/tmp"), + _state( + enablement_origin="eval", + enablement_baseline_eval_kind="accuracy_unavailable", + enablement_succeeded=False, + enablement_validation_pending=True, + ), + [], + ) + assert pending_out.get("succeeded") is False + assert pending_out.get("validation_pending") is True + + done_out = collect_enablement( + Path("/tmp"), + _state( + enablement_origin="", + enablement_baseline_eval_kind="accuracy_unavailable", + enablement_succeeded=True, + enablement_validation_pending=False, + ), + [], + ) + assert done_out.get("succeeded") is True diff --git a/src/hyperloom/orchestrator/loop/writeback.py b/src/hyperloom/orchestrator/loop/writeback.py index 897f63391..dcab375eb 100644 --- a/src/hyperloom/orchestrator/loop/writeback.py +++ b/src/hyperloom/orchestrator/loop/writeback.py @@ -2413,12 +2413,17 @@ async def _promote_baseline( } changed = True audit_decision = "promoted" if isinstance(tput, (int, float)) and tput > 0 else "discarded" + _task_params = task.params if task is not None else {} audit_extras = { "materialized_config": result.get("materialized_config"), "accuracy": result.get("accuracy"), "baseline_tput": (float(tput) if isinstance(tput, (int, float)) else None), # Stamp canonical params fingerprint for the self-loop denial helper. - "fingerprint": _baseline_params_fingerprint(task.params if task is not None else None), + "fingerprint": _baseline_params_fingerprint(_task_params), + # Record revalidation context for history. + "is_revalidation": bool(_task_params.get("reason") == "enablement_eval_revalidation"), + "enablement_succeeded": bool(getattr(self.shared_state, "enablement_succeeded", False)), + "enablement_accuracy_floor": float(getattr(self.shared_state, "enablement_accuracy_floor", 0.0) or 0.0), } # seed the gaps[] ledger from baseline (best-effort). await self._refresh_gaps(reason="baseline_done") @@ -2977,6 +2982,12 @@ async def _promote_integrate_patch( "accuracy_pass": result.get("accuracy_pass"), "patches_applied": result.get("patches_applied") or [], "patches_reverted": result.get("patches_reverted") or [], + # Enablement eval-origin verdict fields for history. + "correctness_verified": result.get("correctness_verified"), + "enablement_eval_failure_kind": result.get("enablement_eval_failure_kind"), + "enablement_observed_accuracy": result.get("enablement_observed_accuracy"), + "enablement_eval_contract_drift": result.get("enablement_eval_contract_drift"), + "provisional": result.get("provisional"), } outcome.changed = changed outcome.audit_decision = audit_decision From 0596c2d2ca8247e2bbd34b70181bb4279ce14fc2 Mon Sep 17 00:00:00 2001 From: ZhengGong-amd Date: Tue, 28 Jul 2026 02:13:08 +0000 Subject: [PATCH 14/32] fix(model_gate): recognize HuggingFace tokenizer model_max_length for max position lookup Add "model_max_length" to _MAXPOS_CONFIG_KEYS so the gate can resolve max position embeddings from HuggingFace tokenizer_config fields and custom models such as kimi_linear. --- src/hyperloom/inference_optimizer/cli/model_gate.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/hyperloom/inference_optimizer/cli/model_gate.py b/src/hyperloom/inference_optimizer/cli/model_gate.py index 7adc38a21..566a8cfe7 100644 --- a/src/hyperloom/inference_optimizer/cli/model_gate.py +++ b/src/hyperloom/inference_optimizer/cli/model_gate.py @@ -206,6 +206,7 @@ def _resolve_amd_gpu_type(explicit: str | None = None) -> str | None: "max_sequence_length", "seq_length", "max_seq_len", + "model_max_length", # HuggingFace tokenizer_config field; used by some custom models (e.g. kimi_linear) ) _ROPE_CONFIG_KEYS = ("rope_scaling", "rope_parameters", "rope_theta") From 08da16dcff97d66e06afe7a85a70082d1c05de27 Mon Sep 17 00:00:00 2001 From: ZhengGong-amd Date: Tue, 28 Jul 2026 07:17:09 +0000 Subject: [PATCH 15/32] fix(baseline): close the disable_run_eval escape from the accuracy anchor A genuine baseline exists to establish the accuracy reference, but _maybe_stop_on_missing_baseline_accuracy let any run that had turned the eval off skip the guard entirely. Orchestration found that door: after a baseline measured throughput fine but failed the accuracy gate (so it was correctly not promoted), the LLM re-proposed the same baseline with disable_run_eval=true purely "to set baseline_tput on throughput", the Critic approved it, and the eval-less retry anchored the run. That single promotion poisons everything downstream. The materialized YAML it writes carries RUN_EVAL=false and becomes SharedState.baseline_config_path, which later explore/integrate tasks inherit as their config_path, so their variants render with the eval off too. baseline_accuracy stays 0.0, and both _grade_accuracy and _framework_run_eval_envs key on baseline > 0, so the per-variant accuracy gates silently degrade to throughput-only KEEPs. Remove the opt-out. Disabling the eval does not make a missing accuracy acceptable; it only means the reference was never measured, which is exactly what the guard exists to reject. With eval-on-fail enablement active (the default) the result is stamped as an eval-failure contract and routed to enablement, and _is_promotable_result then keeps it from anchoring baseline_tput / baseline_accuracy / baseline_config_path. This also removes the incentive: disable_run_eval no longer buys a baseline_tput at all. The kernel lane drives this same executor through synthetic tasks that carry kind="baseline" literally (integrate re-baseline, the paired-A/B pristine arm, stack validation). Those are throughput-only A/B probes against an already-anchored baseline and must not be caught by the guard -- stack validation in particular passes the live SharedState, so a missing accuracy there would have dragged a healthy run back into enablement. They now opt out via params["quality_ref_exempt"], handled in _should_establish_quality_ref so the exemption also stops them from overwriting the scriptable image-quality reference via establish_quality_ref -- previously a patched candidate could redefine the very reference it was being judged against. Verified by replaying the two real baselines from the Kimi-Linear-48B session (20260726T162542Z) through the fixed path: both now report promotable=False, where the eval-less retry previously promoted and anchored baseline_tput. --- .../tests/test_baseline_eval_fallback.py | 65 +++++++++++++++---- .../test_baseline_quality_ref_gate_unit.py | 18 +++++ .../actions/executors/baseline.py | 60 ++++++++++------- .../orchestrator/kernel/request_handlers.py | 8 +++ .../orchestrator/phases/kernel_stack.py | 7 ++ 5 files changed, 121 insertions(+), 37 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_baseline_eval_fallback.py b/src/hyperloom/inference_optimizer/tests/test_baseline_eval_fallback.py index cddc0d287..5f18665a3 100644 --- a/src/hyperloom/inference_optimizer/tests/test_baseline_eval_fallback.py +++ b/src/hyperloom/inference_optimizer/tests/test_baseline_eval_fallback.py @@ -363,9 +363,14 @@ def fake_run(cmd, *args, **kwargs): assert state.stop_reason == "baseline_accuracy_failed" -def test_baseline_operator_disabled_eval_does_not_stop(tmp_path): - """When the operator explicitly disables the serving eval, accuracy is - intentionally off: a missing accuracy result must NOT stop the run.""" +def test_baseline_operator_disabled_eval_still_stops(tmp_path): + """Disabling the serving eval on a genuine baseline is not an opt-out. + + A baseline exists to establish the accuracy reference, so turning the eval + off does not make a missing accuracy acceptable -- it only means the + reference was never measured. The run halts either way, which is what makes + ``disable_run_eval`` useless as a way around the accuracy gate. + """ from hyperloom.orchestrator.state.shared_state import SharedState base = tmp_path / "base.yaml" @@ -399,7 +404,7 @@ def fake_run(cmd, *args, **kwargs): result = _run(executor(ctx)) assert result["status"] == "succeeded" - assert state.stop_reason == "" + assert state.stop_reason == "baseline_accuracy_failed" def test_baseline_eval_failure_fallback_stops_run(tmp_path): @@ -459,16 +464,20 @@ def set_stop_reason(self, value, **_kwargs): return value -def _stop_ctx(framework: str, recorder) -> SimpleNamespace: - task = SimpleNamespace(task_id="t-bl", kind="baseline", params={"framework": framework}) +def _stop_ctx(framework: str, recorder, params: dict | None = None) -> SimpleNamespace: + task = SimpleNamespace( + task_id="t-bl", + kind="baseline", + params={"framework": framework, **(params or {})}, + ) return SimpleNamespace(task=task, extra={"shared_state": recorder}) -def _stopped(framework: str, result: dict) -> str: +def _stopped(framework: str, result: dict, *, params: dict | None = None) -> str: """Run ``_maybe_stop_on_missing_baseline_accuracy`` and return the reason.""" executor = BaselineExecutor() rec = _StopRecorder() - executor._maybe_stop_on_missing_baseline_accuracy(_stop_ctx(framework, rec), result) + executor._maybe_stop_on_missing_baseline_accuracy(_stop_ctx(framework, rec, params), result) return rec.stop_reason @@ -498,13 +507,25 @@ def test_stop_serving_zero_accuracy(): assert reason == "baseline_accuracy_failed" -def test_no_stop_serving_operator_disabled_via_config(): - # Finding: YAML/reference-env RUN_EVAL=false folds into run_eval_disabled; - # operator opt-out must NOT stop even without disable_run_eval param. +def test_stop_serving_operator_disabled_via_config(): + # A YAML/reference-env RUN_EVAL=false folds into run_eval_disabled, but a + # genuine baseline has no opt-out: turning the eval off does not make a + # missing accuracy reference acceptable, so the run still stops. reason = _stopped( "sglang", {"status": "succeeded", "run_eval_disabled": True}, ) + assert reason == "baseline_accuracy_failed" + + +def test_no_stop_when_quality_ref_exempt(): + # Synthetic kernel-lane re-baselines (kind="baseline" + quality_ref_exempt) + # are throughput-only A/B probes: no accuracy, no stop. + reason = _stopped( + "sglang", + {"status": "succeeded", "run_eval_disabled": True}, + params={"quality_ref_exempt": True}, + ) assert reason == "" @@ -716,8 +737,28 @@ def test_eval_enablement_multi_node_falls_back_to_stop(monkeypatch): assert BASELINE_EVAL_FAILED_KEY not in result -def test_eval_enablement_operator_optout_not_routed(monkeypatch): +def test_eval_enablement_operator_optout_is_routed(monkeypatch): + """A disabled eval is not an opt-out: with enablement on it routes there. + + Rather than stopping, the missing reference is stamped as an eval-failure + contract so enablement can repair the eval path. ``_is_promotable_result`` + then keeps this baseline from anchoring tput / accuracy / config. + """ result = {"status": "succeeded", "run_eval_disabled": True} reason = _route(monkeypatch, "sglang", result) assert reason == "" + assert result[BASELINE_EVAL_FAILED_KEY] is True + + +def test_eval_enablement_quality_ref_exempt_not_routed(monkeypatch): + """Synthetic kernel-lane re-baselines are neither routed nor stopped.""" + result = {"status": "succeeded", "run_eval_disabled": True} + executor = BaselineExecutor() + rec = _StopRecorder() + monkeypatch.setenv("INFERENCE_OPTIMIZER_ENABLEMENT_ON_EVAL_FAIL", "1") + monkeypatch.delenv("INFERENCE_OPTIMIZER_NODES", raising=False) + executor._maybe_stop_on_missing_baseline_accuracy( + _stop_ctx("sglang", rec, {"quality_ref_exempt": True}), result + ) + assert rec.stop_reason == "" assert BASELINE_EVAL_FAILED_KEY not in result diff --git a/src/hyperloom/inference_optimizer/tests/test_baseline_quality_ref_gate_unit.py b/src/hyperloom/inference_optimizer/tests/test_baseline_quality_ref_gate_unit.py index 365343310..8f66b5065 100644 --- a/src/hyperloom/inference_optimizer/tests/test_baseline_quality_ref_gate_unit.py +++ b/src/hyperloom/inference_optimizer/tests/test_baseline_quality_ref_gate_unit.py @@ -34,3 +34,21 @@ def test_missing_or_empty_kind_does_not_establish(): """Defensive: ``None`` / empty kind must not be treated as a baseline.""" assert _should_establish_quality_ref(None) is False assert _should_establish_quality_ref("") is False + + +def test_quality_ref_exempt_baseline_does_not_establish(): + """Synthetic kernel-lane re-baselines opt out of establishing the ref. + + Integrate re-baseline, the paired-A/B pristine arm and stack validation all + carry ``kind="baseline"`` literally, but are throughput-only probes against + an already-anchored baseline. + """ + assert _should_establish_quality_ref("baseline", {"quality_ref_exempt": True}) is False + + +def test_plain_params_still_establish(): + """Params without the exemption leave a genuine baseline untouched.""" + assert _should_establish_quality_ref("baseline", {}) is True + assert _should_establish_quality_ref("baseline", None) is True + assert _should_establish_quality_ref("baseline", {"framework": "sglang"}) is True + assert _should_establish_quality_ref("baseline", {"quality_ref_exempt": False}) is True diff --git a/src/hyperloom/orchestrator/actions/executors/baseline.py b/src/hyperloom/orchestrator/actions/executors/baseline.py index a77224ab9..82113874d 100644 --- a/src/hyperloom/orchestrator/actions/executors/baseline.py +++ b/src/hyperloom/orchestrator/actions/executors/baseline.py @@ -313,7 +313,7 @@ def _resolve_result_dir(output_dir: Path, override_result_dir: str | None) -> Pa return (output_dir / result_dir).resolve() -def _should_establish_quality_ref(task_kind: str | None) -> bool: +def _should_establish_quality_ref(task_kind: str | None, params: dict[str, Any] | None = None) -> bool: """Only a genuine ``baseline`` task may establish/overwrite the quality reference. ``replay_warm_recipe`` reuses this executor but is an optimization @@ -321,14 +321,25 @@ def _should_establish_quality_ref(task_kind: str | None) -> bool: than redefine it (otherwise the gate would mask the warm recipe's own deviation from the baseline output). + The kernel lane also drives this executor through synthetic tasks that + carry ``kind="baseline"`` literally (integrate re-baseline, the paired-A/B + pristine arm, stack validation). Those are throughput-only A/B probes + against an already-anchored baseline -- they never anchor one -- so they + opt out via ``params["quality_ref_exempt"]`` and are treated exactly like + ``replay_warm_recipe``: compare, never establish. + Args: task_kind: The task kind (``ctx.task.kind``); ``None`` is treated as "not a baseline". + params: The task params (``ctx.task.params``); a truthy + ``quality_ref_exempt`` disqualifies an otherwise-genuine baseline. Returns: - bool: ``True`` only when the task kind is exactly ``"baseline"``. + bool: ``True`` only for a ``"baseline"`` task that is not exempt. """ - return str(task_kind or "") == "baseline" + if str(task_kind or "") != "baseline": + return False + return not (params or {}).get("quality_ref_exempt") def _resolve_reference_base( @@ -1235,7 +1246,7 @@ def _eval_enablement_active(self, ctx: RunnerContext) -> bool: return False if is_multi_node(): return False - return _should_establish_quality_ref(getattr(ctx.task, "kind", "")) + return _should_establish_quality_ref(getattr(ctx.task, "kind", ""), ctx.task.params or {}) def _stamp_eval_failure_contract( self, @@ -1298,21 +1309,25 @@ def _maybe_stop_on_missing_baseline_accuracy( workloads record ``accuracy=0.0`` (fail-closed) when the quality gate is absent, and serving records no accuracy at all, so both are covered. - "Expected" means accuracy was not intentionally turned off. Scriptable - workloads always carry the quality gate. For serving, the operator can - opt out via ``disable_run_eval``, an explicit ``RUN_EVAL=false`` env, or - a YAML/reference-env ``RUN_EVAL=false`` -- all folded into the - authoritative ``run_eval_disabled`` on the result. The eval-failure - fallback also disables eval, but it is tagged - ``accuracy_source="eval_unavailable"`` and must still stop (eval was - expected and broke). Throughput-level baseline failures are handled by - the Coordinator's existing ``baseline_failed`` streak logic. + There is no opt-out. Disabling the eval (via ``disable_run_eval``, an + explicit ``RUN_EVAL=false`` env, or a YAML/reference-env value) does not + make a missing accuracy acceptable on a genuine baseline -- it only + means the reference was never measured, which is exactly the state this + guard exists to reject. Synthetic kernel-lane re-baselines that reuse + ``kind="baseline"`` opt out earlier, via ``quality_ref_exempt``. + + With eval-on-fail enablement active (the default), the result is stamped + as an eval-failure contract and routed to enablement rather than + stopping; ``_is_promotable_result`` then blocks it from anchoring + ``baseline_tput`` / ``baseline_accuracy`` / ``baseline_config_path``. + Otherwise the run stops. Throughput-level baseline failures are handled + by the Coordinator's existing ``baseline_failed`` streak logic. Args: ctx (RunnerContext): The runner context (task kind + shared_state). result (dict): The final baseline result dict. """ - if not _should_establish_quality_ref(getattr(ctx.task, "kind", "")): + if not _should_establish_quality_ref(getattr(ctx.task, "kind", ""), ctx.task.params or {}): return if result.get("status") != "succeeded": return @@ -1328,18 +1343,13 @@ def _maybe_stop_on_missing_baseline_accuracy( return # a usable baseline accuracy exists params = ctx.task.params or {} framework = str(params.get("framework") or "").strip() or os.environ.get("FRAMEWORK", "").strip() or None - from hyperloom.inference_optimizer import framework_registry from ._accuracy_gate import request_baseline_accuracy_stop - scriptable = framework_registry.is_scriptable(framework) - # Operator opt-out only when eval was disabled AND this is not the - # eval-failure fallback (which forces RUN_EVAL=false but still means the - # eval was expected and broke). Scriptable always runs its quality gate. - operator_disabled_eval = bool(result.get("run_eval_disabled")) and ( - result.get("accuracy_source") != "eval_unavailable" - ) - if not (scriptable or not operator_disabled_eval): - return + # No opt-out: a genuine baseline exists to establish the accuracy + # reference, so turning the eval off does not make a missing accuracy + # acceptable -- it only means the reference was never measured. Every + # disable path (the ``disable_run_eval`` param, an explicit + # ``RUN_EVAL=false`` env, a YAML/reference-env value) now lands here. # Route into enablement instead of stopping/salvaging: the throughput # baseline stays for diagnostics but is blocked from anchoring. if eval_enablement: @@ -1470,7 +1480,7 @@ async def _run_once( # Only a genuine ``baseline`` task may establish/overwrite the quality # reference; ``replay_warm_recipe`` reuses this executor but must compare # against the pure baseline reference rather than redefine it. - is_genuine_baseline = _should_establish_quality_ref(getattr(ctx.task, "kind", "")) + is_genuine_baseline = _should_establish_quality_ref(getattr(ctx.task, "kind", ""), params) config_path = Path(params.get("config_path") or self.default_config_path or self._resolve_default_config()) if not config_path.exists(): raise FileNotFoundError(f"baseline config not found: {config_path}") diff --git a/src/hyperloom/orchestrator/kernel/request_handlers.py b/src/hyperloom/orchestrator/kernel/request_handlers.py index 60af0ea1d..bf95e8785 100644 --- a/src/hyperloom/orchestrator/kernel/request_handlers.py +++ b/src/hyperloom/orchestrator/kernel/request_handlers.py @@ -5482,6 +5482,11 @@ async def integrate_handler( "timeout_sec": int(payload.get("budget_minutes", 20)) * 60, "extra_server_args": extra_args, "extra_envs": dict(payload.get("extra_envs") or {}), + # Synthetic kind="baseline": a throughput-only A/B probe of the + # patched kernel against the already-anchored baseline. It never + # establishes the quality reference, so it is exempt from the + # genuine-baseline accuracy guard. + "quality_ref_exempt": True, }, idempotency_key=f"{fake_task_id}-rebaseline", ) @@ -5692,6 +5697,9 @@ def _restore_aiter_rebuild_env() -> None: "timeout_sec": int(payload.get("budget_minutes", 20)) * 60, "extra_server_args": extra_args, "extra_envs": dict(payload.get("extra_envs") or {}), + # Pristine arm of the paired A/B: compares against the + # anchored baseline, never establishes it. + "quality_ref_exempt": True, }, idempotency_key=f"{fake_task_id}-pairedbase", ) diff --git a/src/hyperloom/orchestrator/phases/kernel_stack.py b/src/hyperloom/orchestrator/phases/kernel_stack.py index f035c3c84..ae6729d9c 100644 --- a/src/hyperloom/orchestrator/phases/kernel_stack.py +++ b/src/hyperloom/orchestrator/phases/kernel_stack.py @@ -471,6 +471,13 @@ async def _run_kernel_stack_validation_e2e( "output_dir": str(workspace), "timeout_sec": 20 * 60, "extra_server_args": ((self.shared_state.current_best or {}).get("extra_server_args") or ""), + # Synthetic kind="baseline": validates the stacked kernels + # against the already-anchored baseline on throughput alone. + # Exempt from the genuine-baseline accuracy guard -- this ctx + # carries the live SharedState, so without the exemption a + # missing accuracy here would stamp an eval-failure contract + # and drag a healthy run back into enablement. + "quality_ref_exempt": True, }, idempotency_key=f"integrate-stack-{stack_id}-rebaseline", ) From cfd02abea85954b488213b37fc95ff163a7e4cec Mon Sep 17 00:00:00 2001 From: ZhengGong-amd Date: Tue, 28 Jul 2026 07:58:30 +0000 Subject: [PATCH 16/32] fix(baseline,install): unblock cold bring-up of very large checkpoints Bringing up Kimi-K3 (1454 GiB MXFP4 MoE, vLLM, TP=8) from scratch failed at two separate gates before any measurement could happen. install.sh (both the inference_optimizer entrypoint and the chained kernel-agent one) rejected the single-gateway SAFE_API_KEY + OPENAI_BASE_URL setup that SKILL.md documents as the common deployment and that the CLI's own _validate_credentials() accepts, so IR-2 could never pass. Derive the Anthropic aliases from the gateway pair the same way _resolve_llm_endpoints() does (strip a trailing /v1, reuse the gateway key), with explicit values still winning. The baseline executor bumped its subprocess timeout on COLD_START but left Magpie's server_ready_timeout_s at the flat 2700s default, so the inner gate always fired first and the cold-start bump was unreachable for any slow-booting model. A healthy server 82% through loading weights was killed at 2881s and reported as subprocess_nonzero, then leaked its VRAM and cascaded the next attempt into server_init_dead. Weight loading dominates boot on large checkpoints and scales with bytes, not with aiter JIT warmth, so size the budget from the checkpoint instead: models under 200 GiB keep the old default, larger ones add an allowance at a pessimistic 0.20 GiB/s floor, capped at 6h so a genuine hang still times out. Because this lands in inject_lifecycle, explore/sweep/integrate rounds benefit too. Also floor the subprocess timeout above the boot budget so the outer kill can no longer preempt a legitimate load. test_aiter_jit was reading $MODEL_PATH from the ambient shell, which now changes the resolved timeouts; pin it hermetic. Co-authored-by: Cursor --- .../agents/kernel/scripts/install.sh | 19 +++- .../inference_optimizer/assets/install.sh | 20 +++- .../tests/test_aiter_jit.py | 12 +++ .../actions/executors/_server_lifecycle.py | 97 +++++++++++++++++-- .../actions/executors/baseline.py | 48 ++++++++- 5 files changed, 183 insertions(+), 13 deletions(-) diff --git a/src/hyperloom/agents/kernel/scripts/install.sh b/src/hyperloom/agents/kernel/scripts/install.sh index 2eb6d6445..d527d5e07 100644 --- a/src/hyperloom/agents/kernel/scripts/install.sh +++ b/src/hyperloom/agents/kernel/scripts/install.sh @@ -246,6 +246,21 @@ if [ -z "${ANTHROPIC_BASE_URL:-}" ] || [ -z "${ANTHROPIC_API_KEY:-}" ] \ echo "[kernel-agent] loaded credentials fallback from $REPO_ROOT/.env (env wins)" fi fi +# Single-gateway (AMD / LiteLLM-style) setup: only SAFE_API_KEY + OPENAI_BASE_URL +# are configured. Mirror the CLI's _resolve_llm_endpoints(): the Anthropic base +# is OPENAI_BASE_URL with a trailing /v1 stripped (the SDK re-appends it) and the +# gateway key doubles as the Anthropic key. Explicit values always win. +if [ -n "${SAFE_API_KEY:-}" ] && [ -n "${OPENAI_BASE_URL:-}" ]; then + if [ -z "${ANTHROPIC_BASE_URL:-}" ]; then + _gw_url="${OPENAI_BASE_URL%/}" + export ANTHROPIC_BASE_URL="${_gw_url%/v1}" + unset _gw_url + echo "[kernel-agent] derived ANTHROPIC_BASE_URL from OPENAI_BASE_URL (single gateway)" + fi + if [ -z "${ANTHROPIC_API_KEY:-}" ] && [ -z "${ANTHROPIC_AUTH_TOKEN:-}" ]; then + export ANTHROPIC_API_KEY="$SAFE_API_KEY" + fi +fi # e2e whole-pipeline optimizer — Hyperloom calls it simply "geak" (formerly the # standalone PerfSkills repo / GEAK_v4). Its code lives IN GEAK (interface/run_e2e.py # + e2e_workflow/), tracked on the ``main`` branch. Hyperloom calls @@ -415,8 +430,8 @@ preflight_validate_credentials() { local has_url=0 has_key=0 { [ -n "${ANTHROPIC_BASE_URL:-}" ] || [ -n "${DEEPSEEK_BASE_URL:-}" ] || [ -n "${DEEPSEEK_API_KEY:-}" ]; } && has_url=1 { [ -n "${ANTHROPIC_API_KEY:-}" ] || [ -n "${ANTHROPIC_AUTH_TOKEN:-}" ] || [ -n "${DEEPSEEK_API_KEY:-}" ]; } && has_key=1 - [ "$has_url" -eq 0 ] && missing+=("ANTHROPIC_BASE_URL or DEEPSEEK_BASE_URL (DeepSeek may omit the URL)") - [ "$has_key" -eq 0 ] && missing+=("ANTHROPIC_API_KEY, ANTHROPIC_AUTH_TOKEN, or DEEPSEEK_API_KEY") + [ "$has_url" -eq 0 ] && missing+=("ANTHROPIC_BASE_URL or DEEPSEEK_BASE_URL (DeepSeek may omit the URL), or SAFE_API_KEY + OPENAI_BASE_URL") + [ "$has_key" -eq 0 ] && missing+=("ANTHROPIC_API_KEY, ANTHROPIC_AUTH_TOKEN, DEEPSEEK_API_KEY, or SAFE_API_KEY") if [ "$has_url" -eq 1 ] && [ "$has_key" -eq 1 ]; then log "credentials preflight: usable LLM base URL + key present" return 0 diff --git a/src/hyperloom/inference_optimizer/assets/install.sh b/src/hyperloom/inference_optimizer/assets/install.sh index a6e4e27e9..b29284a86 100755 --- a/src/hyperloom/inference_optimizer/assets/install.sh +++ b/src/hyperloom/inference_optimizer/assets/install.sh @@ -383,10 +383,26 @@ preflight_validate_credentials() { preflight_load_dotenv local missing=() local has_url=0 has_key=0 + # Single-gateway (AMD / LiteLLM-style) setup: only SAFE_API_KEY + + # OPENAI_BASE_URL are configured. Mirror the CLI's _resolve_llm_endpoints(): + # the Anthropic base is OPENAI_BASE_URL with a trailing /v1 stripped (the SDK + # re-appends it) and the gateway key doubles as the Anthropic key, so the + # chained kernel-agent installer sees a complete provider pair. Explicit + # values always win. + if [ -n "${SAFE_API_KEY:-}" ] && [ -n "${OPENAI_BASE_URL:-}" ]; then + if [ -z "${ANTHROPIC_BASE_URL:-}" ]; then + local _gw_url="${OPENAI_BASE_URL%/}" + export ANTHROPIC_BASE_URL="${_gw_url%/v1}" + log "derived ANTHROPIC_BASE_URL from OPENAI_BASE_URL (single gateway)" + fi + if [ -z "${ANTHROPIC_API_KEY:-}" ] && [ -z "${ANTHROPIC_AUTH_TOKEN:-}" ]; then + export ANTHROPIC_API_KEY="$SAFE_API_KEY" + fi + fi { [ -n "${ANTHROPIC_BASE_URL:-}" ] || [ -n "${DEEPSEEK_BASE_URL:-}" ] || [ -n "${DEEPSEEK_API_KEY:-}" ]; } && has_url=1 { [ -n "${ANTHROPIC_API_KEY:-}" ] || [ -n "${ANTHROPIC_AUTH_TOKEN:-}" ] || [ -n "${DEEPSEEK_API_KEY:-}" ]; } && has_key=1 - [ "$has_url" -eq 0 ] && missing+=("ANTHROPIC_BASE_URL or DEEPSEEK_BASE_URL (DeepSeek may omit the URL)") - [ "$has_key" -eq 0 ] && missing+=("ANTHROPIC_API_KEY, ANTHROPIC_AUTH_TOKEN, or DEEPSEEK_API_KEY") + [ "$has_url" -eq 0 ] && missing+=("ANTHROPIC_BASE_URL or DEEPSEEK_BASE_URL (DeepSeek may omit the URL), or SAFE_API_KEY + OPENAI_BASE_URL") + [ "$has_key" -eq 0 ] && missing+=("ANTHROPIC_API_KEY, ANTHROPIC_AUTH_TOKEN, DEEPSEEK_API_KEY, or SAFE_API_KEY") if [ "$has_url" -eq 1 ] && [ "$has_key" -eq 1 ]; then log "credentials preflight: usable LLM base URL + key present" return 0 diff --git a/src/hyperloom/inference_optimizer/tests/test_aiter_jit.py b/src/hyperloom/inference_optimizer/tests/test_aiter_jit.py index 3753a2bc8..f1dc8f2f7 100644 --- a/src/hyperloom/inference_optimizer/tests/test_aiter_jit.py +++ b/src/hyperloom/inference_optimizer/tests/test_aiter_jit.py @@ -173,6 +173,18 @@ def test_sweep_unknown_falls_back_to_mtime_gate(monkeypatch, tmp_path): # --------------------------------------------------------------------------- # _resolve_timeout integration # --------------------------------------------------------------------------- +@pytest.fixture(autouse=True) +def _no_ambient_model_path(monkeypatch): + """Keep ``_resolve_timeout`` off the server-ready floor in these tests. + + The floor scales with checkpoint size read from ``$MODEL_PATH``, so an + operator shell that exports a real model would otherwise change the + timeouts these tests assert on. + """ + monkeypatch.delenv("MODEL_PATH", raising=False) + monkeypatch.delenv("INFERENCE_OPTIMIZER_BASELINE_SERVER_READY_SEC", raising=False) + + def _cold_probe(*_a, **_k): return { "path": "/fake/jit", diff --git a/src/hyperloom/orchestrator/actions/executors/_server_lifecycle.py b/src/hyperloom/orchestrator/actions/executors/_server_lifecycle.py index 67f7949cd..3bc18dde7 100644 --- a/src/hyperloom/orchestrator/actions/executors/_server_lifecycle.py +++ b/src/hyperloom/orchestrator/actions/executors/_server_lifecycle.py @@ -61,6 +61,23 @@ # Override via ``INFERENCE_OPTIMIZER_BASELINE_SERVER_READY_SEC``. SERVER_READY_TIMEOUT_SEC = 2700 +# Weight loading dominates server boot for very large checkpoints, and it +# scales with checkpoint bytes rather than with anything the flat +# ``SERVER_READY_TIMEOUT_SEC`` default can express. A 1.4 TiB MoE checkpoint on +# a network filesystem needs well over an hour just to read the safetensors +# shards, so the flat 2700s default kills a perfectly healthy server mid-load +# (Magpie reports "Shared server launcher timed out"). Budget the read at a +# deliberately pessimistic floor throughput so the allowance stays generous on +# slow NFS mounts while still bounding a genuine hang. +CHECKPOINT_LOAD_FLOOR_GIB_PER_SEC = 0.20 + +# Upper bound on the size-derived allowance: past this a stuck loader should +# surface as a timeout rather than idling for the rest of the budget. +SERVER_READY_MAX_TIMEOUT_SEC = 21600 # 6 h + +# Checkpoints below this size keep the flat default (no scaling). +CHECKPOINT_SCALING_MIN_GIB = 200.0 + def _pick_free_port() -> int: """Return an OS-assigned free TCP port. @@ -200,6 +217,79 @@ def resolve_lifecycle_params(materialized_config_path: Path) -> dict[str, Any]: return info +def _checkpoint_size_gib(model_path: str | None) -> float: + """Return the on-disk weight size of ``model_path`` in GiB (0.0 when unknown). + + Sums the top-level weight shards only (``*.safetensors`` / ``*.bin``); a + recursive walk over an HF cache tree is not worth the stat storm on a + network filesystem. + + Args: + model_path: Local model directory, or ``None`` / an HF repo id. + + Returns: + The summed shard size in GiB, or ``0.0`` when the path is not a + readable local directory. + """ + raw = str(model_path or "").strip() + if not raw: + return 0.0 + try: + root = Path(raw) + if not root.is_dir(): + return 0.0 + total = 0 + for pattern in ("*.safetensors", "*.bin"): + for shard in root.glob(pattern): + try: + total += shard.stat().st_size + except OSError: + continue + return total / (1024.0**3) + except OSError: + return 0.0 + + +def resolve_server_ready_timeout(model_path: str | None) -> int: + """Return the server-boot budget in seconds, scaled by checkpoint size. + + ``INFERENCE_OPTIMIZER_BASELINE_SERVER_READY_SEC`` wins when set. Otherwise + small checkpoints keep the flat :data:`SERVER_READY_TIMEOUT_SEC` default, + and larger ones add an allowance for reading the weights at + :data:`CHECKPOINT_LOAD_FLOOR_GIB_PER_SEC`, clamped to + :data:`SERVER_READY_MAX_TIMEOUT_SEC` so a hung loader still times out. + + Callers that also impose an outer subprocess timeout must keep it above + this value (plus Magpie's own 180s launcher grace), or the outer kill + preempts the boot with a less specific error. + + Args: + model_path: Local model directory used to size the allowance. + + Returns: + The server-ready timeout in seconds. + """ + env_override = str(os.environ.get("INFERENCE_OPTIMIZER_BASELINE_SERVER_READY_SEC", "") or "").strip() + if env_override: + return int(env_override) + size_gib = _checkpoint_size_gib(model_path) + if size_gib < CHECKPOINT_SCALING_MIN_GIB: + return SERVER_READY_TIMEOUT_SEC + load_allowance = int(size_gib / CHECKPOINT_LOAD_FLOOR_GIB_PER_SEC) + scaled = min(SERVER_READY_TIMEOUT_SEC + load_allowance, SERVER_READY_MAX_TIMEOUT_SEC) + log.info( + "server_lifecycle: checkpoint %.0f GiB at %s — server_ready_timeout_s " + "%d -> %d (weight-load allowance %ds at a %.2f GiB/s floor)", + size_gib, + model_path, + SERVER_READY_TIMEOUT_SEC, + scaled, + load_allowance, + CHECKPOINT_LOAD_FLOOR_GIB_PER_SEC, + ) + return scaled + + def inject_lifecycle( bench: dict[str, Any], *, @@ -219,12 +309,7 @@ def inject_lifecycle( pid_dir: Shared directory for the server pid/meta files. port: HTTP port pinned for the persistent server. """ - ready_timeout = int( - os.environ.get( - "INFERENCE_OPTIMIZER_BASELINE_SERVER_READY_SEC", - SERVER_READY_TIMEOUT_SEC, - ) - ) + ready_timeout = resolve_server_ready_timeout(bench.get("model")) bench["server_lifecycle"] = { "enabled": True, "cleanup": bool(cleanup), diff --git a/src/hyperloom/orchestrator/actions/executors/baseline.py b/src/hyperloom/orchestrator/actions/executors/baseline.py index 5b0d8393a..31a59b28c 100644 --- a/src/hyperloom/orchestrator/actions/executors/baseline.py +++ b/src/hyperloom/orchestrator/actions/executors/baseline.py @@ -264,6 +264,14 @@ def _classify_subprocess_error( BASELINE_DEFAULT_TIMEOUT_SEC = 7800 # WARM-start cap, 130 min BASELINE_COLD_START_TIMEOUT_SEC = 9000 # COLD-start cap, 150 min (includes ~20 min cuda graph capture) + +# Slack the subprocess keeps beyond the server-ready budget: Magpie's launcher +# adds its own 180s grace on top of ``server_ready_timeout_s``, then the client +# benchmark and teardown still have to run. Without this the outer timeout can +# fire while the server is legitimately still booting, which reports the far +# less actionable ``timeout`` / ``subprocess_nonzero`` instead of a real +# server-boot failure. +BASELINE_POST_READY_MARGIN_SEC = 1800 # COLD_START_KERNEL_THRESHOLD and AITER_JIT_PROBE_PATHS live in ``_aiter_jit``; # re-exported below for callers/tests that import them from this module. @@ -1030,7 +1038,7 @@ def _resolve_timeout(self, params: dict[str, Any]) -> int: self.default_timeout_sec, cold_cap, ) - return cold_cap + return self._apply_server_ready_floor(cold_cap, params) if cache["probe_status"] == "found": log.info( "baseline_executor: WARM start — aiter jit/build/ at %s has %d .so, %d MB. Using default timeout=%ds.", @@ -1039,7 +1047,7 @@ def _resolve_timeout(self, params: dict[str, Any]) -> int: cache["size_mb"], self.default_timeout_sec, ) - return self.default_timeout_sec + return self._apply_server_ready_floor(self.default_timeout_sec, params) log.warning( "baseline_executor: aiter jit cache not located " "(probe_status=%s). Using default timeout=%ds. Cold-start " @@ -1047,7 +1055,41 @@ def _resolve_timeout(self, params: dict[str, Any]) -> int: cache["probe_status"], self.default_timeout_sec, ) - return self.default_timeout_sec + return self._apply_server_ready_floor(self.default_timeout_sec, params) + + def _apply_server_ready_floor(self, timeout_sec: int, params: dict[str, Any]) -> int: + """Raise ``timeout_sec`` so it cannot preempt the server-boot budget. + + The lifecycle ``server_ready_timeout_s`` scales with checkpoint size, + so on very large models it can exceed the aiter-probe-derived + subprocess cap. When that happens the subprocess is killed while the + server is still loading weights and the failure is misreported as a + benchmark timeout. + + Args: + timeout_sec: The probe-derived subprocess timeout. + params: Task params, consulted for ``model_path``. + + Returns: + ``timeout_sec``, or the server-ready budget plus + :data:`BASELINE_POST_READY_MARGIN_SEC` when that is larger. + """ + model_path = str(params.get("model_path") or os.environ.get("MODEL_PATH") or "").strip() + ready_sec = _lifecycle.resolve_server_ready_timeout(model_path) + floor = ready_sec + BASELINE_POST_READY_MARGIN_SEC + if floor <= timeout_sec: + return timeout_sec + log.warning( + "baseline_executor: raising timeout %ds -> %ds so it stays above " + "the server-boot budget (server_ready_timeout_s=%ds + %ds margin) " + "for %s.", + timeout_sec, + floor, + ready_sec, + BASELINE_POST_READY_MARGIN_SEC, + model_path or "", + ) + return floor @staticmethod def _inferencex_root_from_config(config_path: Path) -> str: From f8819e4549963e01af467b732b15bc44c98fac81 Mon Sep 17 00:00:00 2001 From: ZhengGong-amd Date: Wed, 29 Jul 2026 03:01:30 +0000 Subject: [PATCH 17/32] fix(enable-accuracy): harden eval patching, arch fallback, and PR MCP defaults Catch microbenchmark timeouts during gpu_arch_json population, derive the PR Monitor MCP URL from PRIMUS_CORTEX_PR_API, support grouped run_lm_eval parsers for --concurrent-requests, and skip commit-on-KEEP for non-git framework roots. Co-authored-by: Cursor --- .../kernel/tools/tracelens_arch_benchmark.py | 9 ++- .../inference_optimizer/cli/parser.py | 23 +++++- .../actions/executors/_magpie_patcher.py | 73 ++++++++++++++++--- .../actions/executors/integrate_patch.py | 36 +++++---- 4 files changed, 114 insertions(+), 27 deletions(-) diff --git a/src/hyperloom/agents/kernel/tools/tracelens_arch_benchmark.py b/src/hyperloom/agents/kernel/tools/tracelens_arch_benchmark.py index 7d5ade2e7..332202536 100644 --- a/src/hyperloom/agents/kernel/tools/tracelens_arch_benchmark.py +++ b/src/hyperloom/agents/kernel/tools/tracelens_arch_benchmark.py @@ -14,6 +14,7 @@ import json import os +import subprocess import sys from pathlib import Path from typing import Callable @@ -473,8 +474,12 @@ def populate_gpu_arch_json( log(f"gpu_arch_json: measured spec ready at {out_path}") return out_path - except RuntimeError as exc: - mb_error = exc + except (RuntimeError, subprocess.SubprocessError, OSError) as exc: + # A microbenchmark that overruns ``timeout_s`` surfaces as + # subprocess.TimeoutExpired, not RuntimeError, so catching RuntimeError + # alone let the most common failure escape and killed the whole + # trace_analyze instead of taking the documented fallback below. + mb_error = exc if isinstance(exc, RuntimeError) else RuntimeError(f"{type(exc).__name__}: {exc}") log(f"gpu_arch_json: microbenchmark unusable ({exc}); falling back to hyperloom achievable spec") hyperloom_spec = write_hyperloom_arch_spec(tracelens_root, canonical, log) diff --git a/src/hyperloom/inference_optimizer/cli/parser.py b/src/hyperloom/inference_optimizer/cli/parser.py index 6945bc497..21b24d66c 100644 --- a/src/hyperloom/inference_optimizer/cli/parser.py +++ b/src/hyperloom/inference_optimizer/cli/parser.py @@ -29,6 +29,22 @@ DEFAULT_PRECISION = "bf16" +def _default_pr_monitor_mcp_url() -> str | None: + """Derive the PR Monitor MCP URL from ``$PRIMUS_CORTEX_PR_API``. + + Follows the same ``/mcp`` convention the gbrain ``cortex_kb`` server + uses, so one env var wires both the REST client and the specialist MCP + server instead of silently leaving the MCP half disabled. The trailing + slash is kept because the bare ``/mcp`` form answers 307 and MCP clients do + not re-POST across the redirect. + + Returns: + The derived URL, or ``None`` when the env var is unset. + """ + base = (os.environ.get("PRIMUS_CORTEX_PR_API") or "").strip().rstrip("/") + return f"{base}/mcp/" if base else None + + class _RetiredFlag(argparse.Action): """Argparse action that hard-fails on a retired CLI flag with a migration hint (exits 2).""" @@ -967,10 +983,11 @@ def _build_parser() -> argparse.ArgumentParser: "--pr-monitor-mcp-url", dest="pr_monitor_mcp_url", type=str, - default=None, + default=_default_pr_monitor_mcp_url(), help="PR Monitor MCP URL handed to specialist LLM backends (flag " - "wins). Default: unset, which disables PR Monitor MCP tools. The " - "trailing slash is mandatory when configured.", + "wins). Default: $PRIMUS_CORTEX_PR_API + '/mcp/', else unset, which " + "disables PR Monitor MCP tools. The trailing slash is mandatory when " + "configured.", ) opt.add_argument( "--degraded-pr", diff --git a/src/hyperloom/orchestrator/actions/executors/_magpie_patcher.py b/src/hyperloom/orchestrator/actions/executors/_magpie_patcher.py index 530fc3aff..5725e4f0c 100644 --- a/src/hyperloom/orchestrator/actions/executors/_magpie_patcher.py +++ b/src/hyperloom/orchestrator/actions/executors/_magpie_patcher.py @@ -171,6 +171,30 @@ ' *) echo "Unknown parameter: $1"; return 1 ;;\n' ) +# Newer InferenceX folded every value-taking flag into one shared ``case`` arm +# that dispatches through an inner ``case``, so the flat ``--top-p) ...; +# shift 2 ;;`` line the block above anchors on no longer exists and the patch +# silently missed (install.sh then aborts on eval_flag_ok=False). Match that +# shape by its fallthrough arm instead and splice a dedicated arm in front of +# it. Scoped to the ``run_lm_eval`` body by the caller, and the ``>&2`` + +# ``return 2`` pair distinguishes this parser from the other two ``Unknown +# parameter`` fallthroughs in the same file. +_RUN_LM_EVAL_PARSER_GROUPED_FALLTHROUGH_RE = re.compile( + r"^(?P[ \t]*)\*\)\n" + r"[ \t]*echo \"Unknown parameter: \$1\" >&2\n" + r"[ \t]*return 2\n" + r"[ \t]*;;\n", + re.MULTILINE, +) +_RUN_LM_EVAL_PARSER_GROUPED_ARM = ( + "{indent}# HYPERLOOM_EVAL_CONCURRENCY_ARG: accept the redundant flag\n" + "{indent}# (concurrency also flows via EVAL_CONCURRENT_REQUESTS/CONC).\n" + "{indent}--concurrent-requests|--concurrent_requests)\n" + '{indent} concurrent_requests="$2"\n' + "{indent} shift 2\n" + "{indent} ;;\n" +) + # InferenceX benchmark_lib.sh::run_lm_eval reads concurrency from env # (EVAL_CONCURRENT_REQUESTS, fallback CONC). Passing --concurrent-requests to # run_eval is rejected as an unknown argument. @@ -437,6 +461,36 @@ def _apply_eval_flag_patch_atomic(scripts_dir: Path) -> bool: return ok +def _patch_grouped_run_lm_eval_parser(text: str) -> str | None: + """Insert a ``--concurrent-requests`` arm into the grouped-``case`` shape of + ``run_lm_eval``'s arg parser. + + Args: + text: Full ``benchmark_lib.sh`` source. + + Returns: + The patched source, or ``None`` when ``run_lm_eval`` or its + fallthrough arm could not be located. + """ + start = text.find("\nrun_lm_eval() {") + if start < 0: + return None + # Function bodies end at the first column-0 ``}``; bound the edit to this + # one so the other ``Unknown parameter`` fallthroughs stay untouched. + end = text.find("\n}\n", start) + if end < 0: + return None + + body = text[start:end] + match = _RUN_LM_EVAL_PARSER_GROUPED_FALLTHROUGH_RE.search(body) + if match is None: + return None + + arm = _RUN_LM_EVAL_PARSER_GROUPED_ARM.format(indent=match.group("indent")) + patched_body = body[: match.start()] + arm + body[match.start() :] + return text[:start] + patched_body + text[end:] + + def _apply_run_lm_eval_arg_patch_atomic(benchmark_lib: Path) -> bool: """Teach InferenceX's ``benchmark_lib.sh::run_lm_eval`` to accept the ``--concurrent-requests`` flag instead of aborting on ``Unknown parameter``. @@ -465,7 +519,16 @@ def _apply_run_lm_eval_arg_patch_atomic(benchmark_lib: Path) -> bool: if _RUN_LM_EVAL_PARSER_SENTINEL in original or _EVAL_CONCURRENCY_FLAG_MARKER in original: return True - if _RUN_LM_EVAL_PARSER_LEGACY_BLOCK not in original: + if _RUN_LM_EVAL_PARSER_LEGACY_BLOCK in original: + patched = original.replace( + _RUN_LM_EVAL_PARSER_LEGACY_BLOCK, + _RUN_LM_EVAL_PARSER_PATCHED_BLOCK, + 1, + ) + else: + patched = _patch_grouped_run_lm_eval_parser(original) + + if patched is None or patched == original: log.warning( "_magpie_patcher: run_lm_eval arg-parser block not found in %s; " "cannot make it tolerate '--concurrent-requests'. RUN_EVAL=true " @@ -474,14 +537,6 @@ def _apply_run_lm_eval_arg_patch_atomic(benchmark_lib: Path) -> bool: ) return False - patched = original.replace( - _RUN_LM_EVAL_PARSER_LEGACY_BLOCK, - _RUN_LM_EVAL_PARSER_PATCHED_BLOCK, - 1, - ) - if patched == original: - return False - if not atomic_write_text( benchmark_lib, patched, diff --git a/src/hyperloom/orchestrator/actions/executors/integrate_patch.py b/src/hyperloom/orchestrator/actions/executors/integrate_patch.py index f65bbbfd1..afb67e6d4 100644 --- a/src/hyperloom/orchestrator/actions/executors/integrate_patch.py +++ b/src/hyperloom/orchestrator/actions/executors/integrate_patch.py @@ -2612,21 +2612,31 @@ async def _gate_perf( extra=extra, ) # Commit the KEEP so a later REVERT checkout fallback can't wipe this - # win (best-effort, non-fatal). - try: - touched = _patch_touched_paths(framework_root, applied) - ok, note = _git_commit_kept( + # win (best-effort, non-fatal). Non-git roots (e.g. a pip-installed + # framework in site-packages) have no checkout fallback to guard + # against and get their durability from snapshot_source_layer below, + # so skip the commit instead of failing it — matches framework_agent. + if framework_root is not None and _is_git_tree(framework_root): + try: + touched = _patch_touched_paths(framework_root, applied) + ok, note = _git_commit_kept( + framework_root, + f"hyperloom KEEP {specialist_task_id} ({delta_pct:+.2f}%)", + touched, + ) + if not ok: + log.warning( + "integrate_patch: commit-on-KEEP failed (%s); win remains uncommitted in the working tree", + note, + ) + except Exception: # noqa: BLE001 — commit durability is best-effort + log.exception("integrate_patch: commit-on-KEEP raised") + else: + log.info( + "integrate_patch: non-git framework root %s; skipping commit-on-KEEP " + "(backup-based revert + source snapshot provide durability)", framework_root, - f"hyperloom KEEP {specialist_task_id} ({delta_pct:+.2f}%)", - touched, ) - if not ok: - log.warning( - "integrate_patch: commit-on-KEEP failed (%s); win remains uncommitted in the working tree", - note, - ) - except Exception: # noqa: BLE001 — commit durability is best-effort - log.exception("integrate_patch: commit-on-KEEP raised") source_snapshot_dir = "" source_base_sha = "" From c0b2b063261b5cbfde48de4db8535b48b0e2666a Mon Sep 17 00:00:00 2001 From: ZhengGong-amd Date: Wed, 29 Jul 2026 06:40:12 +0000 Subject: [PATCH 18/32] fix(enablement): keep the measured trigger when a re-baseline skips the eval A baseline that never ran the eval characterizes nothing: it lands in _persist_eval_failure as accuracy_unavailable with no task/metric/source. Because that path wrote every field unconditionally, such a round overwrote a stored accuracy_below_floor trigger -- so the next enablement attempt lost the very measurement it was supposed to reproduce. Make the overwrite one-way. An incoming accuracy_unavailable that carries no observed accuracy now leaves the stored characterization intact, while still registering the round as an eval-rooted failure (origin, pending, and the stall accounting that lets enablement_stalled terminate). A real measurement still replaces an earlier empty trigger. Contract fingerprints cannot gate this: RUN_EVAL is itself a contract field, so the eval-less run's fingerprint never matches the measured one. Co-Authored-By: Claude Opus 5 (1M context) --- .../tests/test_coordinator_runtime.py | 82 +++++++++++++++++++ src/hyperloom/orchestrator/loop/writeback.py | 36 +++++++- 2 files changed, 115 insertions(+), 3 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_coordinator_runtime.py b/src/hyperloom/inference_optimizer/tests/test_coordinator_runtime.py index 0427bc0a8..436b35303 100644 --- a/src/hyperloom/inference_optimizer/tests/test_coordinator_runtime.py +++ b/src/hyperloom/inference_optimizer/tests/test_coordinator_runtime.py @@ -1031,6 +1031,88 @@ async def test_persist_eval_failure_clears_pending_and_counts_stall(session_dir, await c.stop() +def _eval_unavailable_result() -> dict: + """An eval-less baseline: succeeded, but measured no accuracy at all.""" + return { + "status": "succeeded", + "baseline_eval_failed": True, + "baseline_eval_failure_kind": "accuracy_unavailable", + "baseline_eval_observed_accuracy": None, + "baseline_eval_accuracy_floor": 0.0, + "baseline_eval_evidence": ( + "baseline accuracy did not meet floor: accuracy=None floor=0.0 task=None metric=None source=None" + ), + "baseline_eval_contract_fingerprint": "noeval-fp", + "materialized_config": "/runs/baseline/noeval.yaml", + } + + +@pytest.mark.asyncio +async def test_eval_less_baseline_does_not_downgrade_measured_trigger(session_dir, monkeypatch): + """An eval-less re-baseline must not overwrite a measured enablement trigger. + + ``disable_run_eval`` re-baselines report ``accuracy_unavailable`` with no + task/metric/source. They still count as a failed round, but the stored + ``accuracy_below_floor`` evidence (the measurement enablement must + reproduce) has to survive. + """ + monkeypatch.delenv("INFERENCE_OPTIMIZER_NODES", raising=False) + c = Coordinator(session_dir, backends=_silent_backends()) + _mute_action_scoring(c) + try: + st = c.shared_state + st.enablement_baseline_eval_kind = "accuracy_below_floor" + st.enablement_observed_accuracy = 0.0 + st.enablement_observed_task = "gsm8k" + st.enablement_observed_metric = "exact_match,strict-match" + st.enablement_baseline_eval_evidence = "measured: accuracy=0.0 task=gsm8k source=/runs/.../results.json" + st.enablement_probe_config_path = "/runs/baseline/measured.yaml" + st.enablement_eval_contract_fingerprint = "measured-fp" + + await c._handle_unpromotable_result( + _mk_task("baseline", "t-noeval"), _eval_unavailable_result() + ) + + # The measured characterization survives... + assert st.enablement_baseline_eval_kind == "accuracy_below_floor" + assert st.enablement_observed_task == "gsm8k" + assert st.enablement_observed_metric == "exact_match,strict-match" + assert "accuracy=0.0" in st.enablement_baseline_eval_evidence + assert st.enablement_probe_config_path == "/runs/baseline/measured.yaml" + assert st.enablement_eval_contract_fingerprint == "measured-fp" + # ...while the round still registers as an eval-rooted failure. + assert st.enablement_origin == "eval" + assert st.enablement_pending is True + finally: + await c.stop() + + +@pytest.mark.asyncio +async def test_measured_trigger_overwrites_earlier_unavailable(session_dir, monkeypatch): + """The guard is one-way: a real measurement still replaces an empty trigger.""" + monkeypatch.delenv("INFERENCE_OPTIMIZER_NODES", raising=False) + c = Coordinator(session_dir, backends=_silent_backends()) + _mute_action_scoring(c) + try: + st = c.shared_state + st.enablement_baseline_eval_kind = "accuracy_unavailable" + st.enablement_baseline_eval_evidence = "accuracy=None" + + measured = _eval_unavailable_result() + measured["baseline_eval_failure_kind"] = "accuracy_below_floor" + measured["baseline_eval_observed_accuracy"] = 0.0 + measured["accuracy_task"] = "gsm8k" + measured["baseline_eval_evidence"] = "measured: accuracy=0.0 task=gsm8k" + + await c._handle_unpromotable_result(_mk_task("baseline", "t-measured"), measured) + + assert st.enablement_baseline_eval_kind == "accuracy_below_floor" + assert st.enablement_observed_task == "gsm8k" + assert "measured" in st.enablement_baseline_eval_evidence + finally: + await c.stop() + + @pytest.mark.asyncio async def test_revalidation_boot_failure_clears_pending_and_rearmes(session_dir, monkeypatch): """Any revalidation failure (including plain boot failures) clears pending and increments stall.""" diff --git a/src/hyperloom/orchestrator/loop/writeback.py b/src/hyperloom/orchestrator/loop/writeback.py index 19c93be17..7d25f7ee6 100644 --- a/src/hyperloom/orchestrator/loop/writeback.py +++ b/src/hyperloom/orchestrator/loop/writeback.py @@ -49,6 +49,7 @@ BASELINE_EVAL_FAILED_KEY, BASELINE_EVAL_FAILURE_KIND_KEY, BASELINE_EVAL_OBSERVED_ACCURACY_KEY, + EVAL_KIND_ACCURACY_UNAVAILABLE, accuracy_meets_floor, ) @@ -441,9 +442,29 @@ def _persist_eval_failure(self, result_payload: dict[str, Any]) -> None: Records origin, floor, probe config, contract fingerprint, evidence, kind and observed accuracy, and seeds ``enablement_launch_log`` so the FRAMEWORK pump dispatches even when the failure carries no boot log. + + A run that never executed the eval characterizes nothing: it reports + ``accuracy_unavailable`` with no task/metric/source. Such a run must + still register as a failed round (origin / pending / stall accounting + below), but it must NOT overwrite a stored trigger that actually + measured an accuracy -- otherwise an eval-less re-baseline downgrades + real ``accuracy_below_floor`` evidence to an empty + ``accuracy_unavailable`` and the next enablement attempt loses the + measurement it is supposed to reproduce. Contract fingerprints cannot + gate this: ``RUN_EVAL`` is itself a contract field, so the eval-less + run's fingerprint never matches the measured one. """ state = self.shared_state was_validation_pending = bool(getattr(state, "enablement_validation_pending", False)) + incoming_kind = result_payload.get(BASELINE_EVAL_FAILURE_KIND_KEY) + measured_incoming = to_float(result_payload.get(BASELINE_EVAL_OBSERVED_ACCURACY_KEY)) is not None + stored_kind = str(getattr(state, "enablement_baseline_eval_kind", "") or "") + preserve_measured_trigger = ( + incoming_kind == EVAL_KIND_ACCURACY_UNAVAILABLE + and not measured_incoming + and bool(stored_kind) + and stored_kind != EVAL_KIND_ACCURACY_UNAVAILABLE + ) state.enablement_origin = "eval" state.enablement_pending = True # A failed revalidation reopens the authoring loop and counts as a @@ -456,15 +477,24 @@ def _persist_eval_failure(self, result_payload: dict[str, Any]) -> None: floor = to_float(result_payload.get(BASELINE_EVAL_ACCURACY_FLOOR_KEY)) if floor is not None: state.enablement_accuracy_floor = float(floor) + if preserve_measured_trigger: + log.info( + "enablement: keeping measured trigger kind=%s (accuracy=%s task=%s); " + "an eval-less baseline reported accuracy_unavailable and must not " + "overwrite it", + stored_kind, + getattr(state, "enablement_observed_accuracy", None), + getattr(state, "enablement_observed_task", ""), + ) + return cfg = result_payload.get("materialized_config") if isinstance(cfg, str) and cfg: state.enablement_probe_config_path = cfg fp = result_payload.get(BASELINE_EVAL_CONTRACT_FINGERPRINT_KEY) if isinstance(fp, str) and fp: state.enablement_eval_contract_fingerprint = fp - kind = result_payload.get(BASELINE_EVAL_FAILURE_KIND_KEY) - if isinstance(kind, str) and kind: - state.enablement_baseline_eval_kind = kind + if isinstance(incoming_kind, str) and incoming_kind: + state.enablement_baseline_eval_kind = incoming_kind observed = to_float(result_payload.get(BASELINE_EVAL_OBSERVED_ACCURACY_KEY)) if observed is not None: state.enablement_observed_accuracy = float(observed) From 48c64cd85219d68eaf85198dc6de6914b7b9d2e7 Mon Sep 17 00:00:00 2001 From: ZhengGong-amd Date: Wed, 29 Jul 2026 10:24:01 +0000 Subject: [PATCH 19/32] fix(baseline): salvage sibling accuracy before routing to enablement The cold-start guard splits one baseline into two rounds: warmup_round runs with RUN_EVAL=true and is the only round that measures accuracy, while measure_round runs with RUN_EVAL=false to get a hot throughput number off the already-warm server. The accuracy decision is then made on the measure_round result, which by construction never carries an accuracy. With enablement active (INFERENCE_OPTIMIZER_ENABLEMENT_ON_EVAL_FAIL=1, the default) that missing value hit the enablement branch first and dispatched an authoring specialist to chase a quality regression that never happened. The sibling-salvage path that already exists to recover exactly this value sat below the branch's `return` and was unreachable, so baseline_tput was never anchored and the run spun in PRELUDE at gain 0 for the rest of its budget. Move the salvage ahead of the enablement branch so a sibling round's score is recovered first. A salvaged score at/above the floor anchors the baseline as usual; one below the floor is a real quality signal and still falls through to enablement, now with the observed value instead of None. Observed on Kimi-K3 (vllm, mi355x, mxfp4, TP=8): measure_round reported 1452.7 tok/s/GPU with no accuracy while warmup_round held gsm8k 0.8954, and the run stalled. After the fix the same session anchors baseline_tput=1444.4 with baseline_accuracy=0.9287. Co-Authored-By: Claude Opus 5 (1M context) --- .../actions/executors/baseline.py | 66 ++++++++++++------- 1 file changed, 41 insertions(+), 25 deletions(-) diff --git a/src/hyperloom/orchestrator/actions/executors/baseline.py b/src/hyperloom/orchestrator/actions/executors/baseline.py index 0d820af3a..efc45ab41 100644 --- a/src/hyperloom/orchestrator/actions/executors/baseline.py +++ b/src/hyperloom/orchestrator/actions/executors/baseline.py @@ -1677,31 +1677,6 @@ def _maybe_stop_on_missing_baseline_accuracy( framework = str(params.get("framework") or "").strip() or os.environ.get("FRAMEWORK", "").strip() or None from ._accuracy_gate import request_baseline_accuracy_stop - # No opt-out: a genuine baseline exists to establish the accuracy - # reference, so turning the eval off does not make a missing accuracy - # acceptable -- it only means the reference was never measured. Every - # disable path (the ``disable_run_eval`` param, an explicit - # ``RUN_EVAL=false`` env, a YAML/reference-env value) now lands here. - # Route into enablement instead of stopping/salvaging: the throughput - # baseline stays for diagnostics but is blocked from anchoring. - if eval_enablement: - kind = classify_accuracy_failure(acc, floor) - observed = float(acc) if isinstance(acc, (int, float)) else None - evidence = ( - f"baseline accuracy did not meet floor: accuracy={acc} floor={floor} " - f"task={result.get('accuracy_task')} metric={result.get('accuracy_metric')} " - f"source={result.get('accuracy_source')}" - ) - self._stamp_eval_failure_contract( - ctx, result, kind=kind or "", observed_accuracy=observed, evidence=evidence - ) - log.warning( - "baseline_executor: accuracy %s below floor %.4f; routing to " - "enablement instead of stopping the run.", - acc, - floor, - ) - return extra = getattr(ctx, "extra", None) or {} shared_state = extra.get("shared_state") or self.shared_state # Session-level salvage (complements #942): the cold-start guard and the @@ -1712,6 +1687,17 @@ def _maybe_stop_on_missing_baseline_accuracy( # produced a valid ``results*.json``. Before halting, reuse a positive # accuracy measured by any sibling attempt rather than discarding a good # baseline and stopping the whole run. + # + # This runs BEFORE the enablement branch below, not after it. The + # cold-start guard splits one baseline into warmup_round (RUN_EVAL=true + # -- the only round that measures accuracy) and measure_round + # (RUN_EVAL=false -- hot throughput only), then decides on the + # measure_round result, which by construction carries no accuracy. With + # enablement active the branch below used to fire first and dispatch a + # specialist to chase a quality regression that never happened, while + # the sibling warmup_round held a perfectly good score (gsm8k 0.8954 on + # Kimi-K3, 2026-07-29). Only a genuinely missing or below-floor accuracy + # should reach enablement. salvaged = self._salvage_sibling_baseline_accuracy(result, framework) if salvaged is not None: acc_val = float(salvaged["accuracy"]) @@ -1733,6 +1719,36 @@ def _maybe_stop_on_missing_baseline_accuracy( acc_val, salvaged.get("source_file", ""), ) + if not eval_enablement or accuracy_meets_floor(acc_val, floor): + return + # Salvaged, but still under the floor: that is a real quality + # signal, so fall through to enablement with the observed value. + acc = acc_val + + # No opt-out: a genuine baseline exists to establish the accuracy + # reference, so turning the eval off does not make a missing accuracy + # acceptable -- it only means the reference was never measured. Every + # disable path (the ``disable_run_eval`` param, an explicit + # ``RUN_EVAL=false`` env, a YAML/reference-env value) now lands here. + # Route into enablement instead of stopping: the throughput baseline + # stays for diagnostics but is blocked from anchoring. + if eval_enablement: + kind = classify_accuracy_failure(acc, floor) + observed = float(acc) if isinstance(acc, (int, float)) else None + evidence = ( + f"baseline accuracy did not meet floor: accuracy={acc} floor={floor} " + f"task={result.get('accuracy_task')} metric={result.get('accuracy_metric')} " + f"source={result.get('accuracy_source')}" + ) + self._stamp_eval_failure_contract( + ctx, result, kind=kind or "", observed_accuracy=observed, evidence=evidence + ) + log.warning( + "baseline_executor: accuracy %s below floor %.4f; routing to " + "enablement instead of stopping the run.", + acc, + floor, + ) return request_baseline_accuracy_stop( shared_state, From dbcd5bb6d901504e6444552682d86922a8a9694b Mon Sep 17 00:00:00 2001 From: ZhengGong-amd Date: Wed, 29 Jul 2026 11:36:17 +0000 Subject: [PATCH 20/32] fix(cli): read the real argparse dest for --framework-discover-timeout-sec The flag is declared as ``--framework-discover-timeout-sec``, so argparse stores it on ``args.framework_discover_timeout_sec``. The wiring read ``args.framework_agent_discover_timeout_sec`` instead, which never exists, so ``getattr`` fell through to its 0.0 default and the override was silently discarded -- passing the flag had no effect at all. Consequence: discovery always ran on DEFAULT_FA_PHASE_TIMEOUT_SEC (180s), which phases/framework.py then divides across the repo list. With six repos that is 30s each, matching the floor in _FRAMEWORK_MIN_PER_REPO_TIMEOUT_SEC, while one real ``fa phase-discover`` call (Primus Cortex query + LLM ranking) needs ~60s. On 2026-07-29 every GitHub repo timed out at exactly 30.0s and the phase ran on Primus Cortex results alone; re-running a failed request by hand returned rc=0 with 9 candidates, confirming a budget shortfall rather than a broken call. Read the parser's dest first and keep the longer name as a fallback for callers that set the attribute directly. Verified: --framework-discover-timeout-sec 900 now resolves to 900.0 (150s per repo across six repos), and omitting the flag still yields 0.0 so the 180s default path is unchanged. Co-Authored-By: Claude Opus 5 (1M context) --- src/hyperloom/inference_optimizer/cli/__init__.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/hyperloom/inference_optimizer/cli/__init__.py b/src/hyperloom/inference_optimizer/cli/__init__.py index e5f3a1b57..802864745 100644 --- a/src/hyperloom/inference_optimizer/cli/__init__.py +++ b/src/hyperloom/inference_optimizer/cli/__init__.py @@ -2047,9 +2047,16 @@ async def _run_optimize(args: argparse.Namespace) -> int: max_minutes=max_minutes_for_prompt, ) # ``fa phase-discover`` timeout override (falsy -> DEFAULT_FA_PHASE_TIMEOUT_SEC 180s). + # ``--framework-discover-timeout-sec`` lands on argparse dest + # ``framework_discover_timeout_sec``; reading only the ``framework_agent_`` + # spelling always missed it, so the flag silently did nothing and discovery + # stayed on the default budget. Accept the parser's dest first and keep the + # longer name as a fallback for callers that set it directly. try: coordinator.framework_agent_discover_timeout_sec = float( - getattr(args, "framework_agent_discover_timeout_sec", 0.0) or 0.0 + getattr(args, "framework_discover_timeout_sec", 0.0) + or getattr(args, "framework_agent_discover_timeout_sec", 0.0) + or 0.0 ) except (TypeError, ValueError): coordinator.framework_agent_discover_timeout_sec = 0.0 From 30bde3d6591c498b10df7a0101abc01df7b4c15f Mon Sep 17 00:00:00 2001 From: ZhengGong-amd Date: Thu, 30 Jul 2026 03:24:05 +0000 Subject: [PATCH 21/32] fix(kb): record the measured accuracy delta on framework outcomes Both framework KB writeback paths left accuracy_delta_pct at 0.0. integrate_patch looked the value up in the specialist payload, which never carries it, and framework_agent did not pass the field at all. Every lessons.jsonl row therefore claimed the candidate had no accuracy impact. The measured score was already in scope at all five call sites: _bench_patch returns it as gate_evidence["accuracy"], and the baseline is resolved a few lines above for the KEEP gate. Derive the delta from that pair and pass it through; the payload lookup stays as the fallback when a caller has nothing to supply. Observed on Kimi-K3 (vllm, mi355x, mxfp4): AITER_SITUV2_A8W4 dropped gsm8k from 0.9060 to 0.0008 and was reverted, yet the ledger recorded accuracy_delta_pct 0.0 alongside tps_delta_pct +5.07 -- reading as a purely throughput-driven revert. Co-Authored-By: Claude Opus 5 (1M context) --- .../actions/executors/framework_agent.py | 7 ++++ .../actions/executors/integrate_patch.py | 36 +++++++++++++++---- 2 files changed, 37 insertions(+), 6 deletions(-) diff --git a/src/hyperloom/orchestrator/actions/executors/framework_agent.py b/src/hyperloom/orchestrator/actions/executors/framework_agent.py index 7f8de6c19..1161af166 100644 --- a/src/hyperloom/orchestrator/actions/executors/framework_agent.py +++ b/src/hyperloom/orchestrator/actions/executors/framework_agent.py @@ -36,6 +36,7 @@ from .integrate_patch import ( DEFAULT_KEEP_THRESHOLD_PCT, DEFAULT_VARIANT_TIMEOUT_SEC, + _accuracy_delta_pct, _git_apply_collect_feedback, _git_stash_if_dirty, _with_stash_restore, @@ -873,6 +874,7 @@ async def __call__(self, ctx) -> dict[str, Any]: ) tput_ok = delta_pct is not None and delta_pct >= keep_threshold_pct gate_pass = tput_ok and not acc_block + acc_delta_pct = _accuracy_delta_pct(gate_evidence.get("accuracy"), params.get("accuracy_baseline")) if not gate_pass: reverted = self._revert_patches( @@ -898,6 +900,7 @@ async def __call__(self, ctx) -> dict[str, Any]: tps_delta_pct=float(delta_pct or 0.0), patch_path=str(applied[0]) if applied else "", extra=extra, + accuracy_delta_pct=acc_delta_pct, ) return _with_stash_restore( framework_root, @@ -964,6 +967,7 @@ async def __call__(self, ctx) -> dict[str, Any]: tps_delta_pct=float(delta_pct or 0.0), patch_path=str(applied[0]) if applied else "", extra=extra, + accuracy_delta_pct=acc_delta_pct, ) return _with_stash_restore( framework_root, @@ -997,6 +1001,7 @@ async def _write_kb_record( tps_delta_pct: float, patch_path: str, extra: dict[str, Any], + accuracy_delta_pct: float | None = None, ) -> None: """Append a FRAMEWORK outcome to ``lessons.jsonl`` so the next ``fa phase-discover`` can dedup integrated PRs. @@ -1011,6 +1016,7 @@ async def _write_kb_record( patch_path: Path to the applied patch (for provenance). extra: The runner ``extra`` mapping (provides the shared state / session id). + accuracy_delta_pct: Measured accuracy delta, when available. """ pr_url = str(candidate.get("pr_url") or "").strip() pr_sha = str(candidate.get("head_sha") or "").strip() @@ -1045,6 +1051,7 @@ async def _write_kb_record( precision=str(getattr(ss, "precision", "") if ss is not None else "").strip(), applicability=str(candidate.get("applicability") or "").strip(), provenance="framework_agent", + accuracy_delta_pct=float(accuracy_delta_pct or 0.0), changed_files=[str(f).strip() for f in changed_files if str(f).strip()], session_dir=self.session_dir, ) diff --git a/src/hyperloom/orchestrator/actions/executors/integrate_patch.py b/src/hyperloom/orchestrator/actions/executors/integrate_patch.py index da88b269c..e78731a6d 100644 --- a/src/hyperloom/orchestrator/actions/executors/integrate_patch.py +++ b/src/hyperloom/orchestrator/actions/executors/integrate_patch.py @@ -372,6 +372,22 @@ def _derive_lane(params: dict[str, Any]) -> str: return "perf_explore" +def _accuracy_delta_pct(measured: Any, baseline: Any) -> float | None: + """Percent accuracy change of ``measured`` against ``baseline``. + + Returns ``None`` when either side is missing or the baseline is not + positive, so callers fall back to their existing value. + """ + try: + m = float(measured) + b = float(baseline) + except (TypeError, ValueError): + return None + if b <= 0.0: + return None + return (m - b) / b * 100.0 + + def _preflight_missing_targets( framework_root: Path, patch_paths: list[Path], @@ -2629,6 +2645,7 @@ async def _gate_perf( specialist_task_id, ) gate_pass = delta_pct is not None and delta_pct >= keep_threshold_pct and not acc_block + acc_delta_pct = _accuracy_delta_pct(gate_evidence.get("accuracy"), acc_baseline) if not gate_pass: artifacts_reverted = self._revert_artifacts(applied_artifacts) @@ -2649,6 +2666,7 @@ async def _gate_perf( outcome="reverted_smoke_fail", tps_delta_pct=float(delta_pct or 0.0), extra=extra, + accuracy_delta_pct=acc_delta_pct, ) return _with_stash_restore( framework_root, @@ -2708,6 +2726,7 @@ async def _gate_perf( outcome="reverted_smoke_fail", tps_delta_pct=float(delta_pct or 0.0), extra=extra, + accuracy_delta_pct=acc_delta_pct, ) return _with_stash_restore( framework_root, @@ -2742,6 +2761,7 @@ async def _gate_perf( outcome="integrated", tps_delta_pct=float(delta_pct or 0.0), extra=extra, + accuracy_delta_pct=acc_delta_pct, ) # Commit the KEEP so a later REVERT checkout fallback can't wipe this # win (best-effort, non-fatal). Non-git roots (e.g. a pip-installed @@ -2863,6 +2883,7 @@ async def _maybe_write_framework_kb_record( outcome: str, tps_delta_pct: float, extra: dict[str, Any], + accuracy_delta_pct: float | None = None, ) -> None: """Append a JSONL record to ``lessons.jsonl`` when the patch came from the FRAMEWORK_AGENT phase. @@ -2877,6 +2898,8 @@ async def _maybe_write_framework_kb_record( tps_delta_pct: The measured throughput delta percentage. extra: The runner ``extra`` mapping (provides shared state / session id). + accuracy_delta_pct: Measured accuracy delta; overrides the payload + value when supplied. """ proposal = self._find_frameworkoposal(done_payload) if proposal is None: @@ -2905,12 +2928,13 @@ async def _maybe_write_framework_kb_record( changed_files = proposal.get("changed_files") or (done_payload or {}).get("changed_files") or [] if isinstance(changed_files, str): changed_files = [changed_files] - try: - accuracy_delta_pct = float( - proposal.get("accuracy_delta_pct") or (done_payload or {}).get("accuracy_delta_pct") or 0.0 - ) - except (TypeError, ValueError): - accuracy_delta_pct = 0.0 + if accuracy_delta_pct is None: + try: + accuracy_delta_pct = float( + proposal.get("accuracy_delta_pct") or (done_payload or {}).get("accuracy_delta_pct") or 0.0 + ) + except (TypeError, ValueError): + accuracy_delta_pct = 0.0 written = await write_framework_record( pr_url=pr_url, pr_sha=pr_sha, From e8cfc327688647567c7a40daedaf93850eec2a55 Mon Sep 17 00:00:00 2001 From: ZhengGong-amd Date: Thu, 30 Jul 2026 03:24:14 +0000 Subject: [PATCH 22/32] fix(framework): stop rewarding reverted and accuracy-losing candidates prior_score summed tps_delta_pct across every associated record regardless of outcome, so a candidate that was reverted for destroying accuracy still raised the prior of the next candidate matching its gap. Accuracy was not an input to the score at all. Count throughput only from records that were actually integrated, and scale the result down by the associated share that regressed accuracy. Genuine KEEPs are unaffected; a gap whose history is entirely accuracy failures now scores 0. Measured against the Kimi-K3 ledger: a reverted +5.07% record with a -99.9% accuracy delta scored 0.2887 for an unrelated new candidate and now scores 0.0, while an integrated +6.0% record still scores 0.755. Co-Authored-By: Claude Opus 5 (1M context) --- src/hyperloom/agents/framework/decision.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/hyperloom/agents/framework/decision.py b/src/hyperloom/agents/framework/decision.py index c1fe506c7..8c5263c1d 100644 --- a/src/hyperloom/agents/framework/decision.py +++ b/src/hyperloom/agents/framework/decision.py @@ -115,8 +115,24 @@ def prior_score( weights = [score for score, _ in associated] weight_sum = sum(weights) or 1.0 avg_association = sum(weights) / len(weights) - gain = sum(to_float(rec.get("tps_delta_pct"), default=0.0) * w for w, rec in associated) / weight_sum + + def _realized_gain(rec: dict[str, Any]) -> float: + """Throughput a record actually delivered, else 0. + + A reverted candidate's throughput was never kept, and an accuracy + regression disqualifies it outright, so neither may raise the prior. + """ + if str(rec.get("outcome") or "") != "integrated": + return 0.0 + if to_float(rec.get("accuracy_delta_pct"), default=0.0) < 0.0: + return 0.0 + return to_float(rec.get("tps_delta_pct"), default=0.0) + + gain = sum(_realized_gain(rec) * w for w, rec in associated) / weight_sum gain_score = max(0.0, min(1.0, gain / 20.0)) + accuracy_penalty = ( + sum(w for w, rec in associated if to_float(rec.get("accuracy_delta_pct"), default=0.0) < 0.0) / weight_sum + ) param_score = 0.0 if candidate_params: param_hits = 0.0 @@ -138,7 +154,7 @@ def _success_value(outcome: str) -> float: apply_score = sum(_success_value(str(rec.get("outcome") or "")) * w for w, rec in associated) / weight_sum quality = 0.45 * apply_score + 0.35 * gain_score + 0.20 * param_score - score = min(1.0, avg_association) * quality + score = min(1.0, avg_association) * quality * (1.0 - accuracy_penalty) return round(max(0.0, min(1.0, score)), 4) From 5800b1c577380182303efea5796fb0423f4a4dff Mon Sep 17 00:00:00 2001 From: ZhengGong-amd Date: Thu, 30 Jul 2026 03:26:02 +0000 Subject: [PATCH 23/32] fix(framework): dedup config-lever deliverables by content fingerprint A specialist may reduce an upstream PR to a set of server args and envs, and distinct PRs routinely reduce to the same set. The KB is keyed by pr_url and pr_sha, so those retries looked like new candidates and were benchmarked again. Record the canonical fingerprint of the applied args / envs in the record's applicability field, and check it before routing a config-lever deliverable to integrate_patch: a fingerprint whose ledger entry shows an accuracy regression is skipped. The gate reuses canonical_fingerprint, the same hash the explore ledger dedups on, so the two cannot drift. It is advisory -- any lookup error falls through to the existing dispatch path. Observed on Kimi-K3 (vllm, mi355x, mxfp4): ROCm/vllm PRs 970, 899 and 1066 each reduced to AITER_SITUV2_A8W4, and each spent roughly 95 minutes rediscovering that it drops gsm8k from 0.9060 to 0.0008. Co-Authored-By: Claude Opus 5 (1M context) --- .../actions/executors/integrate_patch.py | 18 ++++++++-- src/hyperloom/orchestrator/phases/explore.py | 33 +++++++++++++++++++ 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/src/hyperloom/orchestrator/actions/executors/integrate_patch.py b/src/hyperloom/orchestrator/actions/executors/integrate_patch.py index e78731a6d..83fc28e31 100644 --- a/src/hyperloom/orchestrator/actions/executors/integrate_patch.py +++ b/src/hyperloom/orchestrator/actions/executors/integrate_patch.py @@ -44,6 +44,7 @@ _revert_patches_no_git, _strip_path_prefix, ) +from ._canonical_fingerprint import canonical_fingerprint from ._grid_runner import ( GridVariant, VariantResult, @@ -2646,6 +2647,10 @@ async def _gate_perf( ) gate_pass = delta_pct is not None and delta_pct >= keep_threshold_pct and not acc_block acc_delta_pct = _accuracy_delta_pct(gate_evidence.get("accuracy"), acc_baseline) + cfg_fingerprint = canonical_fingerprint( + params.get("extra_server_args"), + params.get("extra_envs"), + ) if not gate_pass: artifacts_reverted = self._revert_artifacts(applied_artifacts) @@ -2667,6 +2672,7 @@ async def _gate_perf( tps_delta_pct=float(delta_pct or 0.0), extra=extra, accuracy_delta_pct=acc_delta_pct, + config_fingerprint=cfg_fingerprint, ) return _with_stash_restore( framework_root, @@ -2727,6 +2733,7 @@ async def _gate_perf( tps_delta_pct=float(delta_pct or 0.0), extra=extra, accuracy_delta_pct=acc_delta_pct, + config_fingerprint=cfg_fingerprint, ) return _with_stash_restore( framework_root, @@ -2762,6 +2769,7 @@ async def _gate_perf( tps_delta_pct=float(delta_pct or 0.0), extra=extra, accuracy_delta_pct=acc_delta_pct, + config_fingerprint=cfg_fingerprint, ) # Commit the KEEP so a later REVERT checkout fallback can't wipe this # win (best-effort, non-fatal). Non-git roots (e.g. a pip-installed @@ -2884,6 +2892,7 @@ async def _maybe_write_framework_kb_record( tps_delta_pct: float, extra: dict[str, Any], accuracy_delta_pct: float | None = None, + config_fingerprint: str = "", ) -> None: """Append a JSONL record to ``lessons.jsonl`` when the patch came from the FRAMEWORK_AGENT phase. @@ -2900,6 +2909,8 @@ async def _maybe_write_framework_kb_record( session id). accuracy_delta_pct: Measured accuracy delta; overrides the payload value when supplied. + config_fingerprint: Content fingerprint of the applied server + args / envs, recorded so a retried config can be recognised. """ proposal = self._find_frameworkoposal(done_payload) if proposal is None: @@ -2950,9 +2961,10 @@ async def _maybe_write_framework_kb_record( model_class=str(getattr(shared_state, "model_class", "") if shared_state is not None else "").strip(), gpu_type=str(getattr(shared_state, "gpu_type", "") if shared_state is not None else "").strip(), precision=str(getattr(shared_state, "precision", "") if shared_state is not None else "").strip(), - applicability=str( - proposal.get("applicability") or (done_payload or {}).get("applicability") or "" - ).strip(), + applicability=( + config_fingerprint + or str(proposal.get("applicability") or (done_payload or {}).get("applicability") or "").strip() + ), provenance=str(proposal.get("provenance") or (done_payload or {}).get("provenance") or "").strip(), accuracy_delta_pct=accuracy_delta_pct, changed_files=[str(f).strip() for f in changed_files if str(f).strip()], diff --git a/src/hyperloom/orchestrator/phases/explore.py b/src/hyperloom/orchestrator/phases/explore.py index 2b09c5cf3..a05b2203d 100644 --- a/src/hyperloom/orchestrator/phases/explore.py +++ b/src/hyperloom/orchestrator/phases/explore.py @@ -13,6 +13,7 @@ from pathlib import Path from typing import Any from . import machine_state as _phase_state +from hyperloom.common.coerce import to_float from ..loop.coordinator_helpers import _parse_iso_unix from hyperloom.inference_optimizer.protocol.intent import Intent, IntentType from ..bus.message_bus import Message @@ -1606,6 +1607,8 @@ async def _maybe_autosubmit_framework_config( sid = str(task.task_id or "").strip() if not sid: return + if config_levers and self._config_lever_known_bad(config_levers): + return # Already ruled on (e.g. after resume) — nothing to do. try: if self.shared_state.get_specialist_patch_verdict(sid): @@ -1722,6 +1725,36 @@ async def _maybe_autosubmit_framework_config( sid, ) + def _config_lever_known_bad(self, config_levers: dict[str, Any]) -> bool: + """True when this exact config already lost an accuracy gate. + + Different upstream PRs often reduce to the same server args / envs, so + the ledger is keyed by content fingerprint rather than by PR. + """ + from ..actions.executors._canonical_fingerprint import canonical_fingerprint + + try: + from hyperloom.agents.framework.kb import read_pr_ledger + + fingerprint = canonical_fingerprint( + config_levers.get("extra_server_args"), + config_levers.get("extra_envs"), + ) + for rec in read_pr_ledger(): + if str(rec.get("applicability") or "") != fingerprint: + continue + if to_float(rec.get("accuracy_delta_pct"), default=0.0) < 0.0: + log.info( + "FRAMEWORK: skipping config lever %s — accuracy %.2f%% on %s", + fingerprint, + to_float(rec.get("accuracy_delta_pct"), default=0.0), + rec.get("pr_url") or "a prior candidate", + ) + return True + except Exception: # noqa: BLE001 — advisory gate must never block dispatch + log.debug("FRAMEWORK: config-lever ledger check failed", exc_info=True) + return False + def _build_specialist_round_entry( self, *, From 81bb41a232586073c8b0ca2c42026c426074f8f2 Mon Sep 17 00:00:00 2001 From: ZhengGong-amd Date: Thu, 30 Jul 2026 03:45:55 +0000 Subject: [PATCH 24/32] fix(inference-opt): raise enablement accuracy floor default and judge eval origin by candidate's own task/metric The enablement KEEP gate defaulted its accuracy floor to 0.0, so it degenerated to `accuracy > 0` and once KEPT a candidate scoring gsm8k=0.00076 (0.08% of a 0.906 baseline). Raise the default floor to 0.05 so collapsed-output models are rejected when no operator override is set. Also remove the eval-contract fingerprint drift veto. The stored fingerprint could be poisoned by an eval-less re-baseline, causing every later candidate to be reverted without ever reading its accuracy. Instead, require an eval-origin score to carry the task and metric stamped by the candidate's own run; a bare number with no provenance fails closed. Update tests and writeback to match. --- .../tests/test_accuracy_gate_units.py | 13 +++++- .../tests/test_baseline_eval_fallback.py | 3 +- .../tests/test_integrate_patch_executor.py | 41 ++++++++++++------- .../actions/executors/_accuracy_gate.py | 9 +++- .../actions/executors/integrate_patch.py | 30 +++++++------- src/hyperloom/orchestrator/loop/writeback.py | 1 - 6 files changed, 61 insertions(+), 36 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_accuracy_gate_units.py b/src/hyperloom/inference_optimizer/tests/test_accuracy_gate_units.py index 373636920..93be5b173 100644 --- a/src/hyperloom/inference_optimizer/tests/test_accuracy_gate_units.py +++ b/src/hyperloom/inference_optimizer/tests/test_accuracy_gate_units.py @@ -125,7 +125,16 @@ def test_on_eval_fail_disabled(self, monkeypatch): def test_floor_default(self, monkeypatch): monkeypatch.delenv("INFERENCE_OPTIMIZER_ENABLEMENT_ACCURACY_FLOOR", raising=False) - assert ag.enablement_accuracy_floor() == 0.0 + assert ag.enablement_accuracy_floor() == ag.DEFAULT_ENABLEMENT_ACCURACY_FLOOR + + def test_floor_default_rejects_a_collapsed_model(self): + """The default must be strong enough to be the only correctness authority. + + A run once KEPT a candidate scoring gsm8k=0.00076 because the floor was + 0.0 and the gate degenerated to ``accuracy > 0``. + """ + assert ag.DEFAULT_ENABLEMENT_ACCURACY_FLOOR > 0.0 + assert not ag.accuracy_meets_floor(0.00076, ag.DEFAULT_ENABLEMENT_ACCURACY_FLOOR) def test_floor_valid(self, monkeypatch): monkeypatch.setenv("INFERENCE_OPTIMIZER_ENABLEMENT_ACCURACY_FLOOR", "0.7") @@ -134,7 +143,7 @@ def test_floor_valid(self, monkeypatch): @pytest.mark.parametrize("bad", ["1.5", "-0.1", "nonsense", "nan", "inf"]) def test_floor_invalid_or_out_of_range_falls_back(self, monkeypatch, bad): monkeypatch.setenv("INFERENCE_OPTIMIZER_ENABLEMENT_ACCURACY_FLOOR", bad) - assert ag.enablement_accuracy_floor() == 0.0 + assert ag.enablement_accuracy_floor() == ag.DEFAULT_ENABLEMENT_ACCURACY_FLOOR class TestAccuracyValidator: diff --git a/src/hyperloom/inference_optimizer/tests/test_baseline_eval_fallback.py b/src/hyperloom/inference_optimizer/tests/test_baseline_eval_fallback.py index 70a9fa642..fbd32b684 100644 --- a/src/hyperloom/inference_optimizer/tests/test_baseline_eval_fallback.py +++ b/src/hyperloom/inference_optimizer/tests/test_baseline_eval_fallback.py @@ -737,6 +737,7 @@ def fake_run(cmd, *args, **kwargs): # --- eval-origin enablement routing (flag on) ------------------------------ from hyperloom.orchestrator.actions.executors._accuracy_gate import ( # noqa: E402 BASELINE_EVAL_ACCURACY_FLOOR_KEY, + DEFAULT_ENABLEMENT_ACCURACY_FLOOR, BASELINE_EVAL_CONTRACT_FINGERPRINT_KEY, BASELINE_EVAL_EVIDENCE_KEY, BASELINE_EVAL_FAILED_KEY, @@ -785,7 +786,7 @@ def test_eval_enablement_missing_accuracy_routes_not_stop(monkeypatch, tmp_path) assert result[BASELINE_EVAL_FAILED_KEY] is True assert result[BASELINE_EVAL_FAILURE_KIND_KEY] == EVAL_KIND_ACCURACY_UNAVAILABLE assert result[BASELINE_EVAL_OBSERVED_ACCURACY_KEY] is None - assert result[BASELINE_EVAL_ACCURACY_FLOOR_KEY] == 0.0 + assert result[BASELINE_EVAL_ACCURACY_FLOOR_KEY] == DEFAULT_ENABLEMENT_ACCURACY_FLOOR assert result[BASELINE_EVAL_EVIDENCE_KEY] assert result[BASELINE_EVAL_CONTRACT_FINGERPRINT_KEY] assert result["eval_origin"] == "eval" diff --git a/src/hyperloom/inference_optimizer/tests/test_integrate_patch_executor.py b/src/hyperloom/inference_optimizer/tests/test_integrate_patch_executor.py index dfc585969..19dc2294c 100644 --- a/src/hyperloom/inference_optimizer/tests/test_integrate_patch_executor.py +++ b/src/hyperloom/inference_optimizer/tests/test_integrate_patch_executor.py @@ -699,8 +699,8 @@ async def _run_enablement_integrate( before_signature=None, enablement_origin: str = "", accuracy_floor=None, - expected_fingerprint: str = "", - candidate_fingerprint: str = "", + accuracy_task: str = "gsm8k", + accuracy_metric: str = "exact_match", ): session_dir = tmp_path / "session" session_dir.mkdir() @@ -718,9 +718,8 @@ async def _fake_bench(**_kwargs): return bench_result, { "accuracy_pass": None, "enablement_accuracy": enablement_accuracy, - "enablement_accuracy_task": "gsm8k", - "enablement_accuracy_metric": "exact_match", - "enablement_eval_contract_fingerprint": candidate_fingerprint, + "enablement_accuracy_task": accuracy_task, + "enablement_accuracy_metric": accuracy_metric, "timed_out": False, } @@ -742,8 +741,6 @@ async def _noop_kb(**_kwargs): params["enablement_origin"] = enablement_origin if accuracy_floor is not None: params["enablement_accuracy_floor"] = accuracy_floor - if expected_fingerprint: - params["enablement_eval_contract_fingerprint"] = expected_fingerprint ctx = _make_ctx("t-int-en", params) return await executor(ctx), repo @@ -825,33 +822,47 @@ async def test_enablement_eval_origin_keeps_at_or_above_floor(tmp_path: Path, mo @pytest.mark.asyncio -async def test_enablement_eval_origin_reverts_on_fingerprint_drift(tmp_path: Path, monkeypatch): +async def test_enablement_eval_origin_reverts_when_accuracy_has_no_task_or_metric( + tmp_path: Path, monkeypatch +): + """A score with no task/metric did not come from a real eval. + + The candidate's own run is the only correctness authority; a bare number + with no provenance must not clear the gate. + """ result, _ = await _run_enablement_integrate( tmp_path, monkeypatch, booted=True, enablement_accuracy=0.9, enablement_origin="eval", - expected_fingerprint="fp_baseline", - candidate_fingerprint="fp_changed", + accuracy_task="", + accuracy_metric="", ) assert result["status"] == "reverted" - assert result["enablement_eval_contract_drift"] is True + assert result["correctness_verified"] is False @pytest.mark.asyncio -async def test_enablement_eval_origin_keeps_when_fingerprint_matches(tmp_path: Path, monkeypatch): +async def test_enablement_eval_origin_keeps_a_measured_accuracy(tmp_path: Path, monkeypatch): + """A measured, above-floor accuracy is KEPT. + + Regression for the burned Kimi-Linear run: an unrelated eval-less + re-baseline used to poison the stored eval-contract fingerprint, which + vetoed every later candidate without ever reading its accuracy. Nothing + outside this candidate's own run may decide its correctness. + """ result, _ = await _run_enablement_integrate( tmp_path, monkeypatch, booted=True, enablement_accuracy=0.9, enablement_origin="eval", - expected_fingerprint="fp_same", - candidate_fingerprint="fp_same", + accuracy_task="gsm8k", + accuracy_metric="exact_match,strict-match", ) assert result["status"] == "kept" - assert result["enablement_eval_contract_drift"] is False + assert result["correctness_verified"] is True @pytest.mark.asyncio diff --git a/src/hyperloom/orchestrator/actions/executors/_accuracy_gate.py b/src/hyperloom/orchestrator/actions/executors/_accuracy_gate.py index 7d7e72492..a14dc96a3 100644 --- a/src/hyperloom/orchestrator/actions/executors/_accuracy_gate.py +++ b/src/hyperloom/orchestrator/actions/executors/_accuracy_gate.py @@ -30,9 +30,16 @@ # Enablement-on-eval-fail switch and shared accuracy floor. The floor is used by # BOTH the baseline eval-failure trigger and the enablement KEEP gate so the two # never diverge. +# +# The default is non-zero because the floor is the ONLY correctness authority on +# the enablement KEEP path. At 0.0 the gate degenerates to ``accuracy > 0``, which +# admits a model that is answering essentially nothing: a real run KEPT a +# candidate scoring gsm8k=0.00076 (0.08% of a 0.906 baseline) as "correct". +# 0.05 is a floor of last resort -- it rejects the collapsed-output regime +# without judging genuine quality, which belongs to the operator override below. ENABLEMENT_ON_EVAL_FAIL_ENV = "INFERENCE_OPTIMIZER_ENABLEMENT_ON_EVAL_FAIL" ENABLEMENT_ACCURACY_FLOOR_ENV = "INFERENCE_OPTIMIZER_ENABLEMENT_ACCURACY_FLOOR" -DEFAULT_ENABLEMENT_ACCURACY_FLOOR = 0.0 +DEFAULT_ENABLEMENT_ACCURACY_FLOOR = 0.05 # Result-dict keys stamped by the baseline executor on an eval-rooted failure and # read by writeback promotion/persistence. diff --git a/src/hyperloom/orchestrator/actions/executors/integrate_patch.py b/src/hyperloom/orchestrator/actions/executors/integrate_patch.py index 83fc28e31..07ea3eb8b 100644 --- a/src/hyperloom/orchestrator/actions/executors/integrate_patch.py +++ b/src/hyperloom/orchestrator/actions/executors/integrate_patch.py @@ -2359,17 +2359,21 @@ def _gc_on_revert() -> None: else: # Present but below floor / non-positive / non-finite. correctness_ok = False - # eval-origin only: a changed eval contract (dataset/task/metric/config) - # means the score is not comparable — fail closed. - expected_fp = str(params.get("enablement_eval_contract_fingerprint") or "") - candidate_fp = str(gate_evidence.get("enablement_eval_contract_fingerprint") or "") - fp_drift = eval_origin and bool(expected_fp) and bool(candidate_fp) and expected_fp != candidate_fp - if fp_drift: + # eval-origin only: a score with no task/metric did not come from a real + # eval, so it cannot clear the gate. This reads the candidate's OWN run + # (both keys are stamped beside the accuracy it is judging), unlike the + # contract fingerprint it replaces: RUN_EVAL is itself a hashed contract + # field, so an eval-less re-baseline could poison the stored digest and + # veto every later candidate without ever consulting its accuracy. + if ( + eval_origin + and correctness_ok + and not (gate_evidence.get("enablement_accuracy_task") and gate_evidence.get("enablement_accuracy_metric")) + ): correctness_ok = False log.warning( - "integrate_patch: eval-contract fingerprint drift (expected %s, got %s); reverting", - expected_fp, - candidate_fp, + "integrate_patch: eval-origin accuracy %s carries no task/metric; reverting", + enablement_accuracy, ) eval_provenance = { "enablement_origin": str(params.get("enablement_origin") or ""), @@ -2377,8 +2381,6 @@ def _gc_on_revert() -> None: "enablement_accuracy_floor": floor, "accuracy_task": gate_evidence.get("enablement_accuracy_task") or "", "accuracy_metric": gate_evidence.get("enablement_accuracy_metric") or "", - "enablement_eval_contract_fingerprint": candidate_fp, - "enablement_eval_contract_drift": fp_drift, "enablement_eval_failure_kind": accuracy_kind or "", } @@ -2464,11 +2466,7 @@ def _gc_on_revert() -> None: "enablement": True, "runnable": False, "correctness_verified": correctness_ok is True, - "reason": ( - "enablement eval-contract drift; reverted" - if fp_drift - else f"enablement not runnable: {run_reason}" - ), + "reason": f"enablement not runnable: {run_reason}", "bench_result": bench_result, "workspace": str(output_root), **eval_provenance, diff --git a/src/hyperloom/orchestrator/loop/writeback.py b/src/hyperloom/orchestrator/loop/writeback.py index 4eeb813c8..9050bb4e0 100644 --- a/src/hyperloom/orchestrator/loop/writeback.py +++ b/src/hyperloom/orchestrator/loop/writeback.py @@ -3031,7 +3031,6 @@ async def _promote_integrate_patch( "correctness_verified": result.get("correctness_verified"), "enablement_eval_failure_kind": result.get("enablement_eval_failure_kind"), "enablement_observed_accuracy": result.get("enablement_observed_accuracy"), - "enablement_eval_contract_drift": result.get("enablement_eval_contract_drift"), "provisional": result.get("provisional"), } outcome.changed = changed From 54232ad91af9f33bc12b0b6ab79a9768b8ed3f55 Mon Sep 17 00:00:00 2001 From: ZhengGong-amd Date: Thu, 30 Jul 2026 06:30:11 +0000 Subject: [PATCH 25/32] refactor(enablement): drop the candidate-side eval-contract fingerprint Removing the fingerprint drift veto left its producer and carrier chain with no consumer. The candidate-side digest is computed on every enablement round, threaded through gate_evidence, and read by nothing; the two carrier hops that forwarded the trigger's digest into the integrate task are likewise dead -- correctness is now judged from the task/metric the candidate's own run stamps beside its accuracy. Delete the producer, the gate_evidence key and both carrier hops. The nested enablement/succeeded conditions collapse into one now that the digest no longer has to be computed regardless of bench outcome. The trigger-side chain stays: baseline stamps the digest, writeback persists it and the breakdown collector reads it, so session_breakdown.json keeps emitting eval_contract_fingerprint unchanged. It is diagnostic-only now -- no code path gates on it. Verified in a clean worktree at eae18d5ae: same 8212 passed, and the 13 pre-existing environment-dependent failures are identical before and after. Co-Authored-By: Claude Opus 5 (1M context) --- ...test_enablement_coordinator_wiring_unit.py | 4 +- .../tests/test_framework_local_explore.py | 4 +- .../actions/executors/integrate_patch.py | 38 +++++++------------ src/hyperloom/orchestrator/phases/explore.py | 3 -- .../orchestrator/phases/framework.py | 3 -- 5 files changed, 19 insertions(+), 33 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_enablement_coordinator_wiring_unit.py b/src/hyperloom/inference_optimizer/tests/test_enablement_coordinator_wiring_unit.py index f14985439..43cbbf782 100644 --- a/src/hyperloom/inference_optimizer/tests/test_enablement_coordinator_wiring_unit.py +++ b/src/hyperloom/inference_optimizer/tests/test_enablement_coordinator_wiring_unit.py @@ -111,7 +111,9 @@ def test_build_params_threads_eval_origin_carriers(monkeypatch): assert params["enablement_origin"] == "eval" assert params["enablement_accuracy_floor"] == 0.3 assert params["enablement_probe_config_path"] == "/runs/baseline/materialized.yaml" - assert params["enablement_eval_contract_fingerprint"] == "abc123" + # The eval-contract fingerprint is no longer carried: nothing downstream + # reads it. Correctness is judged from the candidate's own measurement. + assert "enablement_eval_contract_fingerprint" not in params _TRANSFORMERS_UNRECOGNIZED_LOG = ( diff --git a/src/hyperloom/inference_optimizer/tests/test_framework_local_explore.py b/src/hyperloom/inference_optimizer/tests/test_framework_local_explore.py index 894696aeb..4eefebebf 100644 --- a/src/hyperloom/inference_optimizer/tests/test_framework_local_explore.py +++ b/src/hyperloom/inference_optimizer/tests/test_framework_local_explore.py @@ -374,7 +374,9 @@ def test_forward_enablement_carriers_eval_origin(): assert dst["enablement_origin"] == "eval" assert dst["enablement_accuracy_floor"] == 0.4 assert dst["enablement_probe_config_path"] == "/runs/baseline/materialized.yaml" - assert dst["enablement_eval_contract_fingerprint"] == "fp1" + # The eval-contract fingerprint is no longer forwarded: nothing downstream + # reads it. Correctness is judged from the candidate's own measurement. + assert "enablement_eval_contract_fingerprint" not in dst # Benches against the original workload config, not the shipped default. assert dst["config_path"] == "/runs/baseline/materialized.yaml" diff --git a/src/hyperloom/orchestrator/actions/executors/integrate_patch.py b/src/hyperloom/orchestrator/actions/executors/integrate_patch.py index 07ea3eb8b..4da4f3533 100644 --- a/src/hyperloom/orchestrator/actions/executors/integrate_patch.py +++ b/src/hyperloom/orchestrator/actions/executors/integrate_patch.py @@ -30,7 +30,6 @@ accuracy_passed, classify_accuracy_failure, enablement_accuracy_floor, - eval_contract_fingerprint, parse_eval_results, ) from ._apply_feedback import ApplyFeedback, build_apply_feedback @@ -3245,36 +3244,25 @@ async def _bench_patch( enablement_accuracy: float | None = None enablement_accuracy_task = "" enablement_accuracy_metric = "" - candidate_fingerprint = "" - if bool(params.get("enablement")): - # Compute the candidate fingerprint from the materialized config - # regardless of bench outcome — eval crash must still produce a - # comparable fingerprint. - candidate_fingerprint = eval_contract_fingerprint( - config_path=config_path, - framework=params.get("framework") or os.environ.get("FRAMEWORK") or None, - model=resolved_model, - ) - if bench.get("status") == "succeeded": - try: - eval_results = parse_eval_results( - eval_search_root, - framework=params.get("framework") or os.environ.get("FRAMEWORK") or None, - ) - acc = eval_results.get("accuracy") - if isinstance(acc, (int, float)): - enablement_accuracy = float(acc) - enablement_accuracy_task = str(eval_results.get("task") or "") - enablement_accuracy_metric = str(eval_results.get("metric") or "") - except Exception: # noqa: BLE001 — eval may not produce a result - log.debug("integrate_patch: enablement eval parse failed", exc_info=True) + if bool(params.get("enablement")) and bench.get("status") == "succeeded": + try: + eval_results = parse_eval_results( + eval_search_root, + framework=params.get("framework") or os.environ.get("FRAMEWORK") or None, + ) + acc = eval_results.get("accuracy") + if isinstance(acc, (int, float)): + enablement_accuracy = float(acc) + enablement_accuracy_task = str(eval_results.get("task") or "") + enablement_accuracy_metric = str(eval_results.get("metric") or "") + except Exception: # noqa: BLE001 — eval may not produce a result + log.debug("integrate_patch: enablement eval parse failed", exc_info=True) return bench, { "accuracy_pass": accuracy_pass, "enablement_accuracy": enablement_accuracy, "enablement_accuracy_task": enablement_accuracy_task, "enablement_accuracy_metric": enablement_accuracy_metric, - "enablement_eval_contract_fingerprint": candidate_fingerprint, } @staticmethod diff --git a/src/hyperloom/orchestrator/phases/explore.py b/src/hyperloom/orchestrator/phases/explore.py index a05b2203d..62557709c 100644 --- a/src/hyperloom/orchestrator/phases/explore.py +++ b/src/hyperloom/orchestrator/phases/explore.py @@ -41,9 +41,6 @@ def _forward_enablement_carriers(src: dict[str, Any], dst: dict[str, Any]) -> No return dst["enablement_origin"] = origin dst["enablement_accuracy_floor"] = float(src.get("enablement_accuracy_floor") or 0.0) - fp = str(src.get("enablement_eval_contract_fingerprint") or "") - if fp: - dst["enablement_eval_contract_fingerprint"] = fp cfg = str(src.get("enablement_probe_config_path") or "") if cfg: dst["enablement_probe_config_path"] = cfg diff --git a/src/hyperloom/orchestrator/phases/framework.py b/src/hyperloom/orchestrator/phases/framework.py index 5e67546b0..7113c4fe2 100644 --- a/src/hyperloom/orchestrator/phases/framework.py +++ b/src/hyperloom/orchestrator/phases/framework.py @@ -132,9 +132,6 @@ def _enablement_carrier_params(state: Any) -> dict[str, Any]: cfg = str(getattr(state, "enablement_probe_config_path", "") or "") if cfg: out["enablement_probe_config_path"] = cfg - fp = str(getattr(state, "enablement_eval_contract_fingerprint", "") or "") - if fp: - out["enablement_eval_contract_fingerprint"] = fp return out From a43d97c6440049d86a82e9d0ce69763b57879462 Mon Sep 17 00:00:00 2001 From: ZhengGong-amd Date: Thu, 30 Jul 2026 06:58:33 +0000 Subject: [PATCH 26/32] fix(enablement): treat an already-applied patch as a no-op, not apply_failed A specialist routinely writes both a superset patch and the subset it contains -- e.g. an FLA state-layout revert, plus that same revert bundled with the config fix that is the other half of the same enablement. Whichever lands first makes the other's hunks a no-op, and both POSIX `patch` and `git apply --check` report that with a non-zero exit indistinguishable from "does not apply". The executor read it as a hard failure, aborted the combo and reverted a fix that was in fact fully and correctly applied. That cost a real 7-hour run: the Kimi-Linear-48B enablement specialist had root-caused the vLLM 0.20.0 KDA breakage and validated the two-part fix at gsm8k 0.875 (from 0.0008), and it was thrown away at apply time. The lost round pushed the stall streak to its cap and the session stopped at `enablement_stalled` with baseline_tput=0. Resolve the ambiguity with a reverse dry-run, which succeeds only when every hunk's post-state is already in the tree -- exactly what a forward apply would have produced. Fully satisfied => success with no backups (the patch that made the edits owns the backups a revert needs). Partial overlap and non-applying patches keep failing with reauthor feedback, so the narrow no-op cannot swallow a real edit. Also close stdin on the `patch` invocations: it prompts "Assume -R? [n]" on an already-applied hunk and could otherwise block until the 60s timeout. The three failures in the touched trees (artifact stash, isolated venv, specialist gpu gate) are pre-existing and environment-dependent -- identical before and after this change. Co-Authored-By: Claude Opus 5 (1M context) --- .../tests/test_patch_already_applied_unit.py | 199 ++++++++++++++++++ .../actions/executors/_nogit_patch.py | 66 ++++++ .../actions/executors/integrate_patch.py | 42 ++++ 3 files changed, 307 insertions(+) create mode 100644 src/hyperloom/inference_optimizer/tests/test_patch_already_applied_unit.py diff --git a/src/hyperloom/inference_optimizer/tests/test_patch_already_applied_unit.py b/src/hyperloom/inference_optimizer/tests/test_patch_already_applied_unit.py new file mode 100644 index 000000000..4606f2b4f --- /dev/null +++ b/src/hyperloom/inference_optimizer/tests/test_patch_already_applied_unit.py @@ -0,0 +1,199 @@ +# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Already-applied patches must be a satisfied no-op, not an apply failure. + +A specialist commonly writes both a superset patch and the subset it contains +(e.g. an FLA state-layout revert, plus that same revert bundled with a config +fix). Whichever lands first makes the other's hunks a no-op, and both POSIX +``patch`` and ``git apply --check`` reject that with a non-zero exit that looks +exactly like "does not apply". Treating it as a hard failure aborted the whole +enablement combo and reverted a correctly-applied fix. + +Both apply channels resolve the ambiguity with a *reverse* dry-run, which +succeeds only when every hunk's post-state is already present. These tests pin +the three outcomes that matter: no-op on full overlap, real failure on no +overlap, and real failure on *partial* overlap (where a no-op would be wrong). +""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from hyperloom.orchestrator.actions.executors import integrate_patch as ip +from hyperloom.orchestrator.actions.executors._nogit_patch import ( + _apply_patch_no_git, + _reverse_applies_cleanly, +) + + +_SUBSET = """\ +--- a/mod/alpha.py ++++ b/mod/alpha.py +@@ -1,3 +1,3 @@ + header +-old_alpha ++new_alpha + footer +""" + +# Superset: the subset's hunk plus a second file the subset never touches. +_SUPERSET = _SUBSET + """\ +--- a/mod/beta.py ++++ b/mod/beta.py +@@ -1,3 +1,3 @@ + header +-old_beta ++new_beta + footer +""" + +_UNRELATED = """\ +--- a/mod/alpha.py ++++ b/mod/alpha.py +@@ -1,3 +1,3 @@ + header +-nothing_matches_this ++replacement + footer +""" + + +def _tree(root: Path) -> None: + """Materialize the two-file source tree both patches target.""" + (root / "mod").mkdir(parents=True) + (root / "mod" / "alpha.py").write_text("header\nold_alpha\nfooter\n", encoding="utf-8") + (root / "mod" / "beta.py").write_text("header\nold_beta\nfooter\n", encoding="utf-8") + + +def _patch(root: Path, name: str, body: str) -> Path: + """Write ``body`` as a patch file under ``root`` and return its path.""" + p = root / name + p.write_text(body, encoding="utf-8") + return p + + +@pytest.fixture +def workspace(tmp_path): + """A source tree, a patch dir and a backup dir, all under tmp_path.""" + src = tmp_path / "src" + _tree(src) + patches = tmp_path / "patches" + patches.mkdir() + return src, patches, tmp_path / "backups" + + +def test_nogit_superset_then_subset_is_a_noop(workspace): + """Subset after superset succeeds with no backups; both edits survive.""" + src, patches, backups = workspace + sup = _patch(patches, "superset.patch", _SUPERSET) + sub = _patch(patches, "subset.patch", _SUBSET) + + ok, err, recs, fb = _apply_patch_no_git(src, sup, backups, seq_offset=0) + assert ok, err + assert recs, "superset should have backed up the files it changed" + + ok2, err2, recs2, fb2 = _apply_patch_no_git(src, sub, backups, seq_offset=len(recs)) + assert ok2, err2 + assert fb2 is None + # The no-op owns no backups — the patch that made the edits owns them, so a + # revert restores the tree exactly once. + assert recs2 == [] + + assert "new_alpha" in (src / "mod" / "alpha.py").read_text(encoding="utf-8") + assert "new_beta" in (src / "mod" / "beta.py").read_text(encoding="utf-8") + + +def test_nogit_subset_then_superset_fails_closed(workspace): + """The reverse order is only *partly* satisfied, so it must fail, not no-op. + + Applying the subset first leaves the superset's extra ``beta`` hunk still + outstanding. Reporting a no-op there would silently drop a real edit, so the + reverse probe rejects it and the apply fails closed — the executor reverts + and hands the specialist ``retry_feedback`` to reauthor from. The narrow + no-op is deliberately limited to a *fully* satisfied patch. + """ + src, patches, backups = workspace + sup = _patch(patches, "superset.patch", _SUPERSET) + sub = _patch(patches, "subset.patch", _SUBSET) + + ok, _, recs, _ = _apply_patch_no_git(src, sub, backups, seq_offset=0) + assert ok + ok2, err2, recs2, fb2 = _apply_patch_no_git(src, sup, backups, seq_offset=len(recs)) + assert not ok2 + assert "failed at all strip levels" in err2 + assert fb2 is not None, "a real failure must carry reauthor feedback" + # Nothing was half-applied: beta is untouched. + assert "old_beta" in (src / "mod" / "beta.py").read_text(encoding="utf-8") + + +def test_nogit_partial_overlap_still_fails(workspace): + """A patch only *partly* present is a real failure, not a no-op. + + The superset's beta hunk has not been applied, so silently reporting success + would drop a real edit. + """ + src, patches, backups = workspace + sup = _patch(patches, "superset.patch", _SUPERSET) + ok, _, recs, _ = _apply_patch_no_git(src, sup, backups, seq_offset=0) + assert ok + # Undo only beta, leaving alpha applied -> superset is half-present. + (src / "mod" / "beta.py").write_text("header\nold_beta\nfooter\n", encoding="utf-8") + + assert not _reverse_applies_cleanly(src, sup) + + +def test_nogit_unrelated_patch_still_fails(workspace): + """A patch that genuinely does not apply keeps failing with feedback.""" + src, patches, backups = workspace + bad = _patch(patches, "bad.patch", _UNRELATED) + + ok, err, recs, fb = _apply_patch_no_git(src, bad, backups, seq_offset=0) + assert not ok + assert "failed at all strip levels" in err + assert fb is not None + assert recs == [] + + +def _git_tree(root: Path) -> None: + """Initialise ``root`` as a committed git work tree.""" + subprocess.run(["git", "init", "-q", str(root)], check=True) + for k, v in (("user.email", "t@t"), ("user.name", "t")): + subprocess.run(["git", "-C", str(root), "config", k, v], check=True) + subprocess.run(["git", "-C", str(root), "add", "-A"], check=True) + subprocess.run(["git", "-C", str(root), "commit", "-qm", "init"], check=True) + + +def test_git_channel_superset_then_subset_is_a_noop(tmp_path): + """The git channel resolves the same ambiguity via ``git apply -R --check``.""" + src = tmp_path / "src" + _tree(src) + _git_tree(src) + patches = tmp_path / "patches" + patches.mkdir() + sup = _patch(patches, "superset.patch", _SUPERSET) + sub = _patch(patches, "subset.patch", _SUBSET) + + ok, err, _ = ip._git_apply_collect_feedback(src, sup) + assert ok, err + ok2, err2, fb2 = ip._git_apply_collect_feedback(src, sub) + assert ok2, err2 + assert fb2 is None + assert "new_alpha" in (src / "mod" / "alpha.py").read_text(encoding="utf-8") + + +def test_git_channel_unrelated_patch_still_fails(tmp_path): + """A non-applying patch still fails on the git channel, with feedback.""" + src = tmp_path / "src" + _tree(src) + _git_tree(src) + patches = tmp_path / "patches" + patches.mkdir() + bad = _patch(patches, "bad.patch", _UNRELATED) + + ok, err, fb = ip._git_apply_collect_feedback(src, bad) + assert not ok + assert fb is not None diff --git a/src/hyperloom/orchestrator/actions/executors/_nogit_patch.py b/src/hyperloom/orchestrator/actions/executors/_nogit_patch.py index 35c28ebd3..a6d5e7cce 100644 --- a/src/hyperloom/orchestrator/actions/executors/_nogit_patch.py +++ b/src/hyperloom/orchestrator/actions/executors/_nogit_patch.py @@ -111,6 +111,45 @@ def _is_git_tree(path: Path) -> bool: return False +def _reverse_applies_cleanly(framework_root: Path, patch_path: Path) -> bool: + """True when ``patch_path`` is already fully applied in ``framework_root``. + + A reverse dry-run (``patch -R --dry-run``) succeeds only when every hunk's + *post*-state is already present in the tree — i.e. the tree is exactly what + a successful forward apply would have produced. That makes it the reliable + already-applied probe: POSIX ``patch`` exits non-zero for both "does not + apply" and "previously applied", so the forward exit code alone cannot tell + the two apart. + + Strictly a probe: ``--dry-run`` is passed at every level, so the tree is + never mutated. Partial overlap is correctly rejected — a patch whose hunks + are only *partly* present fails the reverse check and stays a real failure. + + Args: + framework_root: The source-tree root the patch targets. + patch_path: The unified-diff patch file to probe. + + Returns: + ``True`` when some strip level reverse-applies cleanly. + """ + for lvl in _P_LEVELS: + try: + cp = subprocess.run( + ["patch", f"-p{lvl}", "-R", "--dry-run", "-i", str(patch_path)], + capture_output=True, + text=True, + timeout=60, + check=False, + cwd=str(framework_root), + stdin=subprocess.DEVNULL, + ) + except (FileNotFoundError, subprocess.TimeoutExpired): + return False + if cp.returncode == 0: + return True + return False + + def _bak_name(patch_stem: str, rel_target: Path, seq: int) -> str: """Return a backup filename unique within a shared ``backup_root``. @@ -192,6 +231,10 @@ def _apply_patch_no_git( timeout=60, check=False, cwd=str(framework_root), + # ``patch`` prompts ("Assume -R? [n]") on an already-applied + # hunk; without a closed stdin it can inherit the parent's and + # block until the 60s timeout. + stdin=subprocess.DEVNULL, ) except (FileNotFoundError, subprocess.TimeoutExpired) as exc: err_msg = f"patch CLI unavailable or timed out: {exc}" @@ -207,6 +250,28 @@ def _apply_patch_no_git( detected_level = lvl break if detected_level is None: + # Before reporting failure, distinguish "does not apply" from "already + # applied". A specialist commonly writes a superset patch AND the subset + # it contains (e.g. an FLA-layout revert plus that same revert bundled + # with a config fix). Applying the superset makes every later subset + # hunk a no-op, and POSIX ``patch`` reports that as "Reversed (or + # previously applied) patch detected ... Skipping patch" with a non-zero + # exit -- indistinguishable, at the exit code, from a patch that simply + # does not fit. Treating it as a hard failure aborted the whole apply and + # reverted a combo that was in fact fully and correctly applied. + # + # A clean *reverse* dry-run is the unambiguous already-applied probe: it + # succeeds only when every hunk's post-state is already present in the + # tree, which is exactly the state a forward apply would produce. In that + # case the apply is a satisfied no-op -- report success with no backups + # (the patch that really made those edits owns the backups needed for a + # correct revert). + if _reverse_applies_cleanly(framework_root, patch_path): + log.info( + "nogit patch: %s is already fully applied (clean reverse dry-run); treating as a no-op", + patch_path.name, + ) + return True, "", [], None combined_stderr = "\n".join(dry_run_stderrs) err_msg = f"patch --dry-run failed at all strip levels for {patch_path.name}" try: @@ -379,6 +444,7 @@ def _backup_existing(abs_path: Path, rel: Path, action: str) -> tuple[dict[str, timeout=120, check=False, cwd=str(framework_root), + stdin=subprocess.DEVNULL, ) except (FileNotFoundError, subprocess.TimeoutExpired) as exc: err_msg = f"patch apply failed: {exc}" diff --git a/src/hyperloom/orchestrator/actions/executors/integrate_patch.py b/src/hyperloom/orchestrator/actions/executors/integrate_patch.py index 4da4f3533..e9a19e096 100644 --- a/src/hyperloom/orchestrator/actions/executors/integrate_patch.py +++ b/src/hyperloom/orchestrator/actions/executors/integrate_patch.py @@ -521,6 +521,34 @@ def _git_apply( ) +def _git_reverse_applies_cleanly(framework_root: Path, patch_path: Path) -> bool: + """True when ``patch_path`` is already fully applied in ``framework_root``. + + The git-channel twin of + :func:`._nogit_patch._reverse_applies_cleanly`. ``git apply -R --check`` + succeeds only when every hunk's *post*-state is already present, i.e. the + tree already equals what a forward apply would produce. Read-only: + ``--check`` never mutates the tree. + + Args: + framework_root: The git checkout the patch targets. + patch_path: The patch file to probe. + + Returns: + ``True`` when some strip level reverse-checks cleanly. + """ + from ._nogit_patch import _P_LEVELS + + for lvl in _P_LEVELS: + cp = _run_git_cp( + ["-C", str(framework_root), "apply", "-R", f"-p{lvl}", "--check", str(patch_path)], + timeout=120.0, + ) + if cp is not None and cp.returncode == 0: + return True + return False + + def _git_apply_collect_feedback( framework_root: Path, patch_path: Path, @@ -575,6 +603,20 @@ def _git_apply_collect_feedback( ok3, err3, fb3 = _git_apply_collect_feedback(framework_root, patch_path, three_way=True) if ok3: return True, "", None + # Still nothing. Distinguish "does not apply" from "already applied": + # a specialist often writes both a superset patch and the subset it + # contains, so applying one leaves the other a satisfied no-op that + # ``git apply --check`` nonetheless rejects. A clean *reverse* check + # succeeds only when every hunk's post-state is already in the tree, + # which is exactly what a forward apply would have produced -- so treat + # it as success rather than failing the whole combo. Partial overlap + # fails the reverse check and stays a real failure. + if _git_reverse_applies_cleanly(framework_root, patch_path): + log.info( + "integrate_patch: %s is already fully applied (clean git apply -R --check); treating as a no-op", + patch_path.name, + ) + return True, "", None # Merge both sets of stderrs. all_stderrs = "\n".join(level_stderrs) if err3: From 22566f3cc3272e187f2a2ecc8362bb03f5e8e6a1 Mon Sep 17 00:00:00 2001 From: ZhengGong-amd Date: Thu, 30 Jul 2026 07:01:30 +0000 Subject: [PATCH 27/32] fix(kb): surface the measured accuracy so the delta is not always zero The KB writeback read gate_evidence["accuracy"], a key _bench_patch never emitted: it graded the eval into an accuracy_pass bool and dropped the score. The raw value survived only on the enablement branch. accuracy_delta_pct therefore stayed 0.0 on every record even after the delta was threaded through. Parse the score alongside the existing grade and return it as gate_evidence["accuracy"]. The baseline side had the same gap -- it only fell back to shared_state.baseline_accuracy when the accuracy gate was required -- so read that fallback unconditionally for the record. Observed on Kimi-K3 (vllm, mi355x, mxfp4): ROCm/vllm PR 1045 dropped gsm8k to 0.0015 against a 0.8393 baseline and was reverted, yet its record still carried accuracy_delta_pct 0.0, leaving the fingerprint dedup gate nothing to match on. Co-Authored-By: Claude Opus 5 (1M context) --- .../actions/executors/integrate_patch.py | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/hyperloom/orchestrator/actions/executors/integrate_patch.py b/src/hyperloom/orchestrator/actions/executors/integrate_patch.py index 4da4f3533..ac1f59dd7 100644 --- a/src/hyperloom/orchestrator/actions/executors/integrate_patch.py +++ b/src/hyperloom/orchestrator/actions/executors/integrate_patch.py @@ -2643,7 +2643,11 @@ async def _gate_perf( specialist_task_id, ) gate_pass = delta_pct is not None and delta_pct >= keep_threshold_pct and not acc_block - acc_delta_pct = _accuracy_delta_pct(gate_evidence.get("accuracy"), acc_baseline) + _ss_kb = extra.get("shared_state") or extra.get("state") + acc_delta_pct = _accuracy_delta_pct( + gate_evidence.get("accuracy"), + acc_baseline or getattr(_ss_kb, "baseline_accuracy", None), + ) cfg_fingerprint = canonical_fingerprint( params.get("extra_server_args"), params.get("extra_envs"), @@ -3240,6 +3244,19 @@ async def _bench_patch( framework=params.get("framework") or os.environ.get("FRAMEWORK") or None, ) + # Raw accuracy for the KB record; ``accuracy_pass`` only carries a verdict. + measured_accuracy: float | None = None + if bench.get("status") == "succeeded": + try: + measured = parse_eval_results( + eval_search_root, + framework=params.get("framework") or os.environ.get("FRAMEWORK") or None, + ).get("accuracy") + if isinstance(measured, (int, float)): + measured_accuracy = float(measured) + except Exception: # noqa: BLE001 — advisory value only + log.debug("integrate_patch: accuracy parse for KB record failed", exc_info=True) + # Enablement path: surface the raw accuracy so the branch can apply a floor. enablement_accuracy: float | None = None enablement_accuracy_task = "" @@ -3260,6 +3277,7 @@ async def _bench_patch( return bench, { "accuracy_pass": accuracy_pass, + "accuracy": measured_accuracy, "enablement_accuracy": enablement_accuracy, "enablement_accuracy_task": enablement_accuracy_task, "enablement_accuracy_metric": enablement_accuracy_metric, From 6bd1fd55af52c2223d0d4a501f1565adfbdd4083 Mon Sep 17 00:00:00 2001 From: ZhengGong-amd Date: Thu, 30 Jul 2026 07:43:54 +0000 Subject: [PATCH 28/32] docs(enablement): correct the stale correctness_ok contract, drop dead import The enablement-gate docstring still described the pre-eval-origin behaviour: "accuracy is None -> correctness_ok=None (KEEP but provisional)". Since 25f8d6e88 that is only true for boot-origin; an eval-origin candidate with no score fails closed, because the trigger *was* an accuracy failure and a candidate producing no score has not shown it fixed anything. The docstring also omitted the eval-origin task/metric requirement entirely. Document the verdict per origin, and say why the two differ. `import math as _math` in the same function lost its last use when the NaN branch folded into accuracy_meets_floor() / classify_accuracy_failure(). Found reviewing the branch's 29 commits after merging origin/main. Docs and an unused import only -- no behaviour change. Co-Authored-By: Claude Opus 5 (1M context) --- .../actions/executors/integrate_patch.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/hyperloom/orchestrator/actions/executors/integrate_patch.py b/src/hyperloom/orchestrator/actions/executors/integrate_patch.py index e9a19e096..221b84a7c 100644 --- a/src/hyperloom/orchestrator/actions/executors/integrate_patch.py +++ b/src/hyperloom/orchestrator/actions/executors/integrate_patch.py @@ -2354,17 +2354,24 @@ async def _gate_enablement( ) -> dict[str, Any]: """Enablement gate: runnability + minimal-correctness. - Three states: - accuracy > floor -> correctness_ok=True (KEEP, verified) - accuracy <= floor/NaN -> correctness_ok=False (REVERT, garbage) - accuracy is None -> correctness_ok=None (KEEP but provisional) + The verdict depends on the trigger origin, because the two origins have + different evidence available: + + * ``accuracy >= floor`` -> ``correctness_ok=True`` (KEEP, verified). + eval-origin additionally requires the score to carry a task + metric; + without them it did not come from a real eval and fails closed. + * present but below floor / non-positive / non-finite -> + ``correctness_ok=False`` (REVERT, garbage output). + * ``accuracy is None`` -> eval-origin fails closed + (``correctness_ok=False``): the trigger *was* an accuracy failure, so a + candidate that produces no score has not shown it fixed anything. + boot-origin stays ``None`` (KEEP but provisional) — it only ever + claimed to make the model boot, and eval-less runs must not be blocked. On KEEP, when an attempt runtime was provisioned, the stack action is recorded in the result (``enablement_kept_stack_action``) so it survives rearm. On REVERT / non-KEEP, the attempt runtime dir is GC'd. """ - import math as _math - stack_action = getattr(ctx, "_ip_stack_action", None) if ctx is not None else None provision_result = getattr(ctx, "_ip_provision_result", None) if ctx is not None else None From e419b9c8a829e026d1a42c43b9523ac16805eac3 Mon Sep 17 00:00:00 2001 From: ZhengGong-amd Date: Thu, 30 Jul 2026 08:02:33 +0000 Subject: [PATCH 29/32] revert(baseline): drop the checkpoint-scaled server-ready timeout dd2691728 sized the server-boot budget from checkpoint bytes because a 1.4 TiB MXFP4 MoE was being killed mid-load at the flat 2700s default. That failure mode is gone: the boot path no longer needs the allowance, so the scaling is dead weight on every model that reaches the executor. It was also only ever reachable above CHECKPOINT_SCALING_MIN_GIB (200 GiB) -- every checkpoint below that already took the flat default unchanged, so the revert is a no-op for them by construction. Verified against the live Kimi-Linear-48B run (92 GiB): resolve_server_ready_timeout() returned the plain 2700s and the materialized lifecycle YAML carries server_ready_timeout_s 2700, i.e. the removed code never fired. Removes resolve_server_ready_timeout / _checkpoint_size_gib and their constants, the _apply_server_ready_floor subprocess-timeout floor, and the test_aiter_jit fixture that existed only to keep the ambient $MODEL_PATH from perturbing the scaled timeouts. No references remain. Deliberately NOT reverted from the two [Temp] commits, having checked each against the current code and the live session: - install.sh single-gateway credential derivation (dd2691728). Not dead: .env.template ships SAFE_API_KEY + OPENAI_BASE_URL uncommented as the default with the ANTHROPIC_* block commented out, so removing it breaks IR-2 for anyone following the documented setup. - integrate_patch non-git commit-on-KEEP skip (c20347403). Live right now: this session's framework root is a pip-installed vLLM in site-packages with no .git, where the old path fails `git add` with rc=128 and logs a spurious commit-on-KEEP failure on every enablement KEEP. - tracelens_arch_benchmark timeout catch (c20347403). A microbench overrun raises subprocess.TimeoutExpired, not RuntimeError, so without it trace_analyze dies instead of taking its fallback. - _magpie_patcher grouped run_lm_eval parser (c20347403). Superseded and extended by 3166da7f2 / b4fd4b73b / 74b549e89 / 1f63b4849, all already in origin/main; the file is now byte-identical to main and owned upstream. 729 passed / 1 skipped across the baseline, lifecycle and magpie test subset. Co-Authored-By: Claude Opus 5 (1M context) --- .../tests/test_aiter_jit.py | 12 --- .../actions/executors/_server_lifecycle.py | 97 ++----------------- .../actions/executors/baseline.py | 48 +-------- 3 files changed, 9 insertions(+), 148 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_aiter_jit.py b/src/hyperloom/inference_optimizer/tests/test_aiter_jit.py index f1dc8f2f7..3753a2bc8 100644 --- a/src/hyperloom/inference_optimizer/tests/test_aiter_jit.py +++ b/src/hyperloom/inference_optimizer/tests/test_aiter_jit.py @@ -173,18 +173,6 @@ def test_sweep_unknown_falls_back_to_mtime_gate(monkeypatch, tmp_path): # --------------------------------------------------------------------------- # _resolve_timeout integration # --------------------------------------------------------------------------- -@pytest.fixture(autouse=True) -def _no_ambient_model_path(monkeypatch): - """Keep ``_resolve_timeout`` off the server-ready floor in these tests. - - The floor scales with checkpoint size read from ``$MODEL_PATH``, so an - operator shell that exports a real model would otherwise change the - timeouts these tests assert on. - """ - monkeypatch.delenv("MODEL_PATH", raising=False) - monkeypatch.delenv("INFERENCE_OPTIMIZER_BASELINE_SERVER_READY_SEC", raising=False) - - def _cold_probe(*_a, **_k): return { "path": "/fake/jit", diff --git a/src/hyperloom/orchestrator/actions/executors/_server_lifecycle.py b/src/hyperloom/orchestrator/actions/executors/_server_lifecycle.py index 3bc18dde7..67f7949cd 100644 --- a/src/hyperloom/orchestrator/actions/executors/_server_lifecycle.py +++ b/src/hyperloom/orchestrator/actions/executors/_server_lifecycle.py @@ -61,23 +61,6 @@ # Override via ``INFERENCE_OPTIMIZER_BASELINE_SERVER_READY_SEC``. SERVER_READY_TIMEOUT_SEC = 2700 -# Weight loading dominates server boot for very large checkpoints, and it -# scales with checkpoint bytes rather than with anything the flat -# ``SERVER_READY_TIMEOUT_SEC`` default can express. A 1.4 TiB MoE checkpoint on -# a network filesystem needs well over an hour just to read the safetensors -# shards, so the flat 2700s default kills a perfectly healthy server mid-load -# (Magpie reports "Shared server launcher timed out"). Budget the read at a -# deliberately pessimistic floor throughput so the allowance stays generous on -# slow NFS mounts while still bounding a genuine hang. -CHECKPOINT_LOAD_FLOOR_GIB_PER_SEC = 0.20 - -# Upper bound on the size-derived allowance: past this a stuck loader should -# surface as a timeout rather than idling for the rest of the budget. -SERVER_READY_MAX_TIMEOUT_SEC = 21600 # 6 h - -# Checkpoints below this size keep the flat default (no scaling). -CHECKPOINT_SCALING_MIN_GIB = 200.0 - def _pick_free_port() -> int: """Return an OS-assigned free TCP port. @@ -217,79 +200,6 @@ def resolve_lifecycle_params(materialized_config_path: Path) -> dict[str, Any]: return info -def _checkpoint_size_gib(model_path: str | None) -> float: - """Return the on-disk weight size of ``model_path`` in GiB (0.0 when unknown). - - Sums the top-level weight shards only (``*.safetensors`` / ``*.bin``); a - recursive walk over an HF cache tree is not worth the stat storm on a - network filesystem. - - Args: - model_path: Local model directory, or ``None`` / an HF repo id. - - Returns: - The summed shard size in GiB, or ``0.0`` when the path is not a - readable local directory. - """ - raw = str(model_path or "").strip() - if not raw: - return 0.0 - try: - root = Path(raw) - if not root.is_dir(): - return 0.0 - total = 0 - for pattern in ("*.safetensors", "*.bin"): - for shard in root.glob(pattern): - try: - total += shard.stat().st_size - except OSError: - continue - return total / (1024.0**3) - except OSError: - return 0.0 - - -def resolve_server_ready_timeout(model_path: str | None) -> int: - """Return the server-boot budget in seconds, scaled by checkpoint size. - - ``INFERENCE_OPTIMIZER_BASELINE_SERVER_READY_SEC`` wins when set. Otherwise - small checkpoints keep the flat :data:`SERVER_READY_TIMEOUT_SEC` default, - and larger ones add an allowance for reading the weights at - :data:`CHECKPOINT_LOAD_FLOOR_GIB_PER_SEC`, clamped to - :data:`SERVER_READY_MAX_TIMEOUT_SEC` so a hung loader still times out. - - Callers that also impose an outer subprocess timeout must keep it above - this value (plus Magpie's own 180s launcher grace), or the outer kill - preempts the boot with a less specific error. - - Args: - model_path: Local model directory used to size the allowance. - - Returns: - The server-ready timeout in seconds. - """ - env_override = str(os.environ.get("INFERENCE_OPTIMIZER_BASELINE_SERVER_READY_SEC", "") or "").strip() - if env_override: - return int(env_override) - size_gib = _checkpoint_size_gib(model_path) - if size_gib < CHECKPOINT_SCALING_MIN_GIB: - return SERVER_READY_TIMEOUT_SEC - load_allowance = int(size_gib / CHECKPOINT_LOAD_FLOOR_GIB_PER_SEC) - scaled = min(SERVER_READY_TIMEOUT_SEC + load_allowance, SERVER_READY_MAX_TIMEOUT_SEC) - log.info( - "server_lifecycle: checkpoint %.0f GiB at %s — server_ready_timeout_s " - "%d -> %d (weight-load allowance %ds at a %.2f GiB/s floor)", - size_gib, - model_path, - SERVER_READY_TIMEOUT_SEC, - scaled, - load_allowance, - CHECKPOINT_LOAD_FLOOR_GIB_PER_SEC, - ) - return scaled - - def inject_lifecycle( bench: dict[str, Any], *, @@ -309,7 +219,12 @@ def inject_lifecycle( pid_dir: Shared directory for the server pid/meta files. port: HTTP port pinned for the persistent server. """ - ready_timeout = resolve_server_ready_timeout(bench.get("model")) + ready_timeout = int( + os.environ.get( + "INFERENCE_OPTIMIZER_BASELINE_SERVER_READY_SEC", + SERVER_READY_TIMEOUT_SEC, + ) + ) bench["server_lifecycle"] = { "enabled": True, "cleanup": bool(cleanup), diff --git a/src/hyperloom/orchestrator/actions/executors/baseline.py b/src/hyperloom/orchestrator/actions/executors/baseline.py index efc45ab41..6f39889a4 100644 --- a/src/hyperloom/orchestrator/actions/executors/baseline.py +++ b/src/hyperloom/orchestrator/actions/executors/baseline.py @@ -286,14 +286,6 @@ def _classify_subprocess_error( BASELINE_DEFAULT_TIMEOUT_SEC = 7800 # WARM-start cap, 130 min BASELINE_COLD_START_TIMEOUT_SEC = 9000 # COLD-start cap, 150 min (includes ~20 min cuda graph capture) - -# Slack the subprocess keeps beyond the server-ready budget: Magpie's launcher -# adds its own 180s grace on top of ``server_ready_timeout_s``, then the client -# benchmark and teardown still have to run. Without this the outer timeout can -# fire while the server is legitimately still booting, which reports the far -# less actionable ``timeout`` / ``subprocess_nonzero`` instead of a real -# server-boot failure. -BASELINE_POST_READY_MARGIN_SEC = 1800 # COLD_START_KERNEL_THRESHOLD and AITER_JIT_PROBE_PATHS live in ``_aiter_jit``; # re-exported below for callers/tests that import them from this module. @@ -1060,7 +1052,7 @@ def _resolve_timeout(self, params: dict[str, Any]) -> int: self.default_timeout_sec, cold_cap, ) - return self._apply_server_ready_floor(cold_cap, params) + return cold_cap if cache["probe_status"] == "found": log.info( "baseline_executor: WARM start — aiter jit/build/ at %s has %d .so, %d MB. Using default timeout=%ds.", @@ -1069,7 +1061,7 @@ def _resolve_timeout(self, params: dict[str, Any]) -> int: cache["size_mb"], self.default_timeout_sec, ) - return self._apply_server_ready_floor(self.default_timeout_sec, params) + return self.default_timeout_sec log.warning( "baseline_executor: aiter jit cache not located " "(probe_status=%s). Using default timeout=%ds. Cold-start " @@ -1077,41 +1069,7 @@ def _resolve_timeout(self, params: dict[str, Any]) -> int: cache["probe_status"], self.default_timeout_sec, ) - return self._apply_server_ready_floor(self.default_timeout_sec, params) - - def _apply_server_ready_floor(self, timeout_sec: int, params: dict[str, Any]) -> int: - """Raise ``timeout_sec`` so it cannot preempt the server-boot budget. - - The lifecycle ``server_ready_timeout_s`` scales with checkpoint size, - so on very large models it can exceed the aiter-probe-derived - subprocess cap. When that happens the subprocess is killed while the - server is still loading weights and the failure is misreported as a - benchmark timeout. - - Args: - timeout_sec: The probe-derived subprocess timeout. - params: Task params, consulted for ``model_path``. - - Returns: - ``timeout_sec``, or the server-ready budget plus - :data:`BASELINE_POST_READY_MARGIN_SEC` when that is larger. - """ - model_path = str(params.get("model_path") or os.environ.get("MODEL_PATH") or "").strip() - ready_sec = _lifecycle.resolve_server_ready_timeout(model_path) - floor = ready_sec + BASELINE_POST_READY_MARGIN_SEC - if floor <= timeout_sec: - return timeout_sec - log.warning( - "baseline_executor: raising timeout %ds -> %ds so it stays above " - "the server-boot budget (server_ready_timeout_s=%ds + %ds margin) " - "for %s.", - timeout_sec, - floor, - ready_sec, - BASELINE_POST_READY_MARGIN_SEC, - model_path or "", - ) - return floor + return self.default_timeout_sec @staticmethod def _inferencex_root_from_config(config_path: Path) -> str: From cc9e86cd9b84c3d8a96e3d87760be265f0c0a382 Mon Sep 17 00:00:00 2001 From: ZhengGong-amd Date: Thu, 30 Jul 2026 08:17:57 +0000 Subject: [PATCH 30/32] fix(enablement): stop an issue citation from aborting a targeted build resolve_build_ref() understood PR URLs and bare "PR:{n}" refs but had no branch for issues, so an "issue:{n}" string fell through to the plain tag/branch/sha case and was handed to `git worktree add` verbatim. An issue is a discussion thread, not a branch: GitHub publishes refs/pull/{n}/head for a pull request but nothing checkoutable for an issue, so the build aborts in workspace preparation with `fatal: invalid reference: issue:41292`. Observed live on the Kimi-Linear-48B run: the enablement specialist correctly root-caused the KDA chunked-prefill state-layout defect, requested a from-source vLLM build, and cited upstream vllm issue 41292 as the rationale. The build died 123s in, before compiling anything, and the round was lost. Citing an issue is a normal and useful signal, so drop the issue number rather than reject the request: an empty ref makes the builder autoselect the newest matching tag, which is where an issue fix would have landed anyway. Handle both the bare "issue:{n}" / "issues:{n}" form and a full issue URL, keeping the URL as source provenance so the audit trail retains the citation. The caller in phases/framework.py merged the result with `ref = _ref or ref`, which would have restored the very string resolution had just rejected. Take the resolved ref verbatim, empty included -- empty now carries the meaning "not checkoutable, autoselect a tag". The 7 failures in the touched trees (test_cli_backends_unit, test_preflight_auth_override) are pre-existing and environment-dependent -- identical with this change stashed. Co-Authored-By: Claude Opus 5 (1M context) --- .../tests/test_build_actions.py | 39 +++++++++++++++++++ .../orchestrator/framework/build_actions.py | 26 ++++++++++++- .../orchestrator/phases/framework.py | 6 ++- 3 files changed, 69 insertions(+), 2 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_build_actions.py b/src/hyperloom/inference_optimizer/tests/test_build_actions.py index 4a7306f98..01b9a5b39 100644 --- a/src/hyperloom/inference_optimizer/tests/test_build_actions.py +++ b/src/hyperloom/inference_optimizer/tests/test_build_actions.py @@ -200,6 +200,45 @@ def test_resolve_empty_candidate_skipped(): assert resolve_build_ref("", "https://github.com/ROCm/aiter") == ("", "", "") +# An issue is a discussion thread, not a branch. GitHub publishes +# refs/pull/{n}/head for a PR but nothing checkoutable for an issue, so an +# ``issue:{n}`` string reaching ``git worktree add`` verbatim aborts the build +# with ``fatal: invalid reference``. Observed live: an enablement specialist +# cited upstream vllm issue 41292 as the rationale for a from-source build and +# the build died in ~2 minutes during workspace preparation. Resolution must +# strip the issue number to an empty ref (falling back to tag autoselect) while +# keeping the citation as provenance. + + +def test_resolve_bare_issue_ref_falls_back_to_autoselect(): + repo, ref, pr_url = resolve_build_ref("issue:41292", "https://github.com/ROCm/aiter") + assert repo == "https://github.com/ROCm/aiter" + assert ref == "", "an issue number is not a checkoutable git ref" + assert pr_url == "" + + +def test_resolve_plural_issues_ref_also_handled(): + _repo, ref, _pr = resolve_build_ref("issues:41292", "https://github.com/ROCm/aiter") + assert ref == "" + + +def test_resolve_github_issue_url_keeps_repo_and_provenance(): + url = "https://github.com/vllm-project/vllm/issues/41292" + repo, ref, pr_url = resolve_build_ref(url, "https://github.com/ROCm/aiter") + assert repo == "https://github.com/vllm-project/vllm" + assert ref == "" + assert pr_url == url, "the citation is kept for the audit trail" + + +def test_resolve_pr_still_wins_over_issue_shapes(): + """A PR remains checkoutable; the issue branch must not shadow it.""" + _repo, ref, _pr = resolve_build_ref( + "https://github.com/vllm-project/vllm/pull/33291", + "https://github.com/ROCm/aiter", + ) + assert ref == "PR:33291" + + def test_source_pr_url_round_trip(): a = _action(ref="PR:99", source_pr_url="https://github.com/ROCm/aiter/pull/99") a2 = TargetedBuildAction.from_state(a.to_state()) diff --git a/src/hyperloom/orchestrator/framework/build_actions.py b/src/hyperloom/orchestrator/framework/build_actions.py index 82fe87ebe..30063a36e 100644 --- a/src/hyperloom/orchestrator/framework/build_actions.py +++ b/src/hyperloom/orchestrator/framework/build_actions.py @@ -256,17 +256,33 @@ def build_novelty_key( _GITHUB_PR_RE = _re.compile(r"https?://github\.com/([^/]+/[^/]+)/pull/(\d+)", _re.IGNORECASE) _PR_REF_RE = _re.compile(r"^PR:(\d+)$") +# An issue is a discussion thread, not a branch: GitHub publishes +# ``refs/pull/{n}/head`` for pull requests but nothing checkoutable for issues. +_GITHUB_ISSUE_RE = _re.compile(r"https?://github\.com/([^/]+/[^/]+)/issues/(\d+)", _re.IGNORECASE) +_ISSUE_REF_RE = _re.compile(r"^issues?:(\d+)$", _re.IGNORECASE) def resolve_build_ref(candidate: str, default_repo_url: str) -> tuple[str, str, str]: """Resolve a discovered candidate string to ``(repo_url, ref, source_pr_url)``. - Handles the three forms discovery produces: + Handles the forms discovery and the enablement specialist produce: - ``https://github.com/{owner}/{repo}/pull/{n}`` → PR ref with full provenance. - ``PR:{n}`` bare ref → PR ref against ``default_repo_url``. + - an issue URL or ``issue:{n}`` → repo with an EMPTY ref, i.e. fall back to + tag autoselect (see below). - plain tag/branch/sha → verbatim ref against ``default_repo_url``. - any other URL → ``("", "", "")`` (skip; cannot derive a checkoutable ref). + Issues need their own branch because they are *not* checkoutable. GitHub + publishes ``refs/pull/{n}/head`` for a PR but exposes no ref for an issue, + so an ``issue:{n}`` string that reaches ``git worktree add`` verbatim dies + with ``fatal: invalid reference``. A specialist citing an upstream issue as + the rationale for a from-source build is a normal and useful signal, so the + issue number is dropped from the ref rather than the whole request being + rejected: the empty ref makes the builder autoselect the newest matching + tag, which is exactly where an issue fix would have landed. The issue URL is + still returned as provenance so the audit trail keeps the citation. + Args: candidate: The candidate ref string from discovery. default_repo_url: Repo URL to use when the candidate carries no origin. @@ -288,6 +304,14 @@ def resolve_build_ref(candidate: str, default_repo_url: str) -> tuple[str, str, number = s.split(":", 1)[1] return (default_repo_url, f"PR:{number}", "") + m = _GITHUB_ISSUE_RE.match(s) + if m: + slug = m.group(1) + return (f"https://github.com/{slug}", "", s) + + if _ISSUE_REF_RE.match(s): + return (default_repo_url, "", "") + if "://" in s or s.startswith("git@"): return ("", "", "") diff --git a/src/hyperloom/orchestrator/phases/framework.py b/src/hyperloom/orchestrator/phases/framework.py index 7113c4fe2..c26ea90e1 100644 --- a/src/hyperloom/orchestrator/phases/framework.py +++ b/src/hyperloom/orchestrator/phases/framework.py @@ -4346,7 +4346,11 @@ def _consume_marker() -> None: if candidate: _repo, _ref, _pr = resolve_build_ref(candidate, repo_url) repo_url = _repo or repo_url - ref = _ref or ref + # Take the resolved ref verbatim, including empty. An empty ref + # means "not checkoutable, autoselect a tag" (an issue citation), + # so falling back to the raw request here would hand the builder + # back the very string resolution just rejected. + ref = _ref source_pr_url = _pr if not _repo_matches_targeted_build_component(repo_url, component): log.warning( From 6f94aa1ee155f1d07617e77d7f1646a698d5c902 Mon Sep 17 00:00:00 2001 From: ZhengGong-amd Date: Thu, 30 Jul 2026 11:41:57 +0000 Subject: [PATCH 31/32] fix(enablement): green the credential test and sync the accuracy-floor docs Two review P0s on #1054. test_setup_cli: the single-gateway branch added by this PR reads ${SAFE_API_KEY:-} in memory to derive ANTHROPIC_BASE_URL/ANTHROPIC_API_KEY (mirroring the CLI's _resolve_llm_endpoints), which tripped a blanket "SAFE_API_KEY:- not in script_text" assertion and broke CI. The invariant that test exists to protect is that the gateway credentials are never *persisted* -- not that they are never read. Scope the assertions to write_env_file() accordingly, and additionally pin the scrubbing behaviour (remove_dotenv_var for SAFE_API_KEY / OPENAI_BASE_URL / OPENAI_API_KEY) so a regression that starts writing them back out still fails. docs/template: DEFAULT_ENABLEMENT_ACCURACY_FLOOR moved 0.0 -> 0.05 in this PR, but .env.template and the environment-variables table still advertised 0.0. Sync both and spell out the semantics the code implements: a score of exactly 0.0 fails regardless of the floor, otherwise score >= floor passes, and the default is a collapse guard rather than a quality bar. --- .env.template | 8 ++++++-- docs/reference/environment-variables.md | 2 +- .../inference_optimizer/tests/test_setup_cli.py | 13 ++++++++++++- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/.env.template b/.env.template index 8f8db8bc8..c2923e0fc 100644 --- a/.env.template +++ b/.env.template @@ -70,10 +70,14 @@ OPENAI_BASE_URL=https:///api/v1/llm-proxy/v1 # into enablement instead of halting the run. Default on (single-node only). # INFERENCE_OPTIMIZER_ENABLEMENT_ON_EVAL_FAIL=1 # (Optional) Shared accuracy floor for the eval-failure trigger AND the -# enablement KEEP gate (finite, in [0,1]). Default 0.0. +# enablement KEEP gate (finite, in [0,1]). Default 0.05. +# The default is a collapse guard, not a quality bar: it rejects a model that +# answers essentially nothing correctly (a real run once KEEPed a candidate at +# gsm8k 0.00076 under the old 0.0 default, i.e. 0.08% of a 0.906 baseline). +# Raise it if you want a genuine quality gate. # Note: score=0.0 always fails regardless of floor (strictly positive required). # score>=floor passes (score==floor is a pass). Single-node only. -# INFERENCE_OPTIMIZER_ENABLEMENT_ACCURACY_FLOOR=0.0 +# INFERENCE_OPTIMIZER_ENABLEMENT_ACCURACY_FLOOR=0.05 # Writable artifact root: hosts every session dir, optimizer_runs/, and the # runtime/ tree generated by install.sh (GEAK e2e checkout, diff --git a/docs/reference/environment-variables.md b/docs/reference/environment-variables.md index 630d4d4d9..69d6d309a 100644 --- a/docs/reference/environment-variables.md +++ b/docs/reference/environment-variables.md @@ -210,7 +210,7 @@ existing behavior. | Variable | Default | Description | |----------|---------|-------------| | `INFERENCE_OPTIMIZER_ENABLEMENT_ON_EVAL_FAIL` | `1` (on) | Route a baseline accuracy-eval failure into enablement instead of halting. Set `0`/`false`/`no`/`off` to keep the legacy salvage/stop behavior. Reader: `_accuracy_gate.enablement_on_eval_fail_enabled`. | -| `INFERENCE_OPTIMIZER_ENABLEMENT_ACCURACY_FLOOR` | `0.0` | Shared accuracy floor used by BOTH the baseline eval-failure trigger and the enablement KEEP gate. Accepts a finite value in `[0, 1]`; out-of-range values are ignored with a warning. Reader: `_accuracy_gate.enablement_accuracy_floor`. | +| `INFERENCE_OPTIMIZER_ENABLEMENT_ACCURACY_FLOOR` | `0.05` | Shared accuracy floor used by BOTH the baseline eval-failure trigger and the enablement KEEP gate. A collapse guard rather than a quality bar — raise it for a genuine quality gate. A score of exactly `0.0` always fails regardless of the floor (strictly positive is required); otherwise `score >= floor` passes. Accepts a finite value in `[0, 1]`; out-of-range values are ignored with a warning. Reader: `_accuracy_gate.enablement_accuracy_floor`. | --- diff --git a/src/hyperloom/inference_optimizer/tests/test_setup_cli.py b/src/hyperloom/inference_optimizer/tests/test_setup_cli.py index 89cb85aec..e6b3e5906 100644 --- a/src/hyperloom/inference_optimizer/tests/test_setup_cli.py +++ b/src/hyperloom/inference_optimizer/tests/test_setup_cli.py @@ -621,7 +621,18 @@ def test_kernel_install_no_longer_exports_openai_safe_credentials(): assert "export OPENAI_API_KEY" not in write_text assert "upsert_dotenv_var OPENAI_BASE_URL" not in write_text assert "upsert_dotenv_var OPENAI_API_KEY" not in write_text - assert "SAFE_API_KEY:-" not in script_text + + # The gateway credentials may be *read* in memory -- the single-gateway + # branch derives ANTHROPIC_BASE_URL/ANTHROPIC_API_KEY from + # OPENAI_BASE_URL + SAFE_API_KEY, mirroring the CLI's + # _resolve_llm_endpoints(). What must never happen is persisting them: + # neither exported into kernel-agent.env.sh nor written back to .env. + assert "export SAFE_API_KEY" not in write_text + assert "upsert_dotenv_var SAFE_API_KEY" not in write_text + # ... and the generated env file scrubs any stale copy left by an older run. + assert "remove_dotenv_var SAFE_API_KEY" in write_text + assert "remove_dotenv_var OPENAI_BASE_URL" in write_text + assert "remove_dotenv_var OPENAI_API_KEY" in write_text def test_packaged_install_sh_resolves_target_workspace_root(tmp_path: Path): From 6214d7d4638b452f3397e7f7ac01649fa97c3b98 Mon Sep 17 00:00:00 2001 From: ZhengGong-amd Date: Fri, 31 Jul 2026 03:51:47 +0000 Subject: [PATCH 32/32] feat(reference-script): carry the Kimi-K3 recipe's serving envs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The whitelist gating parse_reference_script covered the AITER and NCCL channel knobs but not three switches the Kimi-K3 MI355X recipe sets: VLLM_ALLREDUCE_USE_FLASHINFER, VLLM_USE_RUST_FRONTEND and NCCL_DMABUF_ENABLE. A recipe carrying them lifted its serve flags but no envs, so the baseline never recorded them and EXPLORE had no ablation handle for them. All three are boolean serving switches, matching what the list already holds. Deliberately left out: VLLM_ENGINE_READY_TIMEOUT_S, a startup-wait bound with no throughput effect — lifting it would only add a meaningless ablation axis. The list stays an explicit enumeration rather than a VLLM_* glob, so path-valued and secret vars are still excluded; the existing VLLM_CACHE_ROOT / SOME_SECRET assertions cover that and are unchanged. Co-Authored-By: Claude Opus 5 (1M context) --- src/hyperloom/inference_optimizer/reference_script.py | 3 +++ .../inference_optimizer/tests/test_reference_script.py | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/src/hyperloom/inference_optimizer/reference_script.py b/src/hyperloom/inference_optimizer/reference_script.py index 237664c0f..fefd296bb 100644 --- a/src/hyperloom/inference_optimizer/reference_script.py +++ b/src/hyperloom/inference_optimizer/reference_script.py @@ -46,8 +46,11 @@ "VLLM_ROCM_USE_AITER", "VLLM_ROCM_USE_AITER_MHA", "VLLM_ROCM_USE_AITER_MOE", + "VLLM_ALLREDUCE_USE_FLASHINFER", + "VLLM_USE_RUST_FRONTEND", "SGLANG_USE_AITER", "SGLANG_MOE_PADDING", + "NCCL_DMABUF_ENABLE", "NCCL_MIN_NCHANNELS", "NCCL_MAX_NCHANNELS", } diff --git a/src/hyperloom/inference_optimizer/tests/test_reference_script.py b/src/hyperloom/inference_optimizer/tests/test_reference_script.py index 14f239ab5..f7aad010c 100644 --- a/src/hyperloom/inference_optimizer/tests/test_reference_script.py +++ b/src/hyperloom/inference_optimizer/tests/test_reference_script.py @@ -20,6 +20,9 @@ export VLLM_USE_BREAKABLE_CUDAGRAPH=0 export VLLM_ROCM_USE_AITER=1 +export VLLM_ALLREDUCE_USE_FLASHINFER=1 +export VLLM_USE_RUST_FRONTEND=1 +export NCCL_DMABUF_ENABLE=0 export VLLM_CACHE_ROOT=/workspace/cache export SOME_SECRET=hunter2 @@ -83,6 +86,9 @@ def test_parse_env_whitelist(tmp_path): r = parse_reference_script(src, framework="vllm") assert r.envs.get("VLLM_USE_BREAKABLE_CUDAGRAPH") == "0" assert r.envs.get("VLLM_ROCM_USE_AITER") == "1" + assert r.envs.get("VLLM_ALLREDUCE_USE_FLASHINFER") == "1" + assert r.envs.get("VLLM_USE_RUST_FRONTEND") == "1" + assert r.envs.get("NCCL_DMABUF_ENABLE") == "0" assert "VLLM_CACHE_ROOT" not in r.envs assert "SOME_SECRET" not in r.envs