From fa60ef03a0c8e40e950d04515cba28ce585ea5c8 Mon Sep 17 00:00:00 2001 From: mmprotest Date: Fri, 1 May 2026 12:12:44 +1000 Subject: [PATCH 1/3] Add bounded final-answer acceptance gate in runner --- tests/test_final_answer_gate.py | 119 ++++++++++++++++++++++++++++++++ villani_code/state.py | 110 +++++++++++++++++++++++++++++ villani_code/state_runtime.py | 25 +++++++ 3 files changed, 254 insertions(+) create mode 100644 tests/test_final_answer_gate.py diff --git a/tests/test_final_answer_gate.py b/tests/test_final_answer_gate.py new file mode 100644 index 00000000..986a3a6b --- /dev/null +++ b/tests/test_final_answer_gate.py @@ -0,0 +1,119 @@ +from villani_code.state import final_answer_block_reason + + +def test_concrete_edit_claim_without_edits_blocks_once() -> None: + nudge = {} + reason = final_answer_block_reason( + instruction="implement fix", + final_text="I'll update x.py to handle this case.", + has_any_edit=False, + changed_files=[], + known_verification_command=True, + verification_ran_after_last_edit=True, + last_verification_failed=False, + had_corrective_action_after_last_failure=False, + nudge_state=nudge, + ) + assert reason is not None + reason2 = final_answer_block_reason( + instruction="implement fix", + final_text="I'll update x.py to handle this case.", + has_any_edit=False, + changed_files=[], + known_verification_command=True, + verification_ran_after_last_edit=True, + last_verification_failed=False, + had_corrective_action_after_last_failure=False, + nudge_state=nudge, + ) + assert reason2 is None + + +def test_vague_suggestion_does_not_block() -> None: + reason = final_answer_block_reason( + instruction="implement fix", + final_text="One possible fix would be to update parser.py.", + has_any_edit=False, + changed_files=[], + known_verification_command=True, + verification_ran_after_last_edit=True, + last_verification_failed=False, + had_corrective_action_after_last_failure=False, + nudge_state={}, + ) + assert reason is None + + +def test_plan_only_task_does_not_block() -> None: + reason = final_answer_block_reason( + instruction="Plan only, do not implement.", + final_text="I'll update parser.py", + has_any_edit=False, + changed_files=[], + known_verification_command=True, + verification_ran_after_last_edit=False, + last_verification_failed=True, + had_corrective_action_after_last_failure=False, + nudge_state={}, + ) + assert reason is None + + +def test_changed_files_without_verification_blocks_when_command_known() -> None: + reason = final_answer_block_reason( + instruction="implement fix", + final_text="Done", + has_any_edit=True, + changed_files=["src/a.py"], + known_verification_command=True, + verification_ran_after_last_edit=False, + last_verification_failed=False, + had_corrective_action_after_last_failure=False, + nudge_state={}, + ) + assert "have not verified" in str(reason) + + +def test_changed_files_without_known_verification_does_not_block() -> None: + reason = final_answer_block_reason( + instruction="implement fix", + final_text="Done", + has_any_edit=True, + changed_files=["src/a.py"], + known_verification_command=False, + verification_ran_after_last_edit=False, + last_verification_failed=False, + had_corrective_action_after_last_failure=False, + nudge_state={}, + ) + assert reason is None + + +def test_failed_verification_requires_followup_action() -> None: + reason = final_answer_block_reason( + instruction="implement fix", + final_text="Done", + has_any_edit=True, + changed_files=["src/a.py"], + known_verification_command=True, + verification_ran_after_last_edit=True, + last_verification_failed=True, + had_corrective_action_after_last_failure=False, + nudge_state={}, + ) + assert "last verification failed" in str(reason) + + +def test_failed_verification_with_followup_does_not_block() -> None: + reason = final_answer_block_reason( + instruction="implement fix", + final_text="Done", + has_any_edit=True, + changed_files=["src/a.py"], + known_verification_command=True, + verification_ran_after_last_edit=True, + last_verification_failed=True, + had_corrective_action_after_last_failure=True, + nudge_state={}, + ) + assert reason is None diff --git a/villani_code/state.py b/villani_code/state.py index 1a2c80d4..f612ceaf 100644 --- a/villani_code/state.py +++ b/villani_code/state.py @@ -69,6 +69,61 @@ def _dedupe_preserve(items: list[str]) -> list[str]: return ordered +_CONCRETE_EDIT_PATTERNS = ( + r"\bi(?:'ll| will)\s+(?:update|modify|change|patch|edit|add)\b", + r"\bthe fix is to\b", + r"\bchange the\b", +) +_HEDGING_PATTERNS = ( + r"\bone possible fix would be\b", + r"\byou could\b", + r"\bi recommend\b", + r"\bno code changes are needed\b", +) + + +def _is_analysis_or_plan_only_request(instruction: str) -> bool: + lowered = instruction.lower() + return any( + phrase in lowered + for phrase in ("analysis only", "plan only", "just a plan", "do not implement", "no code changes") + ) + + +def _commits_to_concrete_edit(text: str) -> bool: + lowered = text.lower() + if any(re.search(pattern, lowered) for pattern in _HEDGING_PATTERNS): + return False + return any(re.search(pattern, lowered) for pattern in _CONCRETE_EDIT_PATTERNS) + + +def final_answer_block_reason(*, instruction: str, final_text: str, has_any_edit: bool, changed_files: list[str], known_verification_command: bool, verification_ran_after_last_edit: bool, last_verification_failed: bool, had_corrective_action_after_last_failure: bool, nudge_state: dict[str, bool]) -> str | None: + if _is_analysis_or_plan_only_request(instruction): + return None + if ( + _commits_to_concrete_edit(final_text) + and not has_any_edit + and not nudge_state.get("no_edit_after_concrete_claim", False) + ): + nudge_state["no_edit_after_concrete_claim"] = True + return "You described a concrete implementation change, but no file was modified. Apply the minimal implementation edit now. Do not explain further." + if ( + changed_files + and known_verification_command + and not verification_ran_after_last_edit + and not nudge_state.get("no_verification_after_edit", False) + ): + nudge_state["no_verification_after_edit"] = True + return "You modified files but have not verified the change yet. Run the relevant verification command now." + if ( + last_verification_failed + and not had_corrective_action_after_last_failure + and not nudge_state.get("failed_verification_no_followup", False) + ): + nudge_state["failed_verification_no_followup"] = True + return "The last verification failed. Use the failure output to inspect or patch the implementation before finalizing." + return None + def _read_text_excerpt(path: Path, limit: int = 1400) -> str: @@ -965,6 +1020,15 @@ def run( } ) previous_attributed = set() + finalization_nudge_state: dict[str, bool] = {} + known_verification_command = self._task_mode in { + TaskMode.FIX_FAILING_TEST, + TaskMode.FIX_LINT_OR_TYPE, + } + last_edit_turn = 0 + last_verification_turn = 0 + last_verification_failed = False + last_failure_followup_turn = 0 def _attributed_changed_files() -> list[str]: current = set(self._git_changed_files()) @@ -1204,6 +1268,25 @@ def _budget_reason( reason = _budget_reason(completed=True) if reason: return _finish_bounded(response, reason, reason == "completed") + final_text = "\n".join( + block.get("text", "") + for block in response.get("content", []) + if block.get("type") == "text" + ) + block_reason = final_answer_block_reason( + instruction=instruction, + final_text=final_text, + has_any_edit=bool(_change_summary()[0]), + changed_files=_change_summary()[2], + known_verification_command=known_verification_command, + verification_ran_after_last_edit=last_verification_turn >= last_edit_turn, + last_verification_failed=last_verification_failed, + had_corrective_action_after_last_failure=last_failure_followup_turn > last_verification_turn, + nudge_state=finalization_nudge_state, + ) + if block_reason: + messages.append({"role": "user", "content": [{"type": "text", "text": block_reason}]}) + continue transcript["final_assistant_content"] = response.get("content", []) transcript_path = self._save_transcript_and_link(transcript) post = self._run_post_execution_validation(_change_summary()[2]) @@ -1332,6 +1415,25 @@ def _budget_reason( reason = _budget_reason(completed=True) if reason: return _finish_bounded(response, reason, reason == "completed") + final_text = "\n".join( + block.get("text", "") + for block in response.get("content", []) + if block.get("type") == "text" + ) + block_reason = final_answer_block_reason( + instruction=instruction, + final_text=final_text, + has_any_edit=bool(_change_summary()[0]), + changed_files=_change_summary()[2], + known_verification_command=known_verification_command, + verification_ran_after_last_edit=last_verification_turn >= last_edit_turn, + last_verification_failed=last_verification_failed, + had_corrective_action_after_last_failure=last_failure_followup_turn > last_verification_turn, + nudge_state=finalization_nudge_state, + ) + if block_reason: + messages.append({"role": "user", "content": [{"type": "text", "text": block_reason}]}) + continue transcript["final_assistant_content"] = response.get("content", []) transcript_path = None if not self._planning_read_only: @@ -1401,10 +1503,18 @@ def _budget_reason( self._pending_verification = self._run_post_edit_verification( trigger=f"{tool_name} execution" ) + last_edit_turn = turns_used + last_failure_followup_turn = turns_used elif tool_name == "Bash": self._pending_verification = self._run_verification( trigger=f"{tool_name} execution" ) + last_verification_turn = turns_used + content_text = str(self._pending_verification) + last_verification_failed = ("status=fail" in content_text) or ("status=uncertain" in content_text) + elif tool_name in {"Read", "Grep", "Search", "GitDiff", "GitStatus"}: + if last_verification_failed and not result.get("is_error"): + last_failure_followup_turn = turns_used if result.get("is_error"): result_text = str(result.get("content", "")) diff --git a/villani_code/state_runtime.py b/villani_code/state_runtime.py index 9e03824d..dd37aba1 100644 --- a/villani_code/state_runtime.py +++ b/villani_code/state_runtime.py @@ -839,6 +839,24 @@ def _build_compact_validation_summary( "" ) +def _extract_visible_failure_lines(cmd_results: list[dict[str, Any]]) -> list[str]: + patterns = ("AssertionError", "assert", "expected", "actual", "got", "E ", 'File "') + picked: list[str] = [] + for result in cmd_results: + raw = "\n".join( + str(part) + for part in (result.get("stdout", ""), result.get("stderr", "")) + if part + ) + for line in raw.splitlines(): + if any(marker in line for marker in patterns): + normalized = line.rstrip() + if normalized and normalized not in picked: + picked.append(normalized) + if len(picked) >= 12: + return picked + return picked + def run_post_edit_verification(runner: Any, trigger: str = "edit") -> str: had_pending_retry = bool(getattr(runner, "_patch_sanity_retry_pending", False)) @@ -1071,6 +1089,13 @@ def run_verification(runner: Any, trigger: str = "edit") -> str: } ) if verification.status in {VerificationStatus.FAIL, VerificationStatus.UNCERTAIN}: + failure_lines = _extract_visible_failure_lines(cmd_results) + if failure_lines: + lines.append("visible_failure_lines:") + lines.extend(f"- {line}" for line in failure_lines) + lines.append( + "Treat visible test assertions as authoritative evidence. Do not guess replacement values when the expected value appears in the failure output." + ) runner.event_callback( { "type": "confidence_risk", From 82159bc6a83a7d319e255099f7612fe5d27fc930 Mon Sep 17 00:00:00 2001 From: mmprotest Date: Fri, 1 May 2026 12:22:55 +1000 Subject: [PATCH 2/3] Fix finalization gate order before completed budget return --- tests/test_bounded_execution.py | 55 +++++++++++++++++++++++++++++++++ tests/test_final_answer_gate.py | 37 ++++++++++++++++++++++ tests/test_state_runtime.py | 35 ++++++++++++++++++++- villani_code/state.py | 36 ++++++++++++++++----- villani_code/state_runtime.py | 10 ++++-- 5 files changed, 161 insertions(+), 12 deletions(-) diff --git a/tests/test_bounded_execution.py b/tests/test_bounded_execution.py index ed08c70f..ca4f3028 100644 --- a/tests/test_bounded_execution.py +++ b/tests/test_bounded_execution.py @@ -110,3 +110,58 @@ def test_villani_task_reports_completed_when_done(tmp_path: Path) -> None: assert result["execution"]["completed"] is True assert result["execution"]["terminated_reason"] == "completed" + + +def test_completed_budget_does_not_bypass_finalization_gate(tmp_path: Path) -> None: + runner = _runner( + tmp_path, + [ + {"role": "assistant", "content": [{"type": "text", "text": "I'll update x.py to fix this."}]}, + {"role": "assistant", "content": [{"type": "text", "text": "done"}]}, + ], + ) + budget = ExecutionBudget(max_turns=20, max_tool_calls=20, max_seconds=100.0, max_no_edit_turns=20, max_reconsecutive_recon_turns=20) + + result = runner.run("implement fix", execution_budget=budget) + + nudges = [ + block.get("text", "") + for msg in result["messages"] + if msg.get("role") == "user" + for block in msg.get("content", []) + if isinstance(block, dict) and block.get("type") == "text" + ] + assert any("no file was modified" in text for text in nudges) + assert result["execution"]["terminated_reason"] == "completed" + + +def test_failed_bash_verification_uses_exit_code_for_followup_gate(tmp_path: Path, monkeypatch) -> None: + monkeypatch.setattr("villani_code.state_runtime.run_verification", lambda _runner, _trigger="edit": "status=pass") + runner = _runner( + tmp_path, + [ + {"role": "assistant", "content": [{"type": "tool_use", "id": "1", "name": "Bash", "input": {"command": "python -c \"import sys; sys.exit(1)\""}}]}, + {"role": "assistant", "content": [{"type": "text", "text": "done"}]}, + {"role": "assistant", "content": [{"type": "text", "text": "done"}]}, + ], + ) + monkeypatch.setattr( + runner, + "_execute_tool_with_policy", + lambda tool_name, _tool_input, _tool_use_id, _msg_index: { + "content": "simulated bash run", + "is_error": False, + "exit_code": 1 if tool_name == "Bash" else 0, + }, + ) + budget = ExecutionBudget(max_turns=20, max_tool_calls=20, max_seconds=100.0, max_no_edit_turns=20, max_reconsecutive_recon_turns=20) + + result = runner.run("implement fix", execution_budget=budget) + nudges = [ + block.get("text", "") + for msg in result["messages"] + if msg.get("role") == "user" + for block in msg.get("content", []) + if isinstance(block, dict) and block.get("type") == "text" + ] + assert any("last verification failed" in text for text in nudges) diff --git a/tests/test_final_answer_gate.py b/tests/test_final_answer_gate.py index 986a3a6b..d5befe72 100644 --- a/tests/test_final_answer_gate.py +++ b/tests/test_final_answer_gate.py @@ -44,6 +44,43 @@ def test_vague_suggestion_does_not_block() -> None: assert reason is None +def test_new_concrete_edit_phrases_detected() -> None: + phrases = [ + "The fix is simply adding slug_handler to the registry.", + "Need to register the handler.", + "Add slug to REGISTRY.", + ] + for phrase in phrases: + reason = final_answer_block_reason( + instruction="implement fix", + final_text=phrase, + has_any_edit=False, + changed_files=[], + known_verification_command=True, + verification_ran_after_last_edit=True, + last_verification_failed=False, + had_corrective_action_after_last_failure=False, + nudge_state={}, + ) + assert reason is not None + + +def test_vague_add_suggestions_still_do_not_trigger() -> None: + for phrase in ["could add a handler", "might add a handler", "one option would be adding a handler"]: + reason = final_answer_block_reason( + instruction="implement fix", + final_text=phrase, + has_any_edit=False, + changed_files=[], + known_verification_command=True, + verification_ran_after_last_edit=True, + last_verification_failed=False, + had_corrective_action_after_last_failure=False, + nudge_state={}, + ) + assert reason is None + + def test_plan_only_task_does_not_block() -> None: reason = final_answer_block_reason( instruction="Plan only, do not implement.", diff --git a/tests/test_state_runtime.py b/tests/test_state_runtime.py index 16e604a7..19c18d80 100644 --- a/tests/test_state_runtime.py +++ b/tests/test_state_runtime.py @@ -199,7 +199,9 @@ def test_repeated_stale_verification_returns_compact_block(tmp_path: Path, monke _seed_repo(tmp_path) runner = Runner(client=DummyClient(), repo=tmp_path, model="m", stream=False, small_model=True) runner._verification_baseline_changed = set() - monkeypatch.setattr(state_runtime, "git_changed_files", lambda _repo: []) + (tmp_path / "tests").mkdir(exist_ok=True) + (tmp_path / "tests" / "test_x.py").write_text("def test_x():\n assert False\n", encoding="utf-8") + monkeypatch.setattr(state_runtime, "git_changed_files", lambda _repo: ["tests/test_x.py"]) first = runner._run_verification("edit") second = runner._run_verification("edit") @@ -238,6 +240,37 @@ def test_verification_detail_event_keeps_raw_trace(tmp_path: Path, monkeypatch: assert "last_validation_summary:" in out +def test_failed_verification_summary_includes_visible_failure_lines(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + from types import SimpleNamespace + from villani_code.autonomy import VerificationStatus + + _seed_repo(tmp_path) + runner = Runner(client=DummyClient(), repo=tmp_path, model="m", stream=False, small_model=True) + runner._verification_baseline_changed = set() + (tmp_path / "tests").mkdir(exist_ok=True) + (tmp_path / "tests" / "test_x.py").write_text("def test_x():\n assert False\n", encoding="utf-8") + monkeypatch.setattr(state_runtime, "git_changed_files", lambda _repo: ["tests/test_x.py"]) + + def fake_run(_cmd, **_kwargs): + return SimpleNamespace( + returncode=1, + stdout="E AssertionError: expected 2, got 3\nFile \"src/a.py\", line 10\nassert x == y", + stderr="", + ) + + monkeypatch.setattr(state_runtime.subprocess, "run", fake_run) + runner._verification_engine.verify = lambda *args, **kwargs: SimpleNamespace( + status=VerificationStatus.FAIL, + confidence_score=0.1, + findings=[], + summary="failed", + ) + out = runner._run_verification("edit") + assert "AssertionError" in out + assert "expected 2, got 3" in out + assert "Treat visible test assertions as authoritative evidence." in out + + def test_repeated_validation_updates_compact_state_without_repeated_prose( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: diff --git a/villani_code/state.py b/villani_code/state.py index f612ceaf..bca60ac1 100644 --- a/villani_code/state.py +++ b/villani_code/state.py @@ -73,11 +73,22 @@ def _dedupe_preserve(items: list[str]) -> list[str]: r"\bi(?:'ll| will)\s+(?:update|modify|change|patch|edit|add)\b", r"\bthe fix is to\b", r"\bchange the\b", + r"\bthe fix is simply adding\b", + r"\bthe fix is to add\b", + r"\brequires adding\b", + r"\bneed to add\b", + r"\bneed to register\b", + r"\bregister\b.{0,40}\bin\b", + r"\bmissing\b.{0,40}\bregistry\b", + r"\badd\b.{0,40}\bto registry\b", ) _HEDGING_PATTERNS = ( r"\bone possible fix would be\b", r"\byou could\b", r"\bi recommend\b", + r"\bcould add\b", + r"\bmight add\b", + r"\bone option would be\b", r"\bno code changes are needed\b", ) @@ -1265,9 +1276,6 @@ def _budget_reason( self.event_callback({"type": "benchmark_scope_reminder_injected", "task_id": self.benchmark_config.task_id, "reason": "no_meaningful_edit"}) messages.append({"role": "user", "content": [{"type": "text", "text": reminder}]}) continue - reason = _budget_reason(completed=True) - if reason: - return _finish_bounded(response, reason, reason == "completed") final_text = "\n".join( block.get("text", "") for block in response.get("content", []) @@ -1287,6 +1295,9 @@ def _budget_reason( if block_reason: messages.append({"role": "user", "content": [{"type": "text", "text": block_reason}]}) continue + reason = _budget_reason(completed=True) + if reason: + return _finish_bounded(response, reason, reason == "completed") transcript["final_assistant_content"] = response.get("content", []) transcript_path = self._save_transcript_and_link(transcript) post = self._run_post_execution_validation(_change_summary()[2]) @@ -1412,9 +1423,6 @@ def _budget_reason( self.event_callback({"type": "benchmark_scope_reminder_injected", "task_id": self.benchmark_config.task_id, "reason": "no_meaningful_edit"}) messages.append({"role": "user", "content": [{"type": "text", "text": reminder}]}) continue - reason = _budget_reason(completed=True) - if reason: - return _finish_bounded(response, reason, reason == "completed") final_text = "\n".join( block.get("text", "") for block in response.get("content", []) @@ -1434,6 +1442,9 @@ def _budget_reason( if block_reason: messages.append({"role": "user", "content": [{"type": "text", "text": block_reason}]}) continue + reason = _budget_reason(completed=True) + if reason: + return _finish_bounded(response, reason, reason == "completed") transcript["final_assistant_content"] = response.get("content", []) transcript_path = None if not self._planning_read_only: @@ -1510,8 +1521,17 @@ def _budget_reason( trigger=f"{tool_name} execution" ) last_verification_turn = turns_used - content_text = str(self._pending_verification) - last_verification_failed = ("status=fail" in content_text) or ("status=uncertain" in content_text) + exit_code = result.get("exit_code") + if isinstance(exit_code, int): + last_verification_failed = exit_code != 0 + elif result.get("is_error"): + last_verification_failed = True + else: + content_text = str(self._pending_verification) + tool_text = str(result.get("content", "")) + exit_match = re.search(r"exit(?:_code)?\s*[=:]\s*(-?\d+)", tool_text, re.IGNORECASE) + inferred_exit = int(exit_match.group(1)) if exit_match else 0 + last_verification_failed = inferred_exit != 0 or ("status=fail" in content_text) or ("status=uncertain" in content_text) elif tool_name in {"Read", "Grep", "Search", "GitDiff", "GitStatus"}: if last_verification_failed and not result.get("is_error"): last_failure_followup_turn = turns_used diff --git a/villani_code/state_runtime.py b/villani_code/state_runtime.py index dd37aba1..391a484c 100644 --- a/villani_code/state_runtime.py +++ b/villani_code/state_runtime.py @@ -1090,12 +1090,16 @@ def run_verification(runner: Any, trigger: str = "edit") -> str: ) if verification.status in {VerificationStatus.FAIL, VerificationStatus.UNCERTAIN}: failure_lines = _extract_visible_failure_lines(cmd_results) + failure_guidance = ( + "Treat visible test assertions as authoritative evidence. " + "Do not guess replacement values when the expected value appears in the failure output." + ) if failure_lines: lines.append("visible_failure_lines:") lines.extend(f"- {line}" for line in failure_lines) - lines.append( - "Treat visible test assertions as authoritative evidence. Do not guess replacement values when the expected value appears in the failure output." - ) + summary += "; visible_failure_lines=" + " | ".join(failure_lines[:4]) + summary += f"; guidance={failure_guidance}" + lines.append(failure_guidance) runner.event_callback( { "type": "confidence_risk", From 17c140094c3f5d4e003cfaf7d11cfa36f608c2f6 Mon Sep 17 00:00:00 2001 From: mmprotest Date: Fri, 1 May 2026 12:44:18 +1000 Subject: [PATCH 3/3] Widen concrete edit intent detection for short imperative fixes --- tests/test_final_answer_gate.py | 23 +++++++++++++++++++---- villani_code/state.py | 8 +++++++- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/tests/test_final_answer_gate.py b/tests/test_final_answer_gate.py index d5befe72..a2e1fa33 100644 --- a/tests/test_final_answer_gate.py +++ b/tests/test_final_answer_gate.py @@ -46,9 +46,16 @@ def test_vague_suggestion_does_not_block() -> None: def test_new_concrete_edit_phrases_detected() -> None: phrases = [ - "The fix is simply adding slug_handler to the registry.", - "Need to register the handler.", - "Add slug to REGISTRY.", + "The fix is straightforward: register it.", + "Found it. The markdown_handler is imported but never added to the REGISTRY dictionary. The fix is straightforward: register it.", + "The fix is simple: add it to the registry.", + "Need to register it.", + "Register it in the registry.", + "Add it to the registry.", + "Wire it into the registry.", + "Hook it up in the dispatch table.", + "Include it in the mapping.", + "Fix it by registering the handler.", ] for phrase in phrases: reason = final_answer_block_reason( @@ -66,7 +73,15 @@ def test_new_concrete_edit_phrases_detected() -> None: def test_vague_add_suggestions_still_do_not_trigger() -> None: - for phrase in ["could add a handler", "might add a handler", "one option would be adding a handler"]: + for phrase in [ + "One possible fix would be to register it.", + "You could add it to the registry.", + "I recommend registering it.", + "It might need to be added.", + "Maybe wire it into the registry.", + "This looks like a registry issue.", + "No code changes are needed.", + ]: reason = final_answer_block_reason( instruction="implement fix", final_text=phrase, diff --git a/villani_code/state.py b/villani_code/state.py index bca60ac1..619c65d5 100644 --- a/villani_code/state.py +++ b/villani_code/state.py @@ -81,6 +81,10 @@ def _dedupe_preserve(items: list[str]) -> list[str]: r"\bregister\b.{0,40}\bin\b", r"\bmissing\b.{0,40}\bregistry\b", r"\badd\b.{0,40}\bto registry\b", + r"\bthe fix is (?:straightforward|simple)\s*:\s*(?:add|register|wire|hook|include)\b", + r"\b(?:register|add|wire|include)\s+it\b", + r"\bhook it up\b", + r"\bfix it by (?:adding|registering|wiring|including)\b", ) _HEDGING_PATTERNS = ( r"\bone possible fix would be\b", @@ -89,6 +93,8 @@ def _dedupe_preserve(items: list[str]) -> list[str]: r"\bcould add\b", r"\bmight add\b", r"\bone option would be\b", + r"\bit might need to be\b", + r"\bmaybe\b", r"\bno code changes are needed\b", ) @@ -1032,7 +1038,7 @@ def run( ) previous_attributed = set() finalization_nudge_state: dict[str, bool] = {} - known_verification_command = self._task_mode in { + known_verification_command = bool(self.benchmark_config.visible_verification) or self._task_mode in { TaskMode.FIX_FAILING_TEST, TaskMode.FIX_LINT_OR_TYPE, }