diff --git a/bixbench/graders.py b/bixbench/graders.py index f2e81b0..f2b89af 100644 --- a/bixbench/graders.py +++ b/bixbench/graders.py @@ -56,11 +56,20 @@ class GradingFunction(BaseModel): """Base class for grading functions.""" def _parse_grade_response(self, response: str) -> GradeType: - """Parse the grade from LLM response.""" + """Parse the grade from LLM response. + + The grading prompts ask for one of `correct`, `incorrect` or `refused`. + Anything else, including a missing or malformed tag, is graded + incorrect. + """ match = re.search(r"\s*(.*?)\s*", response, re.DOTALL) grade = match[1].strip().lower() if match else None - return GradeType.CORRECT if grade == "correct" else GradeType.INCORRECT + if grade == GradeType.CORRECT: + return GradeType.CORRECT + if grade == GradeType.REFUSED: + return GradeType.REFUSED + return GradeType.INCORRECT async def _grade_str_verifier( self, diff --git a/tests/test_utils.py b/tests/test_utils.py index ab30405..674c260 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,8 +1,9 @@ import sys +from unittest.mock import AsyncMock, MagicMock import pytest -from bixbench.graders import MCQGrader +from bixbench.graders import GradeType, GradingFunction, MCQGrader, OpenEndedGrader from bixbench.utils import ( AnswerMode, compute_metrics, @@ -109,6 +110,83 @@ async def test_grade_mcq_answer( assert grade_result.refusal == expected_refusal +@pytest.mark.parametrize( + ("grader_response", "expected_grade_type"), + [ + pytest.param("correct", GradeType.CORRECT, id="correct"), + pytest.param("incorrect", GradeType.INCORRECT, id="incorrect"), + pytest.param("refused", GradeType.REFUSED, id="refused"), + # Both grading prompts advertise ` correct ` as the example + # output, so padding and casing must not change the verdict. + pytest.param( + " correct ", GradeType.CORRECT, id="correct_padded" + ), + pytest.param( + "Reasoning...\n\n REFUSED\n\n", + GradeType.REFUSED, + id="refused_uppercase_multiline", + ), + # A verdict we cannot read is a formatting failure by the grading model, + # not an abstention by the answerer, so it stays incorrect. + pytest.param( + "The predicted answer is wrong.", GradeType.INCORRECT, id="missing_tag" + ), + pytest.param(" refused", GradeType.INCORRECT, id="unclosed_tag"), + pytest.param( + " partially correct ", + GradeType.INCORRECT, + id="unexpected_verdict", + ), + ], +) +def test_parse_grade_response(grader_response: str, expected_grade_type: GradeType): + grade_type = GradingFunction()._parse_grade_response(grader_response) + + assert grade_type == expected_grade_type + + +def test_refusal_scores_zero_like_an_incorrect_answer(): + """Refusals score 0, so recording them cannot move accuracy or n_correct.""" + assert GradeType.CORRECT.numeric_grade == 1 + assert GradeType.INCORRECT.numeric_grade == 0 + assert GradeType.REFUSED.numeric_grade == 0 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("evaluation_mode", ["llm_verifier", "range_verifier"]) +@pytest.mark.parametrize( + ("grader_response", "expected_grade", "expected_correct", "expected_refusal"), + [ + pytest.param(" correct ", 1, True, False, id="correct"), + pytest.param(" incorrect ", 0, False, False, id="incorrect"), + pytest.param(" refused ", 0, False, True, id="refused"), + pytest.param("no grade tag here", 0, False, False, id="missing_tag"), + ], +) +async def test_open_ended_grading_records_refusal( + evaluation_mode: str, + grader_response: str, + expected_grade: int, + expected_correct: bool, + expected_refusal: bool, +): + """Both LLM grading paths must report a `refused` verdict as a refusal.""" + llm_client = AsyncMock() + llm_client.call_single.return_value = MagicMock(text=grader_response) + + grade_result = await OpenEndedGrader( + evaluation_mode=evaluation_mode, llm_client=llm_client + ).grade( + question="What is the capital of France?", + target="Paris", + predicted="I cannot answer this question.", + ) + + assert grade_result.grade == expected_grade + assert grade_result.correct is expected_correct + assert grade_result.refusal is expected_refusal + + @pytest.mark.parametrize( ("grades", "is_refused", "metrics"), [