fix(scripts): detect and auto-fix quiz answer-length bias using Claude API

scripts/check_quiz_bias.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+"""
+Check quiz.json files for answer-length bias.
+Flags questions where the correct answer is significantly longer than distractors.
+"""
+
+import json
+import os
+import sys
+from pathlib import Path
+
+THRESHOLD = 1.5  # correct answer is 1.5x longer than average distractor length
+
+
+def check_quiz(path: Path) -> list[dict]:
+    issues = []
+    try:
+        with open(path) as f:
+            data = json.load(f)
+    except (json.JSONDecodeError, OSError) as e:
+        print(f"  ERROR reading {path}: {e}")
+        return issues
+
+    # handle both {"questions": [...]} and [...] root formats
+    if isinstance(data, list):
+        questions = data
+    elif isinstance(data, dict):
+        questions = data.get("questions", [])
+    else:
+        return issues
+
+    for i, q in enumerate(questions):
+        if not isinstance(q, dict):
+            continue
+
+        options = q.get("options", [])
+        correct = q.get("correct", -1)
+
+        if not options or correct < 0 or correct >= len(options):
+            continue
+
+        correct_len = len(options[correct])
+        distractors = [len(options[j]) for j in range(len(options)) if j != correct]
+
+        if not distractors:
+            continue
+
+        avg_distractor_len = sum(distractors) / len(distractors)
+
+        if avg_distractor_len > 0 and correct_len / avg_distractor_len > THRESHOLD:
+            issues.append({
+                "file": str(path),
+                "question_index": i,
+                "question": q.get("question", "")[:80],
+                "correct_answer": options[correct][:80],
+                "correct_len": correct_len,
+                "avg_distractor_len": round(avg_distractor_len, 1),
+                "ratio": round(correct_len / avg_distractor_len, 2),
+            })
+
+    return issues
+
+
+def main():
+    root = Path(__file__).parent.parent
+    quiz_files = sorted(root.rglob("quiz.json"))
+
+    print(f"Scanning {len(quiz_files)} quiz files...\n")
+
+    all_issues = []
+    for path in quiz_files:
+        issues = check_quiz(path)
+        all_issues.extend(issues)
+
+    if not all_issues:
+        print("No answer-length bias detected.")
+        sys.exit(0)
+
+    print(f"Found {len(all_issues)} biased question(s):\n")
+    for issue in all_issues:
+        print(f"  File   : {issue['file']}")
+        print(f"  Q #{issue['question_index']}: {issue['question']}")
+        print(f"  Correct: {issue['correct_answer']}")
+        print(f"  Ratio  : {issue['ratio']}x (correct={issue['correct_len']} chars, avg distractor={issue['avg_distractor_len']} chars)")
+        print()
+
+    print(f"Total biased questions: {len(all_issues)}")
+    sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

scripts/fix_quiz_bias.py

@@ -0,0 +1,208 @@
+#!/usr/bin/env python3
+"""
+Fix answer-length bias in quiz.json files using Claude API.
+Set ANTHROPIC_API_KEY environment variable before running.
+
+Usage:
+    export ANTHROPIC_API_KEY=your_key_here
+    python3 scripts/fix_quiz_bias.py
+"""
+
+import json
+import sys
+import urllib.request
+import urllib.error
+from pathlib import Path
+from typing import List
+
+THRESHOLD = 1.5
+API_URL = "https://api.anthropic.com/v1/messages"
+MODEL = "claude-sonnet-4-6"
+TIMEOUT = 30  # seconds
+
+
+def get_api_key() -> str:
+    import os
+    key = os.environ.get("ANTHROPIC_API_KEY", "")
+    if not key:
+        print("Error: ANTHROPIC_API_KEY environment variable not set.")
+        print("Usage: export ANTHROPIC_API_KEY=your_key_here")
+        sys.exit(1)
+    return key
+
+
+def is_biased(options: List[str], correct: int) -> bool:
+    if not options or correct < 0 or correct >= len(options):
+        return False
+    correct_len = len(options[correct])
+    distractors = [len(options[j]) for j in range(len(options)) if j != correct]
+    if not distractors:
+        return False
+    avg = sum(distractors) / len(distractors)
+    return avg > 0 and correct_len / avg > THRESHOLD
+
+
+def is_valid_question(q: dict, expected_correct: int, expected_options_count: int) -> bool:
+    """Validate that rewritten question matches the canonical schema."""
+    if not isinstance(q, dict):
+        return False
+    if "question" not in q or "options" not in q or "correct" not in q:
+        return False
+    if not isinstance(q["options"], list):
+        return False
+    if len(q["options"]) != expected_options_count:
+        return False
+    if not all(isinstance(o, str) for o in q["options"]):
+        return False
+    if q["correct"] != expected_correct:
+        return False
+    return True
+
+
+def rewrite_question(question: dict, api_key: str) -> dict:
+    prompt = f"""You are fixing a quiz question that has answer-length bias.
+The correct answer is significantly longer than the wrong answers, which lets test-takers guess by length alone.
+
+Original question:
+{json.dumps(question, indent=2)}
+
+Rewrite the options so all answers are roughly similar in length.
+Keep the meaning and correctness intact. Keep the same correct index.
+Return only valid JSON in exactly this format, nothing else:
+{{
+  "stage": "{question.get('stage', 'check')}",
+  "question": "...",
+  "options": ["a", "b", "c", "d"],
+  "correct": {question.get('correct', 0)},
+  "explanation": "..."
+}}"""
+
+    payload = json.dumps({
+        "model": MODEL,
+        "max_tokens": 1024,
+        "messages": [{"role": "user", "content": prompt}]
+    }).encode()
+
+    req = urllib.request.Request(
+        API_URL,
+        data=payload,
+        headers={
+            "Content-Type": "application/json",
+            "x-api-key": api_key,
+            "anthropic-version": "2023-06-01",
+        },
+        method="POST"
+    )
+
+    try:
+        with urllib.request.urlopen(req, timeout=TIMEOUT) as resp:
+            data = json.loads(resp.read())
+            text = data["content"][0]["text"].strip()
+            if text.startswith("```"):
+                text = text.split("```")[1]
+                if text.startswith("json"):
+                    text = text[4:]
+            return json.loads(text.strip())
+    except urllib.error.HTTPError as e:
+        print(f"    HTTP error {e.code}: {e.reason} — keeping original")
+    except urllib.error.URLError as e:
+        print(f"    Network error: {e.reason} — keeping original")
+    except (KeyError, IndexError) as e:
+        print(f"    Unexpected response shape: {e} — keeping original")
+    except json.JSONDecodeError as e:
+        print(f"    Failed to parse API response as JSON: {e} — keeping original")
+    except Exception as e:
+        print(f"    Unexpected error: {e} — keeping original")
+
+    return question
+
+
+def fix_quiz(path: Path, api_key: str) -> int:
+    try:
+        with open(path) as f:
+            data = json.load(f)
+    except (json.JSONDecodeError, OSError) as e:
+        print(f"  ERROR reading {path}: {e}")
+        return 0
+
+    if isinstance(data, list):
+        questions = data
+    elif isinstance(data, dict):
+        questions = data.get("questions", [])
+    else:
+        return 0
+
+    fixed = 0
+    changed = False
+    for i, q in enumerate(questions):
+        if not isinstance(q, dict):
+            continue
+        options = q.get("options", [])
+        correct = q.get("correct", -1)
+        if not is_biased(options, correct):
+            continue
+
+        print(f"  Fixing Q#{i}: {q.get('question', '')[:60]}...")
+        rewritten = rewrite_question(q, api_key)
+
+        # validate before overwriting
+        if not is_valid_question(rewritten, correct, len(options)):
+            print(f"    Validation failed — keeping original")
+            continue
+
+        # skip if bias not actually fixed
+        if is_biased(rewritten.get("options", []), rewritten.get("correct", -1)):
+            print(f"    Bias not resolved — keeping original")
+            continue
+
+        questions[i] = rewritten
+        fixed += 1
+        changed = True
+
+    if changed:
+        try:
+            with open(path, "w") as f:
+                json.dump(data, f, indent=2)
+                f.write("\n")
+        except OSError as e:
+            print(f"  ERROR writing {path}: {e}")
+
+    return fixed
+
+
+def main():
+    api_key = get_api_key()
+    root = Path(__file__).parent.parent
+    quiz_files = sorted(root.rglob("quiz.json"))
+
+    print(f"Scanning {len(quiz_files)} quiz files...\n")
+
+    total_fixed = 0
+    for path in quiz_files:
+        try:
+            with open(path) as f:
+                data = json.load(f)
+        except (json.JSONDecodeError, OSError):
+            continue
+
+        if isinstance(data, list):
+            questions = data
+        elif isinstance(data, dict):
+            questions = data.get("questions", [])
+        else:
+            continue
+
+        biased = [
+            q for q in questions
+            if isinstance(q, dict) and is_biased(q.get("options", []), q.get("correct", -1))
+        ]
+
+        if biased:
+            print(f"Fixing {len(biased)} question(s) in {path.relative_to(root)}")
+            total_fixed += fix_quiz(path, api_key)
+
+    print(f"\nDone. Fixed {total_fixed} biased questions.")
+
+
+if __name__ == "__main__":
+    main()