From bcd974dd7da66a47b63f57fc7bf19027c88ad36a Mon Sep 17 00:00:00 2001 From: javy0107 Date: Tue, 23 Jun 2026 09:48:02 +0000 Subject: [PATCH 1/6] fix(scripts): add quiz answer-length bias validation script --- scripts/check_quiz_bias.py | 92 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 scripts/check_quiz_bias.py diff --git a/scripts/check_quiz_bias.py b/scripts/check_quiz_bias.py new file mode 100644 index 0000000000..5118e22caf --- /dev/null +++ b/scripts/check_quiz_bias.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +""" +Check quiz.json files for answer-length bias. +Flags questions where the correct answer is significantly longer than distractors. +""" + +import json +import os +import sys +from pathlib import Path + +THRESHOLD = 1.5 # correct answer is 1.5x longer than average distractor length + + +def check_quiz(path: Path) -> list[dict]: + issues = [] + try: + with open(path) as f: + data = json.load(f) + except (json.JSONDecodeError, OSError) as e: + print(f" ERROR reading {path}: {e}") + return issues + + # handle both {"questions": [...]} and [...] root formats + if isinstance(data, list): + questions = data + elif isinstance(data, dict): + questions = data.get("questions", []) + else: + return issues + + for i, q in enumerate(questions): + if not isinstance(q, dict): + continue + + options = q.get("options", []) + correct = q.get("correct", -1) + + if not options or correct < 0 or correct >= len(options): + continue + + correct_len = len(options[correct]) + distractors = [len(options[j]) for j in range(len(options)) if j != correct] + + if not distractors: + continue + + avg_distractor_len = sum(distractors) / len(distractors) + + if avg_distractor_len > 0 and correct_len / avg_distractor_len > THRESHOLD: + issues.append({ + "file": str(path), + "question_index": i, + "question": q.get("question", "")[:80], + "correct_answer": options[correct][:80], + "correct_len": correct_len, + "avg_distractor_len": round(avg_distractor_len, 1), + "ratio": round(correct_len / avg_distractor_len, 2), + }) + + return issues + + +def main(): + root = Path(__file__).parent.parent + quiz_files = sorted(root.rglob("quiz.json")) + + print(f"Scanning {len(quiz_files)} quiz files...\n") + + all_issues = [] + for path in quiz_files: + issues = check_quiz(path) + all_issues.extend(issues) + + if not all_issues: + print("No answer-length bias detected.") + sys.exit(0) + + print(f"Found {len(all_issues)} biased question(s):\n") + for issue in all_issues: + print(f" File : {issue['file']}") + print(f" Q #{issue['question_index']}: {issue['question']}") + print(f" Correct: {issue['correct_answer']}") + print(f" Ratio : {issue['ratio']}x (correct={issue['correct_len']} chars, avg distractor={issue['avg_distractor_len']} chars)") + print() + + print(f"Total biased questions: {len(all_issues)}") + sys.exit(1) + + +if __name__ == "__main__": + main() From 4970743a5b7cc1f79647bfe124d49387fde16a5d Mon Sep 17 00:00:00 2001 From: javy0107 Date: Wed, 24 Jun 2026 10:48:24 +0000 Subject: [PATCH 2/6] fix(scripts): add AI-powered quiz bias fixer using Claude API --- scripts/fix_quiz_bias.py | 144 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100644 scripts/fix_quiz_bias.py diff --git a/scripts/fix_quiz_bias.py b/scripts/fix_quiz_bias.py new file mode 100644 index 0000000000..526f3b338e --- /dev/null +++ b/scripts/fix_quiz_bias.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +""" +Fix answer-length bias in quiz.json files using Claude API. +Set ANTHROPIC_API_KEY environment variable before running. + +Usage: + export ANTHROPIC_API_KEY=your_key_here + python3 scripts/fix_quiz_bias.py +""" + +import json +import os +import sys +import urllib.request +import urllib.error +from pathlib import Path + +THRESHOLD = 1.5 +API_URL = "https://api.anthropic.com/v1/messages" +MODEL = "claude-sonnet-4-6" + + +def get_api_key() -> str: + key = os.environ.get("ANTHROPIC_API_KEY", "") + if not key: + print("Error: ANTHROPIC_API_KEY environment variable not set.") + print("Usage: export ANTHROPIC_API_KEY=your_key_here") + sys.exit(1) + return key + + +def is_biased(options: list[str], correct: int) -> bool: + if not options or correct < 0 or correct >= len(options): + return False + correct_len = len(options[correct]) + distractors = [len(options[j]) for j in range(len(options)) if j != correct] + if not distractors: + return False + avg = sum(distractors) / len(distractors) + return avg > 0 and correct_len / avg > THRESHOLD + + +def rewrite_question(question: dict, api_key: str) -> dict: + prompt = f"""You are fixing a quiz question that has answer-length bias. +The correct answer is significantly longer than the wrong answers, which lets test-takers guess by length alone. + +Original question: +{json.dumps(question, indent=2)} + +Rewrite the options so all answers are roughly similar in length. +Keep the meaning and correctness intact. Keep the same correct index. +Return only valid JSON in exactly this format, nothing else: +{{ + "stage": "{question.get('stage', 'check')}", + "question": "...", + "options": ["a", "b", "c", "d"], + "correct": {question.get('correct', 0)}, + "explanation": "..." +}}""" + + payload = json.dumps({ + "model": MODEL, + "max_tokens": 1024, + "messages": [{"role": "user", "content": prompt}] + }).encode() + + req = urllib.request.Request( + API_URL, + data=payload, + headers={ + "Content-Type": "application/json", + "x-api-key": api_key, + "anthropic-version": "2023-06-01", + }, + method="POST" + ) + + try: + with urllib.request.urlopen(req) as resp: + data = json.loads(resp.read()) + text = data["content"][0]["text"].strip() + # strip markdown fences if present + if text.startswith("```"): + text = text.split("```")[1] + if text.startswith("json"): + text = text[4:] + return json.loads(text.strip()) + except (urllib.error.HTTPError, json.JSONDecodeError) as e: + print(f" API error: {e} — keeping original") + return question + + +def fix_quiz(path: Path, api_key: str) -> int: + with open(path) as f: + data = json.load(f) + + if isinstance(data, list): + questions = data + elif isinstance(data, dict): + questions = data.get("questions", []) + else: + return 0 + + fixed = 0 + for i, q in enumerate(questions): + if not isinstance(q, dict): + continue + options = q.get("options", []) + correct = q.get("correct", -1) + if is_biased(options, correct): + print(f" Fixing Q#{i}: {q.get('question', '')[:60]}...") + questions[i] = rewrite_question(q, api_key) + fixed += 1 + + with open(path, "w") as f: + json.dump(data, f, indent=2) + f.write("\n") + + return fixed + + +def main(): + api_key = get_api_key() + root = Path(__file__).parent.parent + quiz_files = sorted(root.rglob("quiz.json")) + + print(f"Scanning {len(quiz_files)} quiz files...\n") + + total_fixed = 0 + for path in quiz_files: + issues = [ + q for q in (json.load(open(path)) if isinstance(json.load(open(path)), list) + else json.load(open(path)).get("questions", [])) + if isinstance(q, dict) and is_biased(q.get("options", []), q.get("correct", -1)) + ] + if issues: + print(f"Fixing {len(issues)} question(s) in {path.relative_to(root)}") + total_fixed += fix_quiz(path, api_key) + + print(f"\nDone. Fixed {total_fixed} biased questions.") + + +if __name__ == "__main__": + main() From 33bf278e5a2e7dbd9dbe099326bc688de161a409 Mon Sep 17 00:00:00 2001 From: javy0107 Date: Wed, 24 Jun 2026 10:59:39 +0000 Subject: [PATCH 3/6] fix(scripts): add timeout and broaden exception handling in fix_quiz_bias --- scripts/fix_quiz_bias.py | 59 ++++++++++++++++++++++++++++++---------- 1 file changed, 44 insertions(+), 15 deletions(-) diff --git a/scripts/fix_quiz_bias.py b/scripts/fix_quiz_bias.py index 526f3b338e..3b39cdbc6e 100644 --- a/scripts/fix_quiz_bias.py +++ b/scripts/fix_quiz_bias.py @@ -18,6 +18,7 @@ THRESHOLD = 1.5 API_URL = "https://api.anthropic.com/v1/messages" MODEL = "claude-sonnet-4-6" +TIMEOUT = 30 # seconds def get_api_key() -> str: @@ -76,23 +77,35 @@ def rewrite_question(question: dict, api_key: str) -> dict: ) try: - with urllib.request.urlopen(req) as resp: + with urllib.request.urlopen(req, timeout=TIMEOUT) as resp: data = json.loads(resp.read()) text = data["content"][0]["text"].strip() - # strip markdown fences if present if text.startswith("```"): text = text.split("```")[1] if text.startswith("json"): text = text[4:] return json.loads(text.strip()) - except (urllib.error.HTTPError, json.JSONDecodeError) as e: - print(f" API error: {e} — keeping original") - return question + except urllib.error.HTTPError as e: + print(f" HTTP error {e.code}: {e.reason} — keeping original") + except urllib.error.URLError as e: + print(f" Network error: {e.reason} — keeping original") + except (KeyError, IndexError) as e: + print(f" Unexpected response shape: {e} — keeping original") + except json.JSONDecodeError as e: + print(f" Failed to parse API response as JSON: {e} — keeping original") + except Exception as e: + print(f" Unexpected error: {e} — keeping original") + + return question def fix_quiz(path: Path, api_key: str) -> int: - with open(path) as f: - data = json.load(f) + try: + with open(path) as f: + data = json.load(f) + except (json.JSONDecodeError, OSError) as e: + print(f" ERROR reading {path}: {e}") + return 0 if isinstance(data, list): questions = data @@ -112,9 +125,12 @@ def fix_quiz(path: Path, api_key: str) -> int: questions[i] = rewrite_question(q, api_key) fixed += 1 - with open(path, "w") as f: - json.dump(data, f, indent=2) - f.write("\n") + try: + with open(path, "w") as f: + json.dump(data, f, indent=2) + f.write("\n") + except OSError as e: + print(f" ERROR writing {path}: {e}") return fixed @@ -128,13 +144,26 @@ def main(): total_fixed = 0 for path in quiz_files: - issues = [ - q for q in (json.load(open(path)) if isinstance(json.load(open(path)), list) - else json.load(open(path)).get("questions", [])) + try: + with open(path) as f: + data = json.load(f) + except (json.JSONDecodeError, OSError): + continue + + if isinstance(data, list): + questions = data + elif isinstance(data, dict): + questions = data.get("questions", []) + else: + continue + + biased = [ + q for q in questions if isinstance(q, dict) and is_biased(q.get("options", []), q.get("correct", -1)) ] - if issues: - print(f"Fixing {len(issues)} question(s) in {path.relative_to(root)}") + + if biased: + print(f"Fixing {len(biased)} question(s) in {path.relative_to(root)}") total_fixed += fix_quiz(path, api_key) print(f"\nDone. Fixed {total_fixed} biased questions.") From 82a7f9778dc5c65e91a2dedda6922468f1ee57e0 Mon Sep 17 00:00:00 2001 From: javy0107 Date: Wed, 24 Jun 2026 11:12:18 +0000 Subject: [PATCH 4/6] fix(scripts): validate rewritten questions, remove unused os import, fix Python 3.9+ type hints --- scripts/fix_quiz_bias.py | 61 +++++++++++++++++++++++++++++++--------- 1 file changed, 48 insertions(+), 13 deletions(-) diff --git a/scripts/fix_quiz_bias.py b/scripts/fix_quiz_bias.py index 3b39cdbc6e..20a097f979 100644 --- a/scripts/fix_quiz_bias.py +++ b/scripts/fix_quiz_bias.py @@ -9,11 +9,11 @@ """ import json -import os import sys import urllib.request import urllib.error from pathlib import Path +from typing import List THRESHOLD = 1.5 API_URL = "https://api.anthropic.com/v1/messages" @@ -22,7 +22,10 @@ def get_api_key() -> str: - key = os.environ.get("ANTHROPIC_API_KEY", "") + key = sys.argv[1] if len(sys.argv) > 1 else "" + if not key: + import os + key = os.environ.get("ANTHROPIC_API_KEY", "") if not key: print("Error: ANTHROPIC_API_KEY environment variable not set.") print("Usage: export ANTHROPIC_API_KEY=your_key_here") @@ -30,7 +33,7 @@ def get_api_key() -> str: return key -def is_biased(options: list[str], correct: int) -> bool: +def is_biased(options: List[str], correct: int) -> bool: if not options or correct < 0 or correct >= len(options): return False correct_len = len(options[correct]) @@ -41,6 +44,21 @@ def is_biased(options: list[str], correct: int) -> bool: return avg > 0 and correct_len / avg > THRESHOLD +def is_valid_question(q: dict, expected_correct: int, expected_options_count: int) -> bool: + """Validate that rewritten question matches the canonical schema.""" + if not isinstance(q, dict): + return False + if "question" not in q or "options" not in q or "correct" not in q: + return False + if not isinstance(q["options"], list): + return False + if len(q["options"]) != expected_options_count: + return False + if q["correct"] != expected_correct: + return False + return True + + def rewrite_question(question: dict, api_key: str) -> dict: prompt = f"""You are fixing a quiz question that has answer-length bias. The correct answer is significantly longer than the wrong answers, which lets test-takers guess by length alone. @@ -115,22 +133,39 @@ def fix_quiz(path: Path, api_key: str) -> int: return 0 fixed = 0 + changed = False for i, q in enumerate(questions): if not isinstance(q, dict): continue options = q.get("options", []) correct = q.get("correct", -1) - if is_biased(options, correct): - print(f" Fixing Q#{i}: {q.get('question', '')[:60]}...") - questions[i] = rewrite_question(q, api_key) - fixed += 1 + if not is_biased(options, correct): + continue - try: - with open(path, "w") as f: - json.dump(data, f, indent=2) - f.write("\n") - except OSError as e: - print(f" ERROR writing {path}: {e}") + print(f" Fixing Q#{i}: {q.get('question', '')[:60]}...") + rewritten = rewrite_question(q, api_key) + + # validate before overwriting + if not is_valid_question(rewritten, correct, len(options)): + print(f" Validation failed — keeping original") + continue + + # skip if bias not actually fixed + if is_biased(rewritten.get("options", []), rewritten.get("correct", -1)): + print(f" Bias not resolved — keeping original") + continue + + questions[i] = rewritten + fixed += 1 + changed = True + + if changed: + try: + with open(path, "w") as f: + json.dump(data, f, indent=2) + f.write("\n") + except OSError as e: + print(f" ERROR writing {path}: {e}") return fixed From 633e82655be8e5bb1ebb3e1d2ca71052ad6bd929 Mon Sep 17 00:00:00 2001 From: javy0107 Date: Wed, 24 Jun 2026 11:20:37 +0000 Subject: [PATCH 5/6] fix(scripts): remove sys.argv API key path to avoid process exposure --- scripts/fix_quiz_bias.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/scripts/fix_quiz_bias.py b/scripts/fix_quiz_bias.py index 20a097f979..358f60c9fc 100644 --- a/scripts/fix_quiz_bias.py +++ b/scripts/fix_quiz_bias.py @@ -22,10 +22,8 @@ def get_api_key() -> str: - key = sys.argv[1] if len(sys.argv) > 1 else "" - if not key: - import os - key = os.environ.get("ANTHROPIC_API_KEY", "") + import os + key = os.environ.get("ANTHROPIC_API_KEY", "") if not key: print("Error: ANTHROPIC_API_KEY environment variable not set.") print("Usage: export ANTHROPIC_API_KEY=your_key_here") From 8c29b2cfecb2efe823b974215ce396c67ede437d Mon Sep 17 00:00:00 2001 From: javy0107 Date: Wed, 24 Jun 2026 11:36:20 +0000 Subject: [PATCH 6/6] fix(scripts): validate options elements are strings in is_valid_question --- scripts/fix_quiz_bias.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/fix_quiz_bias.py b/scripts/fix_quiz_bias.py index 358f60c9fc..0489af3090 100644 --- a/scripts/fix_quiz_bias.py +++ b/scripts/fix_quiz_bias.py @@ -52,6 +52,8 @@ def is_valid_question(q: dict, expected_correct: int, expected_options_count: in return False if len(q["options"]) != expected_options_count: return False + if not all(isinstance(o, str) for o in q["options"]): + return False if q["correct"] != expected_correct: return False return True