From e8f4ef8df2134a229de51443b33448b088cd9cb3 Mon Sep 17 00:00:00 2001 From: Kyle Wheeler Date: Mon, 9 Nov 2026 14:42:58 +0000 Subject: [PATCH 1/2] feat: sync with internal-pipette-scores through 14756e4 Bring over the internal changes since the public snapshot (internal e67656a), internal PRs #2-#15: - VLM accuracy evals (7 benchmarks): scoring and dataset builders - ASR evals (Open ASR Leaderboard English, FLEURS): datasets, prompt profiles, WER scoring - eval packages that pipette-mgmt serves - prompt profiles and reasoning parsing for more VL models - MATH-500 2026.10.1 graded as liquid_evals' math500 task - ifbench scored with the liquid_evals checkers (vendor/ifbench-checkers) - ifstruct bumps for nested JSON and oversized numbers - CODEOWNERS Kept private: - ECR publishing in ci.yml (AWS account, IAM role); the public GHCR workflow stays, with only the wheel-smoke check for the patched ifbench-checkers copy ported. - logs/ (Slurm job output) - names of the private CDK repo and the eval-datasets bucket, replaced with "the organization's private storage" - names of internal tools and repos (eval scheduler, other harnesses, the VLMEvalKit fork), replaced with generic descriptions --- .dockerignore | 4 + .github/CODEOWNERS | 14 + .github/workflows/ci.yml | 3 +- .gitignore | 20 + NOTICE | 39 +- README.md | 9 +- THIRD_PARTY_LICENSES.md | 367 ++- docs/dataset-licenses.md | 90 + packages/pipette-scores/docs/asr-evals.md | 614 +++++ packages/pipette-scores/docs/datasets.md | 142 +- packages/pipette-scores/docs/httpapi.md | 148 +- packages/pipette-scores/docs/packages.md | 55 + packages/pipette-scores/docs/scoring.md | 130 +- .../pipette-scores/pipette_scores/api/app.py | 332 ++- packages/pipette-scores/pipette_scores/cli.py | 150 +- .../dataset_catalog/__init__.py | 83 +- .../dataset_catalog/_asr_audio.py | 166 ++ .../dataset_catalog/_asr_build.py | 385 +++ .../dataset_catalog/_asr_common.py | 444 ++++ .../dataset_catalog/_vlm_build.py | 581 +++++ .../dataset_catalog/_vlm_common.py | 193 ++ .../dataset_catalog/asr_datasets.py | 290 +++ .../dataset_catalog/asr_prompt_profiles.py | 346 +++ .../dataset_catalog/math_500.py | 22 +- .../dataset_catalog/prompt_profiles.py | 337 +++ .../pipette-scores/pipette_scores/hashing.py | 37 + .../pipette-scores/pipette_scores/packages.py | 411 +++ .../pipette_scores/scoring/__init__.py | 45 +- .../pipette_scores/scoring/asr/NOTICE.md | 85 + .../pipette_scores/scoring/asr/__init__.py | 1 + .../pipette_scores/scoring/asr/align.py | 76 + .../pipette_scores/scoring/asr/evaluate.py | 150 ++ .../pipette_scores/scoring/asr/normalize.py | 88 + .../asr/open_asr_normalizer/__init__.py | 0 .../english_abbreviations.py | 1934 ++++++++++++++ .../asr/open_asr_normalizer/multilingual.py | 132 + .../asr/open_asr_normalizer/normalizer.py | 753 ++++++ .../pipette_scores/scoring/asr/suite.py | 101 + .../pipette_scores/scoring/asr/wer.py | 179 ++ .../pipette_scores/scoring/completion_text.py | 364 +++ .../scoring/gpqa_diamond/__init__.py | 24 +- .../scoring/ifbench/__init__.py | 29 +- .../scoring/ifbench/upstream.py | 30 +- .../pipette_scores/scoring/ifstruct.py | 28 +- .../scoring/math_500/__init__.py | 142 +- .../scoring/math_500/liquid_evals_parser.py | 717 ++++++ .../pipette_scores/scoring/mmbench_en_dev.py | 83 + .../pipette_scores/scoring/mme.py | 89 + .../pipette_scores/scoring/mmmu_val.py | 68 + .../pipette_scores/scoring/mmstar.py | 57 + .../pipette_scores/scoring/ocrbench.py | 73 + .../pipette_scores/scoring/pope.py | 84 + .../pipette_scores/scoring/profile_answers.py | 118 + .../pipette_scores/scoring/realworldqa.py | 48 + .../pipette_scores/scoring/vlm_answer.py | 181 ++ .../scoring/vlm_upstream/PORTING_NOTES.md | 198 ++ .../scoring/vlm_upstream/__init__.py | 12 + .../scoring/vlm_upstream/matching_util.py | 91 + .../scoring/vlm_upstream/multiple_choice.py | 63 + .../scoring/vlm_upstream/ocrbench.py | 89 + .../scoring/vlm_upstream/yorn.py | 141 ++ .../pipette-scores/pipette_scores/types.py | 137 +- packages/pipette-scores/pyproject.toml | 69 +- .../scripts/build_asr_datasets.py | 74 + .../scripts/build_math_500_dataset.py | 153 +- .../scripts/build_mmbench_en_dev_dataset.py | 54 + .../scripts/build_mme_dataset.py | 52 + .../scripts/build_mmmu_val_dataset.py | 63 + .../scripts/build_mmstar_dataset.py | 47 + .../scripts/build_ocrbench_dataset.py | 51 + .../scripts/build_pope_dataset.py | 51 + .../scripts/build_realworldqa_dataset.py | 57 + .../scripts/build_vlm_datasets.sbatch | 50 + .../gen_math_500_liquid_evals_parity.py | 247 ++ .../scripts/setup_asr_reference_env.sh | 38 + .../scripts/split_asr_dataset.py | 55 + .../scripts/verify_asr_datasets.py | 60 + .../scripts/verify_vlm_datasets.py | 275 ++ .../tests/asr_public_reference.py | 590 +++++ .../tests/asr_public_reference_runner.py | 128 + .../tests/fixtures/asr_public_reference.json | 274 ++ .../math_500_liquid_evals_parity.json | 988 ++++++++ packages/pipette-scores/tests/test_api.py | 412 ++- .../pipette-scores/tests/test_asr_evals.py | 1154 +++++++++ .../pipette-scores/tests/test_asr_parts.py | 214 ++ .../tests/test_asr_public_reference.py | 470 ++++ .../pipette-scores/tests/test_asr_scoring.py | 270 ++ .../pipette-scores/tests/test_asr_suite.py | 123 + .../tests/test_build_math_500_dataset.py | 102 + .../tests/test_dataset_catalog.py | 27 +- .../pipette-scores/tests/test_gpqa_diamond.py | 28 + packages/pipette-scores/tests/test_hashing.py | 98 + packages/pipette-scores/tests/test_ifbench.py | 131 + .../pipette-scores/tests/test_ifstruct.py | 69 + .../pipette-scores/tests/test_math_500.py | 139 +- .../test_math_500_liquid_evals_parity.py | 66 + .../pipette-scores/tests/test_packages.py | 303 +++ .../tests/test_profile_answers.py | 269 ++ .../tests/test_prompt_profiles.py | 406 +++ .../pipette-scores/tests/test_reasoning.py | 644 +++++ packages/pipette-scores/tests/test_scoring.py | 112 + packages/pipette-scores/tests/test_text.py | 255 +- .../pipette-scores/tests/test_vlm_answer.py | 241 ++ .../tests/test_vlm_dataset_builders.py | 461 ++++ .../tests/test_vlm_dataset_loader.py | 149 ++ .../pipette-scores/tests/test_vlm_scoring.py | 641 +++++ pyproject.toml | 4 + uv.lock | 151 +- vendor/ifbench-checkers/LICENSE | 201 ++ vendor/ifbench-checkers/README.md | 73 + vendor/ifbench-checkers/evaluation_lib.py | 240 ++ vendor/ifbench-checkers/instructions.py | 2250 +++++++++++++++++ .../ifbench-checkers/instructions_registry.py | 80 + vendor/ifbench-checkers/instructions_util.py | 1665 ++++++++++++ vendor/ifstruct | 2 +- 115 files changed, 26455 insertions(+), 193 deletions(-) create mode 100644 .github/CODEOWNERS create mode 100644 docs/dataset-licenses.md create mode 100644 packages/pipette-scores/docs/asr-evals.md create mode 100644 packages/pipette-scores/docs/packages.md create mode 100644 packages/pipette-scores/pipette_scores/dataset_catalog/_asr_audio.py create mode 100644 packages/pipette-scores/pipette_scores/dataset_catalog/_asr_build.py create mode 100644 packages/pipette-scores/pipette_scores/dataset_catalog/_asr_common.py create mode 100644 packages/pipette-scores/pipette_scores/dataset_catalog/_vlm_build.py create mode 100755 packages/pipette-scores/pipette_scores/dataset_catalog/_vlm_common.py create mode 100644 packages/pipette-scores/pipette_scores/dataset_catalog/asr_datasets.py create mode 100644 packages/pipette-scores/pipette_scores/dataset_catalog/asr_prompt_profiles.py create mode 100644 packages/pipette-scores/pipette_scores/dataset_catalog/prompt_profiles.py create mode 100644 packages/pipette-scores/pipette_scores/packages.py create mode 100644 packages/pipette-scores/pipette_scores/scoring/asr/NOTICE.md create mode 100644 packages/pipette-scores/pipette_scores/scoring/asr/__init__.py create mode 100644 packages/pipette-scores/pipette_scores/scoring/asr/align.py create mode 100644 packages/pipette-scores/pipette_scores/scoring/asr/evaluate.py create mode 100644 packages/pipette-scores/pipette_scores/scoring/asr/normalize.py create mode 100644 packages/pipette-scores/pipette_scores/scoring/asr/open_asr_normalizer/__init__.py create mode 100644 packages/pipette-scores/pipette_scores/scoring/asr/open_asr_normalizer/english_abbreviations.py create mode 100644 packages/pipette-scores/pipette_scores/scoring/asr/open_asr_normalizer/multilingual.py create mode 100644 packages/pipette-scores/pipette_scores/scoring/asr/open_asr_normalizer/normalizer.py create mode 100644 packages/pipette-scores/pipette_scores/scoring/asr/suite.py create mode 100644 packages/pipette-scores/pipette_scores/scoring/asr/wer.py create mode 100644 packages/pipette-scores/pipette_scores/scoring/math_500/liquid_evals_parser.py create mode 100644 packages/pipette-scores/pipette_scores/scoring/mmbench_en_dev.py create mode 100644 packages/pipette-scores/pipette_scores/scoring/mme.py create mode 100644 packages/pipette-scores/pipette_scores/scoring/mmmu_val.py create mode 100644 packages/pipette-scores/pipette_scores/scoring/mmstar.py create mode 100644 packages/pipette-scores/pipette_scores/scoring/ocrbench.py create mode 100644 packages/pipette-scores/pipette_scores/scoring/pope.py create mode 100644 packages/pipette-scores/pipette_scores/scoring/profile_answers.py create mode 100644 packages/pipette-scores/pipette_scores/scoring/realworldqa.py create mode 100644 packages/pipette-scores/pipette_scores/scoring/vlm_answer.py create mode 100644 packages/pipette-scores/pipette_scores/scoring/vlm_upstream/PORTING_NOTES.md create mode 100644 packages/pipette-scores/pipette_scores/scoring/vlm_upstream/__init__.py create mode 100644 packages/pipette-scores/pipette_scores/scoring/vlm_upstream/matching_util.py create mode 100644 packages/pipette-scores/pipette_scores/scoring/vlm_upstream/multiple_choice.py create mode 100644 packages/pipette-scores/pipette_scores/scoring/vlm_upstream/ocrbench.py create mode 100644 packages/pipette-scores/pipette_scores/scoring/vlm_upstream/yorn.py create mode 100644 packages/pipette-scores/scripts/build_asr_datasets.py create mode 100644 packages/pipette-scores/scripts/build_mmbench_en_dev_dataset.py create mode 100644 packages/pipette-scores/scripts/build_mme_dataset.py create mode 100644 packages/pipette-scores/scripts/build_mmmu_val_dataset.py create mode 100644 packages/pipette-scores/scripts/build_mmstar_dataset.py create mode 100644 packages/pipette-scores/scripts/build_ocrbench_dataset.py create mode 100644 packages/pipette-scores/scripts/build_pope_dataset.py create mode 100644 packages/pipette-scores/scripts/build_realworldqa_dataset.py create mode 100644 packages/pipette-scores/scripts/build_vlm_datasets.sbatch create mode 100644 packages/pipette-scores/scripts/gen_math_500_liquid_evals_parity.py create mode 100755 packages/pipette-scores/scripts/setup_asr_reference_env.sh create mode 100644 packages/pipette-scores/scripts/split_asr_dataset.py create mode 100644 packages/pipette-scores/scripts/verify_asr_datasets.py create mode 100644 packages/pipette-scores/scripts/verify_vlm_datasets.py create mode 100644 packages/pipette-scores/tests/asr_public_reference.py create mode 100644 packages/pipette-scores/tests/asr_public_reference_runner.py create mode 100644 packages/pipette-scores/tests/fixtures/asr_public_reference.json create mode 100644 packages/pipette-scores/tests/fixtures/math_500_liquid_evals_parity.json create mode 100644 packages/pipette-scores/tests/test_asr_evals.py create mode 100644 packages/pipette-scores/tests/test_asr_parts.py create mode 100644 packages/pipette-scores/tests/test_asr_public_reference.py create mode 100644 packages/pipette-scores/tests/test_asr_scoring.py create mode 100644 packages/pipette-scores/tests/test_asr_suite.py create mode 100644 packages/pipette-scores/tests/test_build_math_500_dataset.py create mode 100644 packages/pipette-scores/tests/test_hashing.py create mode 100644 packages/pipette-scores/tests/test_math_500_liquid_evals_parity.py create mode 100644 packages/pipette-scores/tests/test_packages.py create mode 100644 packages/pipette-scores/tests/test_profile_answers.py create mode 100644 packages/pipette-scores/tests/test_prompt_profiles.py create mode 100644 packages/pipette-scores/tests/test_reasoning.py create mode 100644 packages/pipette-scores/tests/test_vlm_answer.py create mode 100644 packages/pipette-scores/tests/test_vlm_dataset_builders.py create mode 100644 packages/pipette-scores/tests/test_vlm_dataset_loader.py create mode 100644 packages/pipette-scores/tests/test_vlm_scoring.py create mode 100644 vendor/ifbench-checkers/LICENSE create mode 100644 vendor/ifbench-checkers/README.md create mode 100644 vendor/ifbench-checkers/evaluation_lib.py create mode 100644 vendor/ifbench-checkers/instructions.py create mode 100644 vendor/ifbench-checkers/instructions_registry.py create mode 100644 vendor/ifbench-checkers/instructions_util.py diff --git a/.dockerignore b/.dockerignore index aec9985..2e0ff60 100644 --- a/.dockerignore +++ b/.dockerignore @@ -23,3 +23,7 @@ datasets/math_500/ # IFBench — keep the third-party test set out of the image; materialized at runtime. datasets/ifbench/ + +# ASR datasets (ESB, FLEURS: ~20 GB, SPGISpeech non-commercial, GigaSpeech +# gated) — never bake them into the image; materialized + mounted at runtime. +datasets/asr_*/ diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..3c982c3 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,14 @@ +# Code owners for pipette-scores. +# +# GitHub requests a review from these accounts on every pull request. A merge +# to the default branch requires an approval from one of them. These same +# accounts can merge their own pull requests without an approval. +# +# The maintainers accountable for the codebase hold ownership. This set is +# narrower than the accounts that hold admin through corporate team +# membership. +# +# Several accounts on one line mean that any single one of them satisfies the +# requirement. + +* @m3m0ryh0l3 @ykhrustalev @ovenmitt @iamstuffed diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8778e46..71624af 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -74,7 +74,7 @@ jobs: # The installed wheel loads IFBench from the bundled # `scoring/ifbench/_upstream` copy (force-included at build time); the editable - # installs above only hit the submodule fallback. Do a non-editable install + # installs above only hit the vendor/ifbench-checkers fallback. Do a non-editable install # (mirrors the Docker build) and import through the bundled path, so a # packaging regression in the force-include block is caught. wheel-smoke: @@ -99,6 +99,7 @@ jobs: f = evaluation_lib.__file__.replace("\\", "/") assert "site-packages" in f, f"not from installed pkg: {f}" assert "scoring/ifbench/_upstream" in f, f"not the bundled copy: {f}" + assert "pipette-scores:" in open(f).read(), f"not the patched vendor/ifbench-checkers copy: {f}" assert callable(test_instruction_following_loose) print("bundled IFBench _upstream OK:", f) PY diff --git a/.gitignore b/.gitignore index 7dab951..e4d4cf8 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,23 @@ datasets/math_500/ # others. The builder (scripts/build_ifbench_dataset.py) materializes it from the # vendored vendor/ifbench submodule. Not committing it means we don't redistribute. datasets/ifbench/ + +# The 7 VLM benchmark datasets are builder-only: a license audit concluded none +# may be redistributed in transformed form from this repo (RealWorldQA is +# CC-BY-ND; the rest have non-commercial / uncleared-image terms — see +# docs/dataset-licenses.md). The builders (scripts/build__dataset.py) +# materialize them locally / at deploy from VLMEvalKit's pinned upstream TSVs; +# they must NEVER be committed. +datasets/mme/ +datasets/mmstar/ +datasets/ocrbench/ +datasets/mmmu_val/ +datasets/realworldqa/ +datasets/mmbench_en_dev/ +datasets/pope/ + +# The 16 ASR datasets (Open ASR Leaderboard, FLEURS) are builder-only too: SPGISpeech is +# non-commercial and GigaSpeech gated (docs/dataset-licenses.md). The builder +# (scripts/build_asr_datasets.py) refuses an output inside this checkout; this +# is the backstop for one copied in. They must NEVER be committed. +datasets/asr_*/ diff --git a/NOTICE b/NOTICE index c126951..dfeb975 100644 --- a/NOTICE +++ b/NOTICE @@ -8,12 +8,42 @@ Vendored verbatim mathematical equivalence grader, from openai/prm800k (MIT, (c) 2023 OpenAI). Its math_normalize.py is, per its header, largely copied from Hendrycks' MATH `math_equivalence` (hendrycks/math, MIT, (c) 2021 Dan Hendrycks). -- vendor/ifbench — the upstream IFBench scoring source, from allenai/IFBench +- vendor/ifbench — the upstream IFBench repository, from allenai/IFBench (Apache-2.0), vendored as a git submodule (LICENSE at vendor/ifbench/LICENSE). - At build time it is bundled into the wheel under + Only its test set is used. +- packages/pipette-scores/pipette_scores/scoring/asr/open_asr_normalizer/ — the Open + ASR Leaderboard's English text normalizer (normalizer.py, english_abbreviations.py), + from huggingface/open_asr_leaderboard @ 67e8bd6acea240819ad67080f6f31e15d4a90da5 + (Apache-2.0, (c) 2022 The OpenAI team and The HuggingFace Team; mostly copied, per + its header, from openai/whisper, MIT, (c) 2022 OpenAI), and its multilingual + normalizer and compound-pair alignment (multilingual.py: unchanged excerpts of + normalizer/data_utils.py and normalizer/eval_utils.py @ + 62a457c5c32d0de4df77e9470308aae7d45b055f, Apache-2.0). + Provenance and hashes of the ASR files: scoring/asr/NOTICE.md. + +Vendored with modifications +- vendor/ifbench-checkers — the IFBench scoring source (evaluation_lib.py, + instructions.py, instructions_registry.py, instructions_util.py), from + allenai/IFBench @ 1a902f9 (Apache-2.0, (c) Allen Institute for AI and The Google + Research Authors). Liquid AI changed instructions.py, instructions_util.py and + evaluation_lib.py; each carries a notice, and the changes are listed in + vendor/ifbench-checkers/README.md. + At build time these files are bundled into the wheel under pipette_scores/scoring/ifbench/_upstream/ (including the LICENSE). Adapted +- packages/pipette-scores/pipette_scores/dataset_catalog/_asr_audio.py — + `resample_16k_mono` follows the decoding sequence of huggingface/datasets + 2.19.0's `Audio.decode_example` (Apache-2.0, (c) The HuggingFace Datasets + Authors) and librosa's `to_mono`, `resample` (res_type "soxr_hq") and + `util.fix_length` (librosa/librosa, ISC, (c) 2013--2026 librosa development team). +- packages/pipette-scores/pipette_scores/scoring/vlm_upstream/, scoring/profile_answers.py, + dataset_catalog/prompt_profiles.py, dataset_catalog/_vlm_build.py and _vlm_common.py, and + the seven VLM dataset builders in packages/pipette-scores/scripts/ — answer extraction and + metrics, a model wrapper's answer extraction, model-specific prompts and the datasets' + default prompts and pinned sources, from open-compass/VLMEvalKit @ 8eca8e5 (Apache-2.0, + (c) 2023 VLMEvalKit Authors), modified by Liquid AI (deterministic only: no LLM judge, no + random choice). Each file names what it ports; THIRD_PARTY_LICENSES.md lists them. - packages/pipette-scores/pipette_scores/scoring/math_500/math_generic.py — the robust-boxed numeric normalization (`normalize_numeric_answer`) is copied from groq/openbench `robust_boxed.py` (MIT, (c) 2025 Groq, Inc.). @@ -30,6 +60,11 @@ Liquid AI original work — vendored from a separate Liquid repo - packages/pipette-scores/pipette_scores/scoring/ifstruct.py and the vendored `vendor/ifstruct` validator are Liquid AI's own (Liquid4All/ifstruct); they carry no third-party license and are covered by this repository's license. +- packages/pipette-scores/pipette_scores/scoring/math_500/liquid_evals_parser.py is + vendored verbatim from Liquid AI's Liquid4All/liquid_evals (liquid_evals/parser.py + @ d5534cf). Its sympy grader derives from openai/prm800k (MIT, (c) 2023 OpenAI) + and its mathd normalization from hendrycks/math (MIT, (c) 2021 Dan Hendrycks); + both licenses are in THIRD_PARTY_LICENSES.md. This repository's original work is licensed under the Apache License, Version 2.0. See LICENSE for the full license text. diff --git a/README.md b/README.md index 53bfbe1..37042c8 100644 --- a/README.md +++ b/README.md @@ -11,14 +11,15 @@ Plus shared infrastructure: - **`datasets/`** — LFS-tracked canonical eval datasets (parquet/jsonl). - **`vendor/ifstruct/`** — submodule: standalone IFStruct evaluator (`Liquid4All/ifstruct`). -- **`vendor/ifbench/`** — submodule: upstream IFBench instruction checkers (`allenai/IFBench`). +- **`vendor/ifbench/`** — submodule: upstream IFBench test set (`allenai/IFBench`). +- **`vendor/ifbench-checkers/`** — the IFBench instruction checkers (`allenai/IFBench` @ `1a902f9`, the revision liquid_evals scores with), vendored with small service patches. - **`docker/pipette-scores.Dockerfile`** — production image for the scores service. ## Supported evals | Eval | Type | Description | |------|------|-------------| -| `ifbench` | Generation | Instruction-following benchmark (backed by `vendor/ifbench`) | +| `ifbench` | Generation | Instruction-following benchmark (test set from `vendor/ifbench`, checkers from `vendor/ifbench-checkers`) | | `ifstruct` | Generation | Structured format (JSON/YAML) generation (backed by `vendor/ifstruct`) | | `gpqa_diamond` | Generation | Graduate-level science MCQ (A–D), generative + regex extraction | | `math_500` | Generation | Competition math, `\boxed{}` answer graded by `score_math_generic` | @@ -47,7 +48,7 @@ uv run ruff format --check ### Running the tests -The full test suite runs with **no credentials and no network access**. All 152 +The full test suite runs with **no credentials and no network access**. All 266 tests pass offline: ```bash @@ -83,7 +84,7 @@ to run tests, lint, or the service against the other evals do not need it. Per-package docs: -- Scores: [api reference](packages/pipette-scores/docs/httpapi.md) · [docker deployment](packages/pipette-scores/docs/docker.md) · [scoring logic](packages/pipette-scores/docs/scoring.md) · [datasets](packages/pipette-scores/docs/datasets.md) +- Scores: [api reference](packages/pipette-scores/docs/httpapi.md) · [docker deployment](packages/pipette-scores/docs/docker.md) · [scoring logic](packages/pipette-scores/docs/scoring.md) · [datasets](packages/pipette-scores/docs/datasets.md) · [eval packages](packages/pipette-scores/docs/packages.md) - Calibration: [architecture](packages/pipette-calibration/docs/architecture.md) · [creating a dataset](packages/pipette-calibration/docs/creating-a-dataset.md) · [representative selection](packages/pipette-calibration/docs/representative-selection.md) · [verification](packages/pipette-calibration/docs/verification.md) · [slurm runs](packages/pipette-calibration/docs/slurm.md) ## License diff --git a/THIRD_PARTY_LICENSES.md b/THIRD_PARTY_LICENSES.md index 528fa67..f785a51 100644 --- a/THIRD_PARTY_LICENSES.md +++ b/THIRD_PARTY_LICENSES.md @@ -11,15 +11,24 @@ repository's license. ## allenai/IFBench — Apache-2.0 -Vendored as the `vendor/ifbench` git submodule — source -`evaluation_lib.py`, `instructions.py`, `instructions_registry.py`, -`instructions_util.py`, with the license at `vendor/ifbench/LICENSE`. At build -time these are bundled into the wheel under -`pipette_scores/scoring/ifbench/_upstream/` (including `LICENSE`). +Two copies are vendored: + +- `vendor/ifbench` — the upstream repository as a git submodule, used only for + its test set. License at `vendor/ifbench/LICENSE`. +- `vendor/ifbench-checkers` — the scoring source (`evaluation_lib.py`, + `instructions.py`, `instructions_registry.py`, `instructions_util.py`) at + upstream commit `1a902f9`, with Liquid AI modifications to + `instructions.py`, `instructions_util.py` and `evaluation_lib.py`. Each + modified file carries a notice, and the changes are listed in + `vendor/ifbench-checkers/README.md`. License at + `vendor/ifbench-checkers/LICENSE`. At build time these files are + bundled into the wheel under `pipette_scores/scoring/ifbench/_upstream/` + (including `LICENSE`). + Source: https://github.com/allenai/IFBench Licensed under the Apache License, Version 2.0 (SPDX: `Apache-2.0`); full text -in `vendor/ifbench/LICENSE`. Upstream ships no `NOTICE` file. You may obtain a +in `vendor/ifbench/LICENSE` (identical in both copies). Upstream ships no `NOTICE` file. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0. IFBench data note: `packages/pipette-scores/scripts/build_ifbench_dataset.py` @@ -32,12 +41,49 @@ Use Guidelines: https://allenai.org/responsible-use. --- +## open-compass/VLMEvalKit — Apache-2.0 + +Adapted, with Liquid AI modifications, from VLMEvalKit at commit `8eca8e5` +(each file names the upstream file and functions it ports, and how it +departs from them): + +- `packages/pipette-scores/pipette_scores/scoring/vlm_upstream/` — answer + matching (`matching_util.py`), yes/no extraction and the MME and POPE + ratings (`yorn.py`), multiple-choice helpers and MMMU preprocessing + (`multiple_choice.py`), OCRBench scoring (`ocrbench.py`). The main change: + no LLM judge and no random choice, an answer the matcher cannot read + scores as wrong. Listed in `PORTING_NOTES.md` there. +- `packages/pipette-scores/pipette_scores/scoring/profile_answers.py` — the + MiniCPM-V wrapper's `extract_answer`. +- `packages/pipette-scores/pipette_scores/dataset_catalog/prompt_profiles.py` + — the prompts VLMEvalKit's model wrappers build, where a profile cites one. +- `packages/pipette-scores/pipette_scores/dataset_catalog/_vlm_build.py`, + `_vlm_common.py` and `packages/pipette-scores/scripts/build_*_dataset.py` + for the seven VLM evals — the datasets' default prompts and the TSV + sources and MD5s VLMEvalKit pins. + +Source: https://github.com/open-compass/VLMEvalKit + +Copyright 2023 VLMEvalKit Authors. + +Licensed under the Apache License, Version 2.0 (SPDX: `Apache-2.0`); the full +text is the same as IFBench's, in `vendor/ifbench/LICENSE`. Upstream ships no +`NOTICE` file. You may obtain a copy of the License at +http://www.apache.org/licenses/LICENSE-2.0. + +--- + ## openai/prm800k — MIT Vendored: `packages/pipette-scores/pipette_scores/scoring/math_500/prm800k/grading/grader.py`, `packages/pipette-scores/pipette_scores/scoring/math_500/prm800k/grading/__init__.py` Source: https://github.com/openai/prm800k +The sympy grader in +`packages/pipette-scores/pipette_scores/scoring/math_500/liquid_evals_parser.py` +(`grade_answer_sympy` and its helpers) derives from this grader and reaches this +package vendored via Liquid4All/liquid_evals. + Copyright (c) 2023 OpenAI Permission is hereby granted, free of charge, to any person obtaining a copy @@ -65,6 +111,9 @@ SOFTWARE. `packages/pipette-scores/pipette_scores/scoring/math_500/prm800k/grading/math_normalize.py` is, per its own header, "largely copied from the Hendrycks' MATH release (math_equivalence)", and reaches this package vendored via prm800k. +`mathd_normalize_answer` and `_strip_string` in +`packages/pipette-scores/pipette_scores/scoring/math_500/liquid_evals_parser.py` +("Dan Hendrycks' code") reach it vendored via Liquid4All/liquid_evals. Source: https://github.com/hendrycks/math Copyright (c) 2021 Dan Hendrycks @@ -115,3 +164,309 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +--- + +## huggingface/datasets — Apache-2.0 + +Adapted: `resample_16k_mono` in +`packages/pipette-scores/pipette_scores/dataset_catalog/_asr_audio.py` follows the +decoding sequence of `datasets` 2.19.0's `Audio.decode_example` +(`src/datasets/features/audio.py`: soundfile read, transpose, `librosa.to_mono`, +`librosa.resample` to the feature's `sampling_rate`), which the Open ASR +Leaderboard applies to every model's audio, so the ASR evals resample as it does +without depending on `datasets` 2.x or librosa. +Source: https://github.com/huggingface/datasets/blob/2.19.0/src/datasets/features/audio.py + +Copyright The HuggingFace Datasets Authors. Licensed under the Apache License, +Version 2.0 (SPDX: `Apache-2.0`); full text in +[Apache License 2.0](#apache-license-20) below. + +--- + +## librosa/librosa — ISC + +Adapted: the same function follows librosa's `to_mono`, `resample` with its +default `res_type="soxr_hq"` (soxr, then `util.fix_length` to +`ceil(n * target_sr / orig_sr)` samples) and `util.fix_length`. +Source: https://github.com/librosa/librosa + +ISC License + +Copyright (c) 2013--2026, librosa development team. + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +--- + +## huggingface/open_asr_leaderboard — Apache-2.0 + +Vendored verbatim (byte for byte): +`packages/pipette-scores/pipette_scores/scoring/asr/open_asr_normalizer/normalizer.py`, +`packages/pipette-scores/pipette_scores/scoring/asr/open_asr_normalizer/english_abbreviations.py` +Source: https://github.com/huggingface/open_asr_leaderboard at +`67e8bd6acea240819ad67080f6f31e15d4a90da5` (`normalizer/`). + +Vendored as unchanged excerpts: +`packages/pipette-scores/pipette_scores/scoring/asr/open_asr_normalizer/multilingual.py` +holds `FILLER_WORDS` and `MultilingualNormalizer` (`normalizer/data_utils.py`) +and `normalize_compound_pairs` (`normalizer/eval_utils.py`) at +`62a457c5c32d0de4df77e9470308aae7d45b055f`, with Liquid AI's header, imports and +an Armenian stand-in (`scoring/asr/NOTICE.md`). + +Copyright 2022 The OpenAI team and The HuggingFace Team. Licensed under the +Apache License, Version 2.0 (SPDX: `Apache-2.0`); full text in +[Apache License 2.0](#apache-license-20) below. Upstream ships no `NOTICE` file. +Per its header, most of the code is copied from openai/whisper (MIT, see +[openai/whisper](#openaiwhisper--mit)). + +--- + +## openai/whisper — MIT + +The origin of the Whisper-derived normalizer above +(`open_asr_normalizer/normalizer.py`), whose header says most of its code is +copied from it (`whisper/normalizers/`). Source: https://github.com/openai/whisper + +MIT License + +Copyright (c) 2022 OpenAI + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +--- + +## Apache License 2.0 + +The license of huggingface/datasets and huggingface/open_asr_leaderboard above +(the same text as this repository's `LICENSE`): + +```text + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/docs/dataset-licenses.md b/docs/dataset-licenses.md new file mode 100644 index 0000000..7240110 --- /dev/null +++ b/docs/dataset-licenses.md @@ -0,0 +1,90 @@ +# VLM benchmark dataset licenses + +A license review (2026-07) of the 7 VLM benchmarks supported by +`pipette-scores` concluded that **none of them may be redistributed in +transformed form** (e.g. the parquet files our builders produce) from this +repository. All 7 are therefore **builder-only**: they are materialized +locally / at deploy by the one-shot scripts +`packages/pipette-scores/scripts/build__dataset.py` from VLMEvalKit's +pinned upstream TSVs, and the `datasets//` directories are gitignored. +This matches the existing repo practice for third-party text evals (ifbench, +gpqa_diamond, math_500). + +## Summary + +| Benchmark | Source (pinned TSV) | License | Images provenance | Redistribution verdict | +|---|---|---|---|---| +| mme | VLMEvalKit `MME.tsv` | none declared; authors state academic-research-only / non-commercial terms | Benchmark authors' collection | Builder-only. Users must self-assess the authors' terms, especially for commercial use | +| mmstar | VLMEvalKit `MMStar.tsv` | none declared | Aggregates source benchmarks incl. SEED-Bench (CC-BY-NC) and ScienceQA (CC-BY-NC-SA), whose terms it inherits | Builder-only. Users must self-assess the inherited non-commercial terms | +| ocrbench | VLMEvalKit `OCRBench.tsv` | MIT (benchmark code) | ~29 aggregated OCR datasets with mixed / partly research-only terms | Builder-only. Users must self-assess per-source terms | +| mmmu_val | VLMEvalKit `MMMU_DEV_VAL.tsv` (VAL rows) | Apache-2.0 (dataset card) | Textbooks and online sources, rights uncleared; card carries a takedown clause | Builder-only | +| realworldqa | VLMEvalKit `RealWorldQA.tsv` | **CC-BY-ND-4.0** | xAI-collected real-world images | Builder-only. The built parquet is Adapted Material under CC-BY-ND and **must not be redistributed in any form** | +| mmbench_en_dev | VLMEvalKit `MMBench_DEV_EN_V11.tsv` | Apache-2.0 (repository) | Public datasets and the internet, rights uncleared | Builder-only | +| pope | VLMEvalKit `POPE.tsv` | MIT (annotations) | COCO val2014 photos under Flickr per-photo terms; no redistribution right | Builder-only | + +Notes: + +- The pinned TSV URLs and MD5s come from VLMEvalKit (commit `8eca8e5`, + `vlmeval/dataset/image_mcq.py`, `image_yorn.py`, `image_vqa.py`); each + builder records the URL and MD5 it used in the dataset's `metadata.json`, + together with `license` / `license_notes` fields and prints a license + notice on completion. +- The lmms-lab / community Hugging Face mirrors of these benchmarks declare + no license of their own; a mirror is not a rights grant. + +## Distribution surfaces + +There are two ways this data could leave a machine, and both count as +redistribution: + +1. **The repository.** Built datasets must never be committed. The + `datasets//` directories for all 7 benchmarks are gitignored, and + builders write only to those locations. +2. **A hosted pipette-mgmt server.** When pipette-mgmt serves a VLM eval, it + pushes the prompt samples — including base64-encoded benchmark images — + to registered clients. Serving these benchmarks to clients outside your + organization is distribution of the benchmark content. Operators should + obtain legal sign-off before hosting these datasets on a publicly + reachable pipette-mgmt instance. + +## Private storage + +Keep built VLM datasets only in the organization's private storage, under +`//`. At deploy, an init container overlays them onto +`PIPETTE_SCORES_DATA_DIR`, the same flow as the gated gpqa_diamond dataset. +The service itself reads only its local data directory, never a bucket. +Private storage does not lift the terms above: the copies remain +internal-use only. In particular, do not share the RealWorldQA (CC-BY-ND) +build onward publicly in any form. + +## ASR datasets (Open ASR Leaderboard English, FLEURS) + +Built by `packages/pipette-scores/scripts/build_asr_datasets.py` from pinned +Hugging Face sources; **builder-only** like the VLM sets: never committed to +this repository or a public one, nor baked into the image. The builder refuses +an output inside this checkout, and `datasets/asr_*/` is gitignored and +dockerignored as a backstop. Internal copies are kept and deployed as the VLM +sets are (see [Private storage](#private-storage)): the scores server reads +them from its private data directory, and pipette-mgmt's dataset cache builds +each set's served corpus from the server's samples stream into private storage, +from which an organization's devices download it for a run. Nothing of an ASR +set is committed to this repository or a public one. Each `metadata.json` +records `license`, `license_notes` and `redistribution`. Licenses are as each +dataset card (or the Open ASR Leaderboard's About table) states them. **Every +ASR set is for internal evaluation only**: pipette uses the sets only for +internal, non-commercial benchmarking of its models, keeps them org-private, +and never redistributes them. SPGISpeech and URGENT24 are licensed for +non-commercial use only, and GigaSpeech is gated; that internal, +non-commercial benchmarking is their only use here. + +| Datasets | Source | License | Verdict | +|---|---|---|---| +| `asr_esb_ami_cleaned` | AMI Meeting Corpus via `hf-audio/open-asr-leaderboard` | CC-BY-4.0 | Builder-only, internal evaluation use only | +| `asr_esb_earnings22_cleaned_aa_chunked` | Earnings-22 (Rev.com; CC-BY-SA-4.0) cleaned and chunked by Artificial Analysis, `ArtificialAnalysis/Earnings22-Cleaned-AA(-chunked)` | Apache-2.0 (the Artificial Analysis cards) | Builder-only, internal evaluation use only | +| `asr_esb_gigaspeech_cleaned` | GigaSpeech via `hf-audio/open-asr-leaderboard` | Apache-2.0, access gated behind the SpeechColab terms | Builder-only, internal evaluation use only | +| `asr_esb_librispeech_test_clean`, `_other` | LibriSpeech via `hf-audio/open-asr-leaderboard` | CC-BY-4.0 | Builder-only, internal evaluation use only | +| `asr_esb_spgispeech` | SPGISpeech via `hf-audio/open-asr-leaderboard` | Kensho user agreement, **non-commercial** | Builder-only, internal evaluation use only | +| `asr_esb_urgent2024` | URGENT 2024 Challenge (noisy) via `hf-audio/open-asr-leaderboard` | mixed, **non-commercial**: speech CC0 / CC-BY-4.0 / ODC-BY / Creative Commons, noise including WHAM! (CC-BY-NC-4.0) and TUT Urban Acoustic Scenes 2018 (non-commercial, research only) | Builder-only, internal evaluation use only | +| `asr_esb_monsoon_en_in` | `VoiceArena/MonsoonASR-Open-ASR-leaderboard-en-IN` | CC-BY-4.0 (its card) | Builder-only, internal evaluation use only | +| `asr_esb_voxpopuli_cleaned_aa` | VoxPopuli (CC0) with references cleaned by Artificial Analysis, `ArtificialAnalysis/VoxPopuli-Cleaned-AA` via `hf-audio/open-asr-leaderboard` | Apache-2.0 (the Artificial Analysis card) | Builder-only, internal evaluation use only | +| `asr_fleurs_*` (`en_us`, `es_419`, `pt_br`, `de_de`, `fr_fr`, `it_it`, `nl_nl`) | `google/fleurs` | CC-BY-4.0 | Builder-only, internal evaluation use only | diff --git a/packages/pipette-scores/docs/asr-evals.md b/packages/pipette-scores/docs/asr-evals.md new file mode 100644 index 0000000..ce256a0 --- /dev/null +++ b/packages/pipette-scores/docs/asr-evals.md @@ -0,0 +1,614 @@ +# ASR evals: the Open ASR Leaderboard (English) and FLEURS + +The ASR evals score speech-recognition quality with the two public protocols of +the Open ASR Leaderboard (huggingface/open_asr_leaderboard; Srivastav et al., +"Open ASR Leaderboard", arXiv 2510.06961): + +- **English**: its nine public English test sets, its English normalizer, its + row filter, its long-form merge and its kaldialign word error rate. +- **FLEURS** (google/fleurs; Conneau et al., arXiv 2205.12446): the + leaderboard's multilingual scoring (normalizer, number handling, alignment) + applied to FLEURS's own `transcription` references. Its multilingual + normalizer spells every number out in the clip's language on both sides, so a + transcript is not charged for writing "twenty" where the reference has "20" + (or the reverse); then its compound-pair step and its kaldialign word error + rate. The leaderboard's published FLEURS rows score against its own cleaned + copy of the references, so its numbers can differ slightly from pipette's. + +Each model is asked as its own public usage guidance says (§2). Where pipette +departs from either protocol, §4 lists it with the reason. Every source below +is public and pinned. The leaderboard code is cited at +huggingface/open_asr_leaderboard @ `62a457c5c32d0de4df77e9470308aae7d45b055f`, +its Space (hf-audio/open_asr_leaderboard) at `960d57b6cf65ea8dbb4c628ea87e60458d75018d` +for the English average and at `9b2dcd1f6318e601432eeb73a2563848fc0a20b3` for +the multilingual tab (`init.py` version "21-10-2026", whose FLEURS columns are +`hf-audio/multilingual_evals` @ `3589b3c077cb16c935e52bdd15c7fb0a3f175f7a`). + +## 1. Datasets + +One eval per dataset, 16 in all (9 English sets, 7 FLEURS locales), version +`2026.10.1`. Sources, pins, clip +counts, licenses and fingerprints are in +`pipette_scores/dataset_catalog/asr_datasets.py`. + +**The leaderboard's English sets.** The Space's English average, as of its +version "09-10-2026: Add URGENT24 to avg English WER" (`init.py` +`default_datasets`), is taken over nine public sets and two private aggregates +that no third party can run. pipette evaluates the nine public sets, in the +leaderboard's column order (`normalizer/eval_utils.score_results`, the +`DATASET_CONFIGS` of its job scripts). The EvalIds keep the `asr_esb_` prefix of +the leaderboard's lineage (the ESB benchmark, arXiv 2210.13352, whose dataset +repo this was) so that consumers' ids stay stable; the method is the +leaderboard's. + +| EvalId | Source @ revision | Clips | +|---|---|---| +| `asr_esb_ami_cleaned` | `hf-audio/open-asr-leaderboard` `ami_cleaned`/test @ `de79e57a` | 7,805 (7,715 scored after the row filter) | +| `asr_esb_earnings22_cleaned_aa_chunked` | `ArtificialAnalysis/Earnings22-Cleaned-AA-chunked` @ `27f7d005` (audio, chunk manifest) + `ArtificialAnalysis/Earnings22-Cleaned-AA` @ `5999936e` (call transcripts) | 341 chunks of 6 calls | +| `asr_esb_gigaspeech_cleaned` | as AMI, `gigaspeech_cleaned`/test | 18,768 (18,757 scored) | +| `asr_esb_librispeech_test_clean` | as AMI, `librispeech`/test.clean | 2,620 | +| `asr_esb_librispeech_test_other` | as AMI, `librispeech`/test.other | 2,939 | +| `asr_esb_spgispeech` | as AMI, `spgispeech`/test | 39,341 | +| `asr_esb_urgent2024` | as AMI, `urgent2024`/test (the noisy URGENT 2024 set; `urgent2024_clean` is outside the leaderboard average and not evaluated) | 1,000 | +| `asr_esb_monsoon_en_in` | `VoiceArena/MonsoonASR-Open-ASR-leaderboard-en-IN` @ `bc1da7b4` (the leaderboard's `VoiceArena/Monsoon_en_IN_test` redirects to it) | 2,102 | +| `asr_esb_voxpopuli_cleaned_aa` | as AMI, `voxpopuli_cleaned_aa`/test | 628 | + +`hf-audio/open-asr-leaderboard` is pinned at `de79e57adf9983993bcc72586d7313460825f5eb` +("Add urgent2024 (#13)"); its other configs are byte-identical to those at the +earlier `b6bdcd0b` (that diff touches only `README.md` and adds the two URGENT +directories), so the fingerprints recorded there hold. + +**FLEURS.** `google/fleurs` ``/test @ `70bb2e84b976b7e960aa89f1c648e09c59f894dd` +(main since 2026-05-15), 7 locales: `en_us`, `es_419`, `pt_br`, `de_de`, +`fr_fr`, `it_it`, `nl_nl` (`FLEURS_LOCALES`, 364 to 919 clips each). These are +English and the languages the leaderboard runs its multilingual FLEURS +evaluation on (`api/run_api_ml.sh` `DATASET_CONFIGS`: German, French, Italian, +Spanish, Portuguese, Dutch and Armenian), without Armenian (§4). FLEURS ships +Spanish and Portuguese as `es_419` and `pt_br` only. + +The **references** are FLEURS's own `transcription` column (Google's +normalized transcript, as the dataset ships it). The leaderboard scores its +published FLEURS rows against a copy of FLEURS whose references it cleaned +itself, starting from FLEURS's `raw_transcription`; that preparation is not +published, so pipette scores against FLEURS's `transcription`, with the same +scoring code. The two reference texts normalize almost alike, so a model's +numbers can differ slightly from what the leaderboard would report for the same +transcripts. + +**Licenses** (as each dataset card, or the leaderboard's About table, states +them). Every set is for internal evaluation only here: pipette uses the sets +only for internal, non-commercial benchmarking of its models, keeps them +org-private, and never redistributes the built data +(docs/dataset-licenses.md). SPGISpeech and URGENT24 are licensed for +non-commercial use only, and that is their only use here: + +| Set | License | +|---|---| +| AMI | CC-BY-4.0 | +| Earnings22-Cleaned-AA(-chunked) | Apache-2.0 (Artificial Analysis cards; Earnings-22 itself CC-BY-SA-4.0) | +| GigaSpeech | Apache-2.0, access gated behind the SpeechColab terms | +| LibriSpeech | CC-BY-4.0 | +| SPGISpeech | Kensho user agreement, **non-commercial** | +| URGENT24 | mixed, **non-commercial**: its speech is CC0 / CC-BY-4.0 / ODC-BY / Creative Commons, but the overlaid noise includes WHAM! (CC-BY-NC-4.0) and TUT Urban Acoustic Scenes 2018 (non-commercial, research only) | +| Voice Arena Monsoon | CC-BY-4.0 | +| VoxPopuli-Cleaned-AA | Apache-2.0 (Artificial Analysis card; VoxPopuli itself CC0) | +| FLEURS | CC-BY-4.0 | + +**Built** by `scripts/build_asr_datasets.py` into +`//2026.10.1/{test.parquet, metadata.json}`, reading each source as +the leaderboard loads it (`normalizer/data_utils.load_data`; for Earnings22 +`load_chunked_data`: the chunk manifest and WAVs, each chunk carrying its call's +`transcript` as read by `get_text`, its `parent_id` and `chunk_index`), at the +catalog's revision and in the split's row order. A clip's position, `doc_id`, is +its row in the source split, and the sample id is `sha256("|")[:12]`. +References: the leaderboard sets' `text`; FLEURS `transcription`; all stripped. The leaderboard's row filter +is applied when scoring (§3). The build streams rows to the parquet one row +group (64 clips) at a time. + +The builder **refuses an output inside this repository's checkout**, and +`datasets/asr_*/` is in `.gitignore` and `.dockerignore` as a backstop. + +**Audio is stored byte for byte** as the source ships it when it is already +16 kHz mono; any other clip is resampled once by the builder and stored as a +16 kHz mono WAV (below). Each clip's format, sample rate, channels and length +are recorded, and the source's rate and channels go in `raw_doc`. FLEURS and VoxPopuli ship 16 kHz 32-bit float WAV, +Earnings22 16 kHz 16-bit WAV, LibriSpeech 16 kHz 16-bit FLAC, MonsoonASR +**44.1 kHz** 16-bit WAV, and URGENT24 16-bit WAV at **mixed rates** (16, 22.05, +24, 32, 44.1 and 48 kHz). Some clips run past 30 s (VoxPopuli 3 of 628, up to +38.4 s; LibriSpeech test.clean 9 of 2,620, up to 35.0 s), which is where Gemma +4's 30 s limit applies. + +**Integrity.** `metadata.json` holds the source pins, the scoring recipe, an +audio summary, the license and `content_sha256`: the audio-inclusive fingerprint +(the same meaning as the VLM sets' field), a digest of every column with the +audio through each row's `audio_sha256`. The server recomputes it at load +without reading audio, and re-hashes every clip it serves. Before a dataset is +scored or served the server also checks that row `i` is `doc_id` `i` with that +position's sample id (so a sample's id says where it stands in the stream), +that every clip is 16 kHz mono, that the scoring recipe, the language and, for +`2026.10.1`, the clip count and source fingerprint are the catalog's, and that +it is no `--limit` smoke build. A full +build of `2026.10.1` must match the catalog before its file lands. +`scripts/verify_asr_datasets.py` runs all of it offline, plus every clip's hash. + +**Split sets.** git-LFS caps the size of one object (2 GB on most GitHub plans), +and a set can be larger (SPGISpeech is 9.6 GB). `scripts/split_asr_dataset.py` +cuts a `test.parquet` over 1.9 GB into ordered parts `test-00000-of-0000N.parquet`, +…, between row groups, with `metadata.json` copied unchanged. The server reads +the parts as one table (`_asr_common.parquet_files`) and refuses both layouts at +once, a missing or misnumbered part, or a stray `test-*.parquet`. The +fingerprint is a digest of the rows, not the files, so the parts carry the +build's (`tests/test_asr_parts.py`). + +**The catalog pins the input, not the output.** A clip already at 16 kHz mono +is stored as shipped; any other (MonsoonASR, most of URGENT24) is resampled +once, by the builder, and stored as the 16 kHz mono WAV serving would write, +with its source bytes' sha256, format, rate, channels and length in the row's +`raw_doc`. Serving resamples nothing. soxr rounds per CPU architecture (§4), so +the stored bytes of those two sets, and their `content_sha256`, depend on the +machine that built them. The catalog's pin is therefore `source_sha256` +(`metadata.json` records it too): the same digest over the rows as the source +shipped them, a resampled clip counted by its source fields. Any machine +reproduces it, and for the other 14 sets it equals `content_sha256`. Pins are +recorded for all 16 sets, and both the builder and the server enforce them; +`content_sha256` keeps guarding the stored files against `metadata.json`. + +## 2. Requests: the prompt profiles + +An ASR dataset holds no prompt. A **prompt profile** +(`dataset_catalog/asr_prompt_profiles.py`) turns each clip into the request the +model's own public guidance describes: the turns, the text, the template +arguments, the model's audio limit, and the cleanup of its answer. Where the +source gives no wording for a language, the profile extends it, labelled +"pipette's extension". + +**Audio, the same for every profile.** Each clip is brought to 16 kHz mono once, +as the leaderboard prepares audio for every model it runs: +`data_utils.prepare_data` casts the audio column to +`datasets.Audio(sampling_rate=16000)`. How that cast decodes depends on the +installed `datasets`: up to 3.x soundfile and librosa, from 4.0 torchcodec +(FFmpeg). The leaderboard pins it in one place, its API-provider image +(`api/Dockerfile`: `datasets==2.19.0`, "to avoid torchcodec dependency"); its GPU +model jobs install `datasets` unpinned (`requirements/requirements_jobs.txt`), so +on a current install they decode through torchcodec. pipette follows the API +image's path, the leaderboard's only pinned one, which is reproducible from +pinned Python packages without FFmpeg. The choice matters only for the clips +that are resampled, MonsoonASR's (44.1 kHz) and URGENT24's non-16 kHz ones; every +other set ships 16 kHz audio, which both paths pass through unchanged (§4). +The path: `datasets` 2.19.0 `Audio.decode_example` reads the +clip with soundfile (float64), averages the channels (`librosa.to_mono`) and +calls `librosa.resample(array, orig_sr=sr, target_sr=16000)`, whose default +`soxr_hq` runs soxr at high quality and then pads or cuts the result to +`ceil(n * 16000 / sr)` samples. `_asr_audio.resample_16k_mono` is that sequence +(with `soundfile==0.14.0` and `soxr==1.1.0`, without depending on librosa), and +matches `datasets` 2.19.0 sample for sample (§5). The clip is then cut to the +model's audio limit where its card states one (Gemma 4: 30 s) and served as a +WAV that carries those samples exactly: **16-bit PCM** when every sample is a +16-bit value (k / 32768 for an integer k), **32-bit float** otherwise. The +leaderboard's `datasets` path decodes every clip to float, and the float +samples are what must reach the model unchanged; a 16 kHz mono 16-bit source +(LibriSpeech, Earnings22, AMI, …) decodes to 16-bit values, and so do some float +sources (VoxPopuli's), so for those clips a 16-bit WAV is the same samples at +half the bytes: soundfile reads it as k / 32768, llama.cpp's audio loader +(miniaudio's `ma_pcm_s16_to_f32`) as k × 0.000030517578125, the same float32. +A resampled clip (MonsoonASR, most of URGENT24) or a float source with finer +values (FLEURS) stays 32-bit float. pipette writes either WAV itself (PCM: +`fmt `, `data`; float: `fmt `, `fact`, `data`, without the timestamped PEAK +chunk libsndfile's float WAVs add), so a clip's served bytes are the same on +every call. + +| Profile | Public source | Turns | Answer cleaned | +|---|---|---|---| +| `gemma4` | google/gemma-4-E2B-it model card @ `3e22461f65e89153144f8adb70e3b8c2cc9845a7`, "Best Practices" §4 (audio after text), §6 (ASR prompt), §7 (audio up to 30 s) | user: "Transcribe the following speech segment in {language} into {language} text.\n\nFollow these specific instructions for formatting the answer:\n* Only output the transcription, with no newlines.\n* When transcribing numbers, write the digits, i.e. write 1.7 and not one point seven, and write 3 instead of three.", then the clip, cut at 30 s (480,000 samples; transformers' `Gemma4AudioFeatureExtractor` truncates at the same `max_length`) | as is | +| `qwen25_omni` | QwenLM/Qwen2.5-Omni @ `d8a31ca56c0456b6edfcbcbf4bdbb6ae2200ef42` `cookbooks/universal_audio_understanding.ipynb` | system "You are a speech recognition model."; user: the clip, then "Transcribe the {language} audio into text without any punctuation marks." (the cookbook's message order, audio then text) | the text after the last `\nassistant\n` (the cookbook decodes prompt and answer together), stripped | +| `voxtral` | mistralai/Voxtral-Mini-3B-2507 model card @ `3060fe34b35ba5d44202ce9ff3c097642914f8f3` ("Transcription"); mistral-common @ `b54161726d8cde801c976a2bad48c23777551927` `protocol/transcription/request.py`; Voxtral technical report, arXiv 2507.13264, App. A.1 Table 4 | user: the clip alone, in the dedicated transcription mode: `chat_template_kwargs` `{"transcription": true, "language": }` | as is | +| `minicpm_o` | OpenBMB/OmniEvalKit @ `1adac0577258d539efc03f16d8a0f4b3f7df6c19` `o_e_Kit/configs/generation_configs.json` (`asr_configs`); openbmb/MiniCPM-o-4_5 @ `d59117d19ddb8a28a62f1b5f4e3b34ac48d510a4` model card (audio understanding) and `modeling_minicpmo.py`; openbmb/MiniCPM-o-2_6 @ `06849bfd36da94da1f5a88fa17e8ba63f08a4c4d` `modeling_minicpmo.py` (`get_sys_prompt`) | user: OpenBMB's evaluation prompt for the dataset, in the clip's language (below), and the newline of its `"\n{audio}"`, then the clip | the text after the last `<|tts_bos|>` and before `<|tts_eos|>` (`_generate_mel_spec`; `chat()` cuts at `<|tts_eos|>`), stripped | +| `lfm25_audio` | LiquidAI/LFM2.5-Audio-1.5B model card @ `c362a0625dfe45aa588dce5f0ada28a7e5707628`, which points to Liquid4All/liquid-audio @ `19e65845923a7f136442c95137884ec61eb386aa` README "ASR" | system "Perform ASR." (the README's fixed ASR prompt), whatever the clip; user: the clip. The card lists English as the supported language and gives no prompt for another; LFM2.5-Audio runs every dataset, and a cell in another language is marked as outside the card's declared languages (pipette-mgmt's catalog entries carry the card's languages) | a trailing `<|im_end|>` dropped | +| `lfm3` | LFM3's native ASR chat format: a placeholder for an unreleased model (no public card yet), to be replaced from its card before release | system "Perform ASR in {language}."; user: the clip | as is | + +`{language}` is the language's English name, pycountry's ISO 639-3 name, +fixed at build time. + +**Qwen2.5-Omni**: the cookbook's English prompt is used verbatim; its other +languages' examples are worded slightly differently ("without including any +punctuation marks"; a Chinese-language prompt for Chinese), and pipette uses the +English wording for every language (pipette's extension). The user turn follows +the cookbook's message order, the audio part then the text part, which the +current Qwen2.5-Omni chat template renders in that order. (The outputs recorded +in the cookbook's cells show the text before the audio, as an earlier version of +the template rendered the same messages; pipette follows the messages.) + +**Voxtral's language hint** is given for the nine languages Mistral reports +Voxtral's transcription on, FLEURS in the technical report (Arabic, Dutch, +English, French, German, Hindi, Italian, Portuguese, Spanish), which include +every language evaluated here, and left empty, so that the model detects the +language (card: "By default, Voxtral automatically predicts the source audio +language"), for any other. mistral-common's +`language` is an optional ISO 639-1 code that "will improve language adherence". + +**MiniCPM-o's prompts** (OmniEvalKit `asr_configs`): SPGISpeech "Please listen +to the financial domain audio and transcribe it accurately."; GigaSpeech, +VoxPopuli and LibriSpeech "Please listen to the audio carefully and transcribe it +with high precision. Please output in lower case without punctuation."; FLEURS +English and the English sets OmniEvalKit does not configure (AMI, Earnings22, +Monsoon, URGENT24) the general "Please listen to the audio carefully and +transcribe it with high precision." (its `fleurs_en`, and its default), +unchanged. + +The other FLEURS languages, which OmniEvalKit does not configure, are asked +**in their own language**. MiniCPM-o picks the language of its answer from the +language of the instruction: the MiniCPM-o 4.5 model card (@ `d59117d1`, +"audio understanding") gives a single English prompt for English ASR and for +Chinese-to-English translation alike, and the MiniCPM-o 2.6 code (@ `06849bfd`, +`get_sys_prompt`) says the model can "select the response language based on the +question language". An English instruction naming the language therefore draws +English translations instead of transcripts. OpenBMB itself writes the +instruction in the target language: the one European language OmniEvalKit +configures, French (`commonvoice_fr`), is asked "Veuillez écouter attentivement +l'audio et le transcrire avec précision.". Each prompt is that instruction in +the clip's language, then a clause asking for a verbatim transcript in the +original language, without translation: + +| Locale | Prompt | +|---|---| +| `es_419` | Por favor, escuche atentamente el audio y transcríbalo con gran precisión. Transcríbalo palabra por palabra en el idioma original (español); no lo traduzca. | +| `pt_br` | Por favor, ouça o áudio com atenção e transcreva-o com alta precisão. Transcreva-o palavra por palavra no idioma original (português); não o traduza. | +| `de_de` | Bitte hören Sie sich die Audioaufnahme aufmerksam an und transkribieren Sie sie mit hoher Präzision. Transkribieren Sie sie wortgetreu in der Originalsprache (Deutsch); übersetzen Sie sie nicht. | +| `fr_fr` | Veuillez écouter attentivement l'audio et le transcrire avec précision. Transcrivez-le mot pour mot dans la langue d'origine (français) ; ne le traduisez pas. | +| `it_it` | Per favore, ascolta attentamente l'audio e trascrivilo con alta precisione. Trascrivilo parola per parola nella lingua originale (italiano); non tradurlo. | +| `nl_nl` | Luister aandachtig naar de audio en transcribeer deze met hoge precisie. Transcribeer het woordelijk in de oorspronkelijke taal (Nederlands); vertaal het niet. | + +The French first sentence is OmniEvalKit's, verbatim. The German, Spanish, +Portuguese, Italian and Dutch translations of it, and the "do not translate" +clause in every language, are pipette's extension, chosen on the FLEURS +validation split as below. English keeps OpenBMB's prompt without the clause. +**The non-French strings await review by native speakers.** + +**How MiniCPM-o's FLEURS prompts were chosen.** OpenBMB publishes no prompt +for these six languages, so pipette had to choose one, and chose it on the +FLEURS **validation** split (`google/fleurs` @ `70bb2e84`, `validation`, every +clip of the six locales), never on the test split the entries score. The +selection rule was written down before any candidate was run: + +1. The candidates, each the user turn before the clip, the same in every run: + - the earlier English extension, "Please listen to the audio snippet + carefully and transcribe the content in {language}."; + - the model card's English usage prompt, "Please listen to the audio + snippet carefully and transcribe the content."; + - OmniEvalKit's general English prompt, "Please listen to the audio + carefully and transcribe it with high precision."; + - that English prompt followed by "Transcribe it verbatim in the original + language ({language}); do not translate it."; + - that English prompt rendered in the clip's language (for French, + OmniEvalKit's `commonvoice_fr` string); + - the rendered prompt followed by the same clause in the clip's language + (the table above). +2. Each candidate ran on every validation clip as the catalog entries run + MiniCPM-o: `openbmb/MiniCPM-o-4_5-gguf` @ `db25077c` (Q8_0 weights, OpenBMB's + audio GGUF), pipette's ASR build of llama.cpp, three beams, repetition + penalty 1.2, 128 tokens, thinking off. Every clip was answered. +3. Each was scored with the then-current selection protocol (corpus word error + rate per locale against google/fleurs' `transcription`), and its answers + were language-identified with fastText's `lid.176` model. +4. The rule: a candidate is eligible only if at most 2% of its answers, pooled + over the six locales, are identified as a language other than the clip's; + the eligible candidate with the lowest mean word error rate over the six + locales wins. + +Only the two candidates carrying the no-translate clause passed the language +filter; the native-language one had the lower mean error rate, and is the +prompt above. Re-scored with the FLEURS protocol of §3 (the leaderboard's +multilingual scoring against `transcription`), the same rule picks the same +prompt: the same two candidates are eligible and the +native-language one has the lower mean error rate. The test split is run once, by the catalog entries, with the +chosen prompt frozen. OpenBMB states that MiniCPM-o's speech modes support only +Chinese and English (MiniCPM-V issues #1056 and #1147), so every non-English +FLEURS result is outside its supported languages. +MiniCPM-o's TTS chat template (`use_tts_template`) opens the assistant turn with +`<|tts_bos|>`, which a server may return with the answer; the cleanup follows +OpenBMB's own split, and is a no-op when it is absent. + +### Server prerequisites + +What a source sets on the model or server, rather than in the request, pipette's +client must provide: + +- **Thinking off, for every family.** The client starts every ASR server with + reasoning off, so no profile sends `enable_thinking`; `/score` refuses + `enable_thinking: true` for an ASR eval. +- **Voxtral's transcription mode.** The `transcription` / `language` template + arguments take effect only under a chat template that renders mistral-common's + transcription request (`tokens/tokenizers/instruct.py` + `_encode_instruct_transcription`: the user turn holding the audio, then + `lang:` when a language is given, then `[TRANSCRIBE]`); a template + without that mode ignores them, and the model then chats instead of + transcribing. +- **LFM3's turn format**: its native ASR chat format, an open assistant turn + with no think block. +- **MiniCPM-o's template switches**: `use_tts_template: true` and + `enable_thinking: false`, on the server and every request. The prompt-clip + newline rides in the profile's text, since a server concatenates content parts. +- **Template arguments merge.** The client sends the profile's server-level + template arguments merged with each sample's `chat_template_kwargs` (the + sample's win; thinking stays off). + +**Generation settings are the benchmark definition's**, not the profile's. The +public recommendations: + +| Family | Public recommendation | +|---|---| +| Gemma 4 E2B | card §1: temperature 1.0, top-p 0.95, top-k 64 "across all use cases"; its examples generate up to 512 new tokens; thinking off | +| Qwen2.5-Omni-3B | cookbook: `thinker_do_sample=False` (greedy), `thinker_max_new_tokens=256` | +| Voxtral Mini 3B | card: temperature 0.0 for transcription, `max_new_tokens=500` | +| MiniCPM-o 4.5 | OmniEvalKit `asr_configs`: `num_beams` 3, `sampling` false, `max_tokens` 128; `modeling_minicpmo.py` `prepare_generation_config`: repetition penalty 1.2 with beams, length penalty 1.0 | +| LFM2.5-Audio 1.5B | liquid-audio README: `generate_sequential` with no text temperature (greedy text), `max_new_tokens=512`, its general default; pipette's entries cap a transcript at 256 (on the English sets 512 only let rare repetition loops run longer) | +| LFM3 | greedy (pipette's own) | + +## 3. Metrics + +Per dataset, from its `metadata.json` `scoring` recipe (checked against the +catalog at load; `scoring/asr/`). The normalizer runs on both the reference and +the hypothesis: + +| Datasets | Normalizer | Alignment | Unit | +|---|---|---|---| +| Leaderboard English (9) | the leaderboard's `EnglishTextNormalizer()` (`normalizer/normalizer.py`, vendored byte for byte at `67e8bd6a`, unchanged at `62a457c5`) | `kaldialign.edit_distance(ref.split(), hyp.split(), merge_compounds=True)`, kaldialign 0.12.0 (`eval_utils._align_pair`; the version `requirements_jobs.txt` pins) | word | +| FLEURS (7) | the leaderboard's multilingual scoring normalizer for the clip's language (`eval_utils._normalize(text, )`): for `es`, `pt`, `de`, `fr`, `it`, `nl` its `ml_normalizer`, `MultilingualNormalizer(remove_diacritics=False)` called with the language (`normalizer/data_utils.py`); for `en_us` its English normalizer, as `_normalize` does for `en` | the leaderboard's multilingual alignment (`_align_pair` with `multilingual=True`): `eval_utils.normalize_compound_pairs`, then the same kaldialign alignment | word | + +The FLEURS recipe is what the leaderboard's `score_results(multilingual=True, +language=)` (and `scripts/score_bucket_results.py --multilingual`) applies +to its FLEURS runs, the code its multilingual tab is scored with. The +multilingual normalizer lowercases, drops bracketed and parenthesized spans, +turns symbols and punctuation into spaces while keeping diacritics and +combining marks, and collapses whitespace; then it joins space-separated +thousands ("10 000" to "10000") and spells every remaining digit run out with +num2words in the language (`es`, `pt`, `de`, `fr`, `it`, `nl`: "20" becomes +"veinte", "vinte", "zwanzig", "vingt", "venti", "twintig"), on the reference and +the hypothesis alike. A model that writes a number in words is therefore not +charged for it, nor one that writes digits. (The English normalizer goes the +other way, number words to digits, with the same effect.) Its +`normalize_compound_pairs` gives both sides the joined form where a mismatched +stretch is the same characters without its spaces ("zum beispiel" against +"zumbei spiel"). The leaderboard's language codes are its two-letter ones +(`ML_LANG_DATASETS`); its Armenian normalizer is not vendored, and a language +not evaluated here is refused. + +Three properties of that code, reproduced as they are: `pt` is num2words' +European Portuguese ("dezasseis", where Brazilian Portuguese writes +"dezesseis"); num2words runs after the lowercase and punctuation passes, so its +own capitals and hyphens stay (German "eine Million", French "vingt-cinq"), +and a hypothesis that writes such a number word itself (lowercased, or split at +the hyphen) is still charged for it; and a parenthesized span is dropped from +both sides, words and all. + +The vendored code is `scoring/asr/open_asr_normalizer/multilingual.py`: the +leaderboard's `FILLER_WORDS` and `MultilingualNormalizer` (`data_utils.py`) and +`normalize_compound_pairs` (`eval_utils.py`) at `62a457c5`, unchanged, with +num2words pinned at 0.5.14, the latest release, which the leaderboard's +unpinned `num2words` requirement installs (`scoring/asr/NOTICE.md`). Every +dataset is scored by word error rate. + +The rate is the **corpus** rate: summed substitutions, deletions and insertions +over summed reference units (the leaderboard's `align_manifests`, for both +protocols), never the mean of per-sample rates. On the leaderboard's English +sets a sample whose normalized reference is empty or reads "ignore time segment +in scoring" is not scored (`data_utils.is_target_text_in_range`, which +`prepare_data` applies); its multilingual runs drop only an empty reference +(`run_eval_ml.py`), and no FLEURS reference is empty. Earnings22 is scored per call +(`eval_utils.merge_chunked_manifest`): the chunks' stripped answers, empty ones +skipped, joined with one space in `chunk_index` order, against the call's +transcript. + +**A result is an answer to every sample, or no result**: `/score` scores an ASR +request only when every sample of the dataset, excluded ones included, has +exactly one completion and none failed (`stop_reason: "failure"`). A request +that leaves a sample unanswered or carries a failed row is refused (400, naming +the unanswered and failed ids), never scored with the gap as an empty +transcript; the scorer refuses it too, for a direct caller. A truncated or +doom-looped generation is an answer and is scored; `stop_reasons` counts them. +A dataset with no reference unit at all has no rate (`error_rate` null). + +`/score`'s `context`: + +| Key | Meaning | +|---|---| +| `error_unit` | `word` (every dataset; the consumer checks it) | +| `error_rate` | the corpus rate as a ratio; null only when the dataset has no reference unit | +| `result` | `error_rate` × 100 (percent, the leaderboard's unit); null with it | +| `hits`, `substitutions`, `deletions`, `insertions`, `reference_units` | summed over the scored units | +| `units_scored` | scoring units (samples, or Earnings22 calls) | +| `samples_total`, `samples_excluded` | dataset samples; removed by the leaderboard's row filter | +| `stop_reasons` | completions per `stop_reason` (`"unreported"` when none was sent) | +| `prompt_profile`, `scoring` | the profile and the dataset's recipe | + +`error_rate` and `result` are unrounded; the leaderboard shows +`round(100 * wer, 2)`. There are no `_blanked_truncated` / +`_pass_at_1_blanked` keys: an error rate has no pass@1. + +Each scored sample carries `measures`, **sum-safe**: a unit's counts and +normalized texts are on one sample, so summing a count over the samples gives +the context's total: + +| Sample | `measures` | +|---|---| +| scored, owning its unit (an utterance; an Earnings22 call's first chunk) | `unit`, `hits`, `substitutions`, `deletions`, `insertions`, `reference_units`, normalized `reference` and `hypothesis` | +| another chunk of an Earnings22 call | `{"unit": }` | +| removed by the leaderboard's row filter | `{"excluded": true}` | + +`is_correct` is "its unit was scored with no error" and false otherwise +(excluded samples included); it is no ASR metric, so never compute an accuracy +from it. Each scored sample's `messages` are the profile's prompt, text only: +each audio part stays in its place as `{"type": "input_audio", "input_audio": +{"data": "", "format": "wav"}}`, so the turn order is auditable, while the clip +itself is the caller's (it was sent with the samples). Each scored sample also +carries the `chat_template_kwargs` it was served with (Voxtral's language hint). + +### Suite headlines + +A suite headline is computed by the consumer (pipette-mgmt) from the per-dataset +results, as no cross-dataset VLM aggregate is computed here either; +`scoring/asr/suite.py` is the reference definition (`open_asr_mean`, +`fleurs_mean`), tested in `tests/test_asr_suite.py`: + +- **Open ASR Leaderboard, English**: the equal-weight mean of the 9 public sets' + `error_rate` (a macro mean; never summed edits across sets, never weighted by + size), × 100 for percent, as the leaderboard averages its public columns. + The leaderboard averages per-set WERs it has already rounded to two + decimals: its public CSV `avg` is that mean, unrounded, and its console + "Composite Results" line shows it to two decimals. `leaderboard_result` is + the console figure: each set's percent WER rounded to two decimals, averaged, + the mean rounded to two decimals. It is within 0.01 of `result` (each + rounding contributes up to 0.005). +- **FLEURS**: the equal-weight mean of the 7 locales' `error_rate` (WER), the + same macro mean, × 100 for percent. + +Each needs every one of its datasets exactly once, each with a corpus rate and +scored in words; otherwise there is no headline. + +## 4. Where pipette departs from the protocols, and why + +**Open ASR Leaderboard (English):** + +1. **The average is the public-set mean.** The Space's displayed English + average also includes two private aggregates ("Private (scripted)", "Private + (conversational)"), which cannot be reproduced; pipette's headline is the + mean over the 9 public sets, as `score_results` averages its public columns + (it averages the per-set WERs rounded to two decimals; pipette's `result` + averages the unrounded rates, see item 6). +2. **Pinned revisions.** The leaderboard loads its datasets unpinned; pipette + pins each, so that a number names its data. +3. **Prompts and decoding are each model's own.** The leaderboard runs Gemma 4 + with a prompt of its own ("Write out, word for word, exactly what is said in + this audio. …") and greedy decoding (256 new tokens), and Voxtral greedy with + `language="en"` (500 new tokens); it does not run Qwen2.5-Omni, MiniCPM-o or + LFM2.5-Audio. pipette runs every model with its own published + recommendation (§2): for Gemma 4 that is Google's ASR prompt and sampling at + temperature 1.0, top-p 0.95, top-k 64. The card names no seed; pipette's + entries fix the sampler seed (`parameter_seed = 0`), pipette's own choice so + that a rerun draws the same way. Sampled numbers are still not comparable + with the leaderboard's greedy Gemma 4 entry. +4. **Filtered samples are served.** The leaderboard drops a sample whose + normalized reference is empty or the ignore marker before inference; + pipette serves it, requires its generation, and leaves it out of the score. + The same samples are scored. +5. **A partial run is no result.** The leaderboard scores whatever its + manifest holds and only warns when a chunk of a call is missing; pipette + refuses to score a run with a missing or failed sample. +6. **Unrounded values.** The rates are kept unrounded; `leaderboard_result` + gives the leaderboard's console rounding of the headline (§3). +7. **Corner cases.** A dataset with no reference word has no rate (the + leaderboard reports 0.0, or infinity with errors); a hypothesis on which the + normalizer raises scores as empty instead of failing the dataset. +8. **Audio decoding.** One resample for every model, the path of the + leaderboard's API image (`datasets` 2.19.0: soundfile, then librosa's + `soxr_hq`); its model jobs on `datasets` ≥ 4 decode and resample through + torchcodec (FFmpeg) instead. Only resampled clips can differ: MonsoonASR's + and URGENT24's non-16 kHz ones. +9. **Answer cleanup.** Each profile removes its model's own control tokens or + echoed prompt (§2) before scoring. + +**FLEURS (the leaderboard's multilingual scoring):** + +1. **7 locales**, pipette's selection: the six languages of the leaderboard's + multilingual FLEURS runs but Armenian (German, French, Italian, Spanish, + Portuguese, Dutch), and English. The leaderboard publishes no English FLEURS + column; `en_us` is scored by the same code path, its multilingual scoring + with the language `en` (its English normalizer, then the compound-pair + alignment). +2. **The references are FLEURS's own `transcription`**, not the leaderboard's + own cleaned copy of them, whose preparation is not published (§1). The + scoring code is the leaderboard's, unchanged, so the same transcripts can + score slightly differently from the leaderboard's published rows only + because of the reference text. +3. **The summary across locales.** The leaderboard shows each language + separately; pipette's macro WER over the 7 locales is its own summary. +4. **num2words is pinned** (0.5.14), which the leaderboard installs unpinned; + its other code is pinned as above. +5. **Pinned revision, completeness, prompts and decoding**, as for the + leaderboard sets above. +6. **MiniCPM-o's non-English prompts are pipette's**, chosen among six + candidates on the FLEURS validation split by a rule fixed beforehand, so + the test split scored no candidate (§2). +7. **Every model runs every locale.** A cell in a language its model card does + not declare for speech recognition is scored and marked, not dropped; the + catalog entries carry each card's languages (pipette-mgmt's + `docs/benchmarks.md`, "The languages a card declares"). + +## 5. Checking against the public references + +`tests/test_asr_public_reference.py` checks pipette's numbers against the public +reference implementations themselves: + +- the leaderboard's own code (huggingface/open_asr_leaderboard @ `62a457c5`: + `data_utils` normalizer and row filter, `eval_utils.merge_chunked_manifest`, + `align_manifests`, `score_manifests`) must give exactly our hits, + substitutions, deletions, insertions, reference words, corpus rate and rounded + WER, set by set and unit by unit; +- the same leaderboard code as its multilingual scoring runs it + (`eval_utils._normalize` with the language, `_align_pair` with + `multilingual=True`, `align_manifests`), with num2words 0.5.14, must give + exactly our normalized texts, counts and corpus rate, locale by locale and + pair by pair, and the vendored excerpts must be its source text; +- `datasets` 2.19.0 `Audio(sampling_rate=16000)` must give exactly our 16 kHz + mono samples, and the served WAV must carry them; +- negative controls: kaldialign without `merge_compounds` on the English sets, + and on FLEURS the multilingual normalizer without its number step, or the + alignment without `normalize_compound_pairs`, must each miss. + +The corpus (`tests/asr_public_reference.py`) is a synthetic part (English and +every FLEURS language, built to exercise what the normalizers rewrite, with +hypotheses as perturbed references, numbers among them written the other way, +plus chunked calls), every FLEURS test transcript at the pinned revision +(`transcription`, the reference, and `raw_transcription`), and every reference and some clips of each built dataset +found under `PIPETTE_SCORES_DATA_DIR` (several roots joined with +`:`). When `PIPETTE_SCORES_DATA_DIR` is set, each root must exist and hold at +least one built ASR dataset, or the live tests fail (a mistyped path never +passes silently). Every answer is matched to its input by name, and the number +of answers must equal the number of inputs. + +The default test run compares against the reference answers recorded on the +synthetic part (`tests/fixtures/asr_public_reference.json`). For the audio it +records, per synthetic clip, the shape and four fixed ±1 projections of the +samples `datasets` 2.19.0 decodes, compared to within 1e-4 (exact equality is +the live layer's, on the platform the reference runs on). soxr's "high" +quality runs libsoxr's single-precision engine, whose SIMD and scalar code +round differently per CPU architecture: a resampled clip's float32 samples +differ by a few ulps (up to ~1.2e-7) between macOS arm64, Linux x86_64 and +Linux aarch64, its projections by up to ~6e-6, while the near misses move them +by 2.2e-2 or more. So a resampled clip (MonsoonASR, most of URGENT24) is the +same on every call on one architecture, not across architectures: the builder +resamples it once and stores the result, so a served clip never passes through +soxr (§1). Its +negative controls, soxr's "very high" +quality in place of "high" and the resample without librosa's length rule, must +miss. With the reference environment it runs the reference code itself: + + packages/pipette-scores/scripts/setup_asr_reference_env.sh /tmp/asr-reference + PIPETTE_ASR_REFERENCE_ENV=/tmp/asr-reference \ + PIPETTE_SCORES_DATA_DIR=/data/pipette-scores \ + uv run pytest packages/pipette-scores/tests/test_asr_public_reference.py + +and `python packages/pipette-scores/tests/asr_public_reference.py --write-fixture` +re-records the fixture. The environment (a separate virtualenv: `datasets` +2.19.0, librosa, soundfile, soxr, kaldialign 0.12.0, num2words 0.5.14, and a +checkout of the leaderboard repository at the pinned commit) is no dependency +of pipette-scores. + +## 6. Serving + +`GET /evals//datasets/2026.10.1/samples?prompt_profile=` +returns the whole set, as the VLM evals' `/samples` does, with the +`X-Samples-Digest` header, and `/samples/digest` reports the digest alone. A +set is never held whole in memory: the server learns its digest first (one +pass over the set, which learns every profile's digest at once, then cached), +sends the header, and streams the body one sample at a time, each sample +written as the canonical JSON its digest hashes; a body that would not hash to +its header is cut off before it closes. The samples are not paged (an `offset` +or `limit` on an ASR request is a 400): a client that keeps a copy, as +pipette-mgmt's dataset cache does, takes the stream once and pins its digest, +and `/score` refuses (409) a `samples_digest` this server no longer serves for +the eval, dataset and profile. The digest covers everything served, the audio +bytes included, so a pin taken from it names the exact requests the model is +sent. The server reads one row group of source audio at a time and re-hashes +each clip; `/score` reads no audio at all. The full contract: +`docs/httpapi.md`. diff --git a/packages/pipette-scores/docs/datasets.md b/packages/pipette-scores/docs/datasets.md index e2c1ffa..835d87e 100644 --- a/packages/pipette-scores/docs/datasets.md +++ b/packages/pipette-scores/docs/datasets.md @@ -1,15 +1,24 @@ # Datasets -Each dataset backs one of the scoreable evals (`pipette_scores.types.EvalId`: `ifbench`, `ifstruct`, `gpqa_diamond`, `math_500`). Some are established public benchmarks, run verbatim; one (`ifstruct`) is a Liquid in-house creation. +Each dataset backs one of the scoreable evals (`pipette_scores.types.EvalId`): the text evals `ifbench`, `ifstruct`, `gpqa_diamond`, `math_500` and the VLM evals `mme`, `mmstar`, `ocrbench`, `mmmu_val`, `realworldqa`, `mmbench_en_dev`, `pope`. Some are established public benchmarks, run verbatim; one (`ifstruct`) is a Liquid in-house creation. ## 1. Overview | Eval | Source | Origin | |------|--------|--------| -| ifbench | [allenai/IFBench](https://github.com/allenai/IFBench) (vendored in `vendor/ifbench`; verifier code Apache-2.0, dataset ODC-By-1.0) | Public | +| ifbench | [allenai/IFBench](https://github.com/allenai/IFBench) (test set from the `vendor/ifbench` submodule, checkers in `vendor/ifbench-checkers`; verifier code Apache-2.0, dataset ODC-By-1.0) | Public | | ifstruct | Generated via the Liquid `ifstruct` package (`vendor/ifstruct`) | Liquid in-house | | gpqa_diamond | [Idavidrein/gpqa](https://huggingface.co/datasets/Idavidrein/gpqa) (Diamond config) | Public (gated, CC BY 4.0) | | math_500 | [HuggingFaceH4/MATH-500](https://huggingface.co/datasets/HuggingFaceH4/MATH-500) | Public (MIT) | +| mme | VLMEvalKit pinned `MME.tsv` | Public (no license; academic/non-commercial terms) | +| mmstar | VLMEvalKit pinned `MMStar.tsv` | Public (no license; inherits CC-BY-NC[-SA] sources) | +| ocrbench | VLMEvalKit pinned `OCRBench.tsv` | Public (MIT code; mixed image sources) | +| mmmu_val | VLMEvalKit pinned `MMMU_DEV_VAL.tsv` (VAL rows) | Public (Apache-2.0 card; uncleared images) | +| realworldqa | VLMEvalKit pinned `RealWorldQA.tsv` | Public (**CC-BY-ND-4.0**) | +| mmbench_en_dev | VLMEvalKit pinned `MMBench_DEV_EN_V11.tsv` | Public (Apache-2.0 repo; uncleared images) | +| pope | VLMEvalKit pinned `POPE.tsv` | Public (MIT annotations; COCO/Flickr images) | + +All 7 VLM datasets are built by `packages/pipette-scores/scripts/build__dataset.py`, never committed here, and kept for internal use only in the organization's private storage; they are not redistributable in transformed form. Licensing details and distribution rules: [dataset-licenses.md](../../../docs/dataset-licenses.md). ## 2. Storage Layout @@ -25,8 +34,8 @@ $PIPETTE_SCORES_DATA_DIR/ ``` - **`PIPETTE_SCORES_DATA_DIR`** — environment variable pointing to the data root. Defaults to the parent directory of the `pipette_scores` package. -- **`dataset_name`** — the pinned dataset version (`ifbench` → `2026.06.1`, `ifstruct` → `release_v1_0`, `gpqa_diamond` → `2026.06.1`, `math_500` → `2026.06.1`). -- **Committed vs materialized** — only the Liquid-owned `ifstruct` dataset is committed to the repo. The third-party datasets (`ifbench`, `gpqa_diamond`, `math_500`) are gitignored and **materialized at deploy** from their pinned upstream revision by `packages/pipette-scores/scripts/build__dataset.py` (a `content_sha256` in the metadata is self-asserted at build and re-verified at load). +- **`dataset_name`** — the pinned dataset version (`ifbench` → `2026.06.1`, `ifstruct` → `release_v1_0`, `gpqa_diamond` → `2026.06.1`, `math_500` → `2026.06.1` and `2026.10.1`, all 7 VLM evals → `2026.10.1`). +- **Committed vs materialized** — only the Liquid-owned `ifstruct` dataset is committed to the repo. The third-party datasets (`ifbench`, `gpqa_diamond`, `math_500`, and the 7 VLM evals) are gitignored and **materialized at deploy** from their pinned upstream revision by `packages/pipette-scores/scripts/build__dataset.py` (a `content_sha256` in the metadata is self-asserted at build; the text-eval loaders re-verify it at load). ## 3. Sample IDs @@ -36,13 +45,31 @@ All sample IDs are deterministic 12-character hex strings produced by SHA256 has Each sample exists in two forms: a **prompt** version containing only the chat messages for inference (no answers), and a **scoring** version containing the full ground truth. Both share the same ID, which links a model completion back to its expected answer. +**Prompt profiles (VLM evals).** By default every model gets the same prompt: the dataset's own VLMEvalKit prompt, stored with the dataset. VLMEvalKit itself prompts some model families differently, some model cards ask for a particular prompt when benchmarking, and the published numbers come from those prompts. A prompt profile, asked for with `?prompt_profile=` on the samples endpoint, rebuilds each prompt's text from the row's `raw_doc` the way VLMEvalKit's wrapper or the model card does for that family. Only the text changes: images, their placement, the dataset and the scoring stay the same. Each profile is one short function in [`dataset_catalog/prompt_profiles.py`](../pipette_scores/dataset_catalog/prompt_profiles.py) citing what it copies (VLMEvalKit `8eca8e5` lines, or the model card), and the tests spell out every profile's text. + +| Profile | Copies the prompts of | What changes | +|---------|------|------| +| `internvl` | InternVL3.5 | multiple choice ends "Answer with the option's letter from the given choices directly."; MME and OCRBench ask for a single word or phrase | +| `internvl_r1` | InternVL3.5 with thinking | the `internvl` prompts, after InternVL's R1 system prompt (its thinking mode has no chat-template switch) | +| `granite_vision` | Granite Vision (IBM's wrapper, written for 3.x) | the same multiple-choice prompt; OCRBench asks for one word, a short phrase or a full address | +| `minicpm_v` | MiniCPM-V, answering directly | multiple choice ends with the letter instruction; OCRBench asks for a single word or phrase | +| `minicpm_v_cot` | MiniCPM-V, reasoning step by step | step-by-step instructions ending in an "Answer: ..." line | +| `lfm_vl` | LFM2-VL, LFM2.5-VL-1.6B | "Please answer directly with only the final answer, do not give any explanation." after every prompt | +| `lfm2_5_vl_3b` | LFM2.5-VL-3B | MME prompts by category; OCRBench asks for a concise answer; the direct-answer line on MMStar, RealWorldQA and MMBench | +| `qwen3_5` | Qwen3.5, from its model card's benchmarking best practices | multiple choice ends "Please show your choice in the `answer` field with only the choice letter, e.g., `"answer": "C"`."; the scorer reads that field | +| `glm_v` | GLM-4.6V-Flash (VLMEvalKit's `GLMThinking`) | the dataset prompts; POPE adds a line "Please answer yes or no." | +| `kimi_vl` | Kimi-VL-A3B-Thinking-2506 (VLMEvalKit's `KimiVL`) | on MMMU and MMStar the closing "Please select the correct answer..." line gives way to Kimi's step-by-step instruction ending in `'Answer: $\boxed{LETTER}$'`, appended to every piece of text (MMMU's question is split at its images); the other benchmarks keep the dataset prompts. The scorer reads the text after the model's `◁think▷…◁/think▷` like any other reasoning and, on MMMU and MMStar, its last `\boxed{}` | +| `nemotron_think` | Nemotron-Nano-12B-v2-VL with thinking | the dataset prompts, each starting with a newline after the image (on MMMU, whose images sit at their `` placeholders, before the first image), after the system message `/think` (its thinking switch; the chat template ignores `enable_thinking`) | +| `phi4_rv_think` | Phi-4-reasoning-vision-15B with thinking forced | the dataset prompts, no system message, and the conversation ends with an assistant message `` that the server continues (the model card's forced-thinking form; its template has no switch). The scorer knows the reply continues a thought the prompt opened ([scoring.md §7](scoring.md#7-reasoning-markup-and-cut-off-reasoning)). Needs a runtime that continues a final assistant message, as llama-server does by default | +| `mimo_vl` | MiMo-VL-7B-RL-2508 (optional) | the dataset prompts under the system message "You are a helpful assistant." | + --- ## 5. ifbench Scoring: [ifbench](scoring.md#2-ifbench) -The upstream IFBench test set — **300** single-turn prompts, each pairing a user-style request with one or more machine-checkable constraints (keyword counts, format/casing rules, line structure, etc.). Run verbatim, vendored from [allenai/IFBench](https://github.com/allenai/IFBench) at a pinned commit. The verifier code is Apache-2.0; the dataset rows are ODC-By-1.0 and should be used in accordance with Ai2's Responsible Use Guidelines. Served with `metadata.repeats = 5` (1500 attempt ids), reported as pass@1. +The upstream IFBench test set — **300** single-turn prompts, each pairing a user-style request with one or more machine-checkable constraints (keyword counts, format/casing rules, line structure, etc.). Run verbatim, read from the [allenai/IFBench](https://github.com/allenai/IFBench) submodule at a pinned commit. The verifier code (the revision liquid_evals uses, see [scoring](scoring.md#2-ifbench)) is Apache-2.0; the dataset rows are ODC-By-1.0 and should be used in accordance with Ai2's Responsible Use Guidelines. Served with `metadata.repeats = 5` (1500 attempt ids), reported as pass@1. **Dataset version:** `2026.06.1` (materialized at deploy; not committed). @@ -72,6 +99,107 @@ The **Diamond** subset of [Idavidrein/gpqa](https://huggingface.co/datasets/Idav Scoring: [math_500](scoring.md#5-math_500) -[HuggingFaceH4/MATH-500](https://huggingface.co/datasets/HuggingFaceH4/MATH-500) — the **500**-problem test split (from OpenAI's PRM800K work) of the Hendrycks MATH benchmark, spanning algebra, geometry, number theory, precalculus, and more at difficulty levels 1–5. Open-ended (not MCQ): the model reasons step by step and gives its final answer in `\boxed{}`. MIT-licensed but, like the other third-party datasets, materialized from a pinned revision rather than committed. Served with `metadata.repeats = 5` (2500 attempt ids), reported as pass@1. +[HuggingFaceH4/MATH-500](https://huggingface.co/datasets/HuggingFaceH4/MATH-500) — the **500**-problem test split (from OpenAI's PRM800K work) of the Hendrycks MATH benchmark, spanning algebra, geometry, number theory, precalculus, and more at difficulty levels 1–5. Open-ended (not MCQ): the model reasons step by step and gives its final answer in `\boxed{}`. MIT-licensed but, like the other third-party datasets, materialized from a pinned revision rather than committed. Reported as pass@1. -**Dataset version:** `2026.06.1` (materialized at deploy; not committed). +Two versions hold the same 500 problems and the upstream `answer` column as gold. Each reproduces one liquid_evals track, prompt and grader together: + +| Version | liquid_evals track | Prompt (one user message) | Repeats | `metadata.grader` | +|---------|--------------------|---------------------------|---------|-------------------| +| `2026.06.1` | `math500_std` (Artificial Analysis methodology) | "Solve the following math problem step by step. Put your answer inside `\boxed{}`." / problem / "Remember to put your answer inside `\boxed{}`." | 5 (2500 attempt ids) | absent, so `math_generic` | +| `2026.10.1` | `math500`, the MATH-500 on Liquid's model cards | problem / "Please reason step by step, and put your final answer within `\boxed{}`." | 5 (2500 attempt ids) | `liquid_evals_math500` | + +liquid_evals' `math500` task takes its gold from `extract_math_answer(solution)`; the builder checks that this equals `answer` on all 500 rows. One `math500` run is one pass over the 500 problems, and the model-card number is the mean of 5 runs, so `2026.10.1` serves 5 repeats: their pass@1 is that mean. + +**Dataset versions:** `2026.06.1`, `2026.10.1` (materialized at deploy; not committed). + +--- + +## 9. mme + +Scoring: [PORTING_NOTES.md](../pipette_scores/scoring/vlm_upstream/PORTING_NOTES.md) + +**2374** yes/no perception + cognition questions over 14 categories. Built from VLMEvalKit's pinned `MME.tsv` by `scripts/build_mme_dataset.py`; the prompt is the raw question and `raw_doc.image_path` pairs the two questions per image for `acc_plus`. No declared license — the authors' terms are academic-research-only / non-commercial (self-assess, especially commercial use). + +**Dataset version:** `2026.10.1` (materialized at deploy; not committed). + +--- + +## 10. mmstar + +Scoring: [PORTING_NOTES.md](../pipette_scores/scoring/vlm_upstream/PORTING_NOTES.md) + +**1500** vision-indispensable 4-option MCQs. Built from VLMEvalKit's pinned `MMStar.tsv` by `scripts/build_mmstar_dataset.py` with the VLMEvalKit MCQ prompt. No declared license; inherits CC-BY-NC (SEED-Bench) and CC-BY-NC-SA (ScienceQA) content terms. + +**Dataset version:** `2026.10.1` (materialized at deploy; not committed). + +--- + +## 11. ocrbench + +Scoring: [PORTING_NOTES.md](../pipette_scores/scoring/vlm_upstream/PORTING_NOTES.md) + +**1000** OCR questions over 10 categories (Final Score / 1000). Built from VLMEvalKit's pinned `OCRBench.tsv` by `scripts/build_ocrbench_dataset.py`; bare-question prompt, `answer` kept as the upstream stringified list, `category` case-exact. MIT benchmark code over ~29 mixed-terms image sources. + +**Dataset version:** `2026.10.1` (materialized at deploy; not committed). + +--- + +## 12. mmmu_val + +Scoring: [PORTING_NOTES.md](../pipette_scores/scoring/vlm_upstream/PORTING_NOTES.md) + +The **900** validation rows of VLMEvalKit's pinned `MMMU_DEV_VAL.tsv` (the 150 dev rows are excluded), built by `scripts/build_mmmu_val_dataset.py`. MCQ prompt with option columns A–I; open questions keep no options and are rewritten by the scorer (`MMMU_preproc`). Multi-image questions keep their `` placeholders; the loader interleaves images at those positions. Apache-2.0 card over uncleared textbook/online images (takedown clause). + +**Dataset version:** `2026.10.1` (materialized at deploy; not committed). + +--- + +## 13. realworldqa + +Scoring: [PORTING_NOTES.md](../pipette_scores/scoring/vlm_upstream/PORTING_NOTES.md) + +**765** real-world MCQs (xAI). Built from VLMEvalKit's pinned `RealWorldQA.tsv` by `scripts/build_realworldqa_dataset.py`. **CC-BY-ND-4.0**: the built parquet is Adapted Material and must not be redistributed in any form. + +**Dataset version:** `2026.10.1` (materialized at deploy; not committed). + +--- + +## 14. mmbench_en_dev + +Scoring: [PORTING_NOTES.md](../pipette_scores/scoring/vlm_upstream/PORTING_NOTES.md) + +MMBench EN dev v1.1 incl. **all circular-rotation rows** (rotations of a base question share `index % 1e6`; circular scoring needs every row). Built from VLMEvalKit's pinned `MMBench_DEV_EN_V11.tsv` by `scripts/build_mmbench_en_dev_dataset.py`; MCQ prompt with optional `Hint:` line; rotation rows' image pointers are resolved to actual bytes. Apache-2.0 repo over uncleared public/internet images. + +**Dataset version:** `2026.10.1` (materialized at deploy; not committed). + +--- + +## 15. pope + +Scoring: [PORTING_NOTES.md](../pipette_scores/scoring/vlm_upstream/PORTING_NOTES.md) + +Yes/no object-hallucination probes over COCO val2014 images (adversarial / popular / random subsets; overlapping questions carry comma-joined categories). Built from VLMEvalKit's pinned `POPE.tsv` by `scripts/build_pope_dataset.py`; bare-question prompt; headline metric is F1. MIT annotations over COCO/Flickr per-photo image terms (no redistribution). + +**Dataset version:** `2026.10.1` (materialized at deploy; not committed). + +## 16. ASR evals (Open ASR Leaderboard English, FLEURS) + +16 evals, `asr_esb_*` (the Open ASR Leaderboard's 9 public English sets) and +`asr_fleurs_` (7 locales: `en_us`, `es_419`, `pt_br`, `de_de`, `fr_fr`, +`it_it`, `nl_nl`), version `2026.10.1`, built by +`scripts/build_asr_datasets.py` from pinned Hugging Face sources in their split's +row order. Each row is one clip: `sample_id`, `doc_id` (its position in the +source split), `reference`, `language` (ISO 639-3), +`session` / `chunk_index` (chunked Earnings22), the source `audio` bytes and +their `audio_format`, `audio_sha256`, `sample_rate`, `channels`, `num_frames`, +and `raw_doc` (the source row's other fields plus `language_name`, +`language_alpha_2`, `audio_subtype`). There is no stored prompt: a prompt +profile builds the request. `/samples` streams a set's requests whole for one +profile, each clip as a 16 kHz mono WAV (16-bit PCM where its samples are 16-bit +values, 32-bit float otherwise), under the digest `/samples/digest` reports. `metadata.json`'s `content_sha256` is the +audio-inclusive fingerprint of the stored files, re-checked at load. The catalog +pins `source_sha256`, the same digest over the rows as the source shipped them, +because the builder's resampling rounds per CPU architecture. The output is +built outside the checkout (the builder refuses it inside) and is gitignored and +dockerignored. Sources, pins, integrity, profiles and metrics: +[asr-evals.md](asr-evals.md). diff --git a/packages/pipette-scores/docs/httpapi.md b/packages/pipette-scores/docs/httpapi.md index e158cda..af88177 100644 --- a/packages/pipette-scores/docs/httpapi.md +++ b/packages/pipette-scores/docs/httpapi.md @@ -8,10 +8,11 @@ Base URL: `http://localhost:8000` (default uvicorn) |----------|-------------|---------| | `PIPETTE_SCORES_DATA_DIR` | Root directory for dataset files (see [Data Storage](datasets.md)) | Parent of the `pipette_scores` package | | `PIPETTE_SCORES_VERSION` | Value reported as `runtime_version` in the `POST /score` response | `dev` | +| `PIPETTE_SCORES_PACKAGES` | The eval packages' store, `s3:///` or a directory: a set's digest is read from its index for `PIPETTE_SCORES_VERSION` rather than computed ([packages.md](packages.md#3-index)) | unset: every digest is computed | ## 2. Caching -The server caches loaded datasets and prompt samples in memory using `functools.cache`. Once an `(eval_id, dataset_name)` pair is loaded, subsequent requests for the same pair are served from cache. The cache persists for the lifetime of the process. +The server caches loaded datasets in memory for the lifetime of the process, and the prompt samples of the four `(eval_id, dataset_name, prompt_profile)` combinations it served last: a VLM set holds its base64 images (POPE about 1.1 GB) and every profile is a set of its own, so an older one is rebuilt from its parquet when asked for again. The samples' digest ([§3.3](#33-get-evalseval_iddatasetsdataset_namesamplesdigest)) is cached too, but alone: the digest route builds the samples, hashes them and drops them, so a caller probing many combinations keeps a few bytes per combination resident rather than its samples. Cold builds run one at a time per process, on a thread of their own, so a burst of them after a deploy neither multiplies memory nor holds up `POST /score`. --- @@ -40,7 +41,33 @@ Get prompt samples for an eval/dataset. Returns the prompts a client should use | `eval_id` | string | Eval identifier (e.g. `math_500`) | | `dataset_name` | string | Dataset to load | -**Response** `200 OK` — `SamplesResponse` +**Query Parameters:** + +| Parameter | Type | Description | +|-----------|------|-------------| +| `prompt_profile` | string, optional | VLM evals: replaces each prompt's text with the prompt one model family is evaluated with (VLMEvalKit's, or its model card's): `internvl`, `internvl_r1`, `granite_vision`, `minicpm_v`, `minicpm_v_cot`, `lfm_vl`, `lfm2_5_vl_3b`, `qwen3_5`, `glm_v`, `kimi_vl`, `nemotron_think`, `phi4_rv_think`, `mimo_vl` ([datasets.md §4](datasets.md#4-prompt-vs-scoring-samples)); omitted, every model gets the dataset's own prompt. **ASR evals: required**, the family whose request each clip becomes: `gemma4`, `qwen25_omni`, `voxtral`, `minicpm_o`, `lfm25_audio`, `lfm3` ([asr-evals.md §2](asr-evals.md#2-requests-the-prompt-profiles)). | + +**ASR evals: one stream.** An ASR set's samples carry their audio (SPGISpeech's +are gigabytes), so the server never builds them whole: it first learns the +set's digest (from the digest route's cache, or one pass over the set, +[§3.3](#33-get-evalseval_iddatasetsdataset_namesamplesdigest)), sends it as +`X-Samples-Digest`, then writes the body one sample at a time, each sample as +the canonical JSON its digest hashes (so a client can hash the body's samples +as they arrive). A body that would not hash to its header is cut off before its +closing `]}`: a client then holds JSON it cannot parse, never a complete body +under the wrong digest. A sample's `id` is its position: the sample at index +`i` is `sha256("|")[:12]`, which the server checks before serving. +Every served clip is re-hashed against the dataset's fingerprint and served as +a 16 kHz mono WAV, resampled once from the source's rate (e.g. MonsoonASR's +44.1 kHz) when the dataset was built, as the Open ASR Leaderboard's +`datasets.Audio(sampling_rate=16000)` does, so no clip is resampled when served: **16-bit PCM** (chunks `fmt `, +`data`) when every sample is a 16-bit value, which decodes to exactly the +samples a float WAV would carry, and **32-bit float** (chunks `fmt `, `fact`, +`data`) otherwise (a resampled or downmixed clip, a float source with finer +values); a clip is the same bytes on every call. ASR samples are not paged: an +`offset` or `limit` on an ASR request is a 400. Details: `asr-evals.md` §6. + +**Response** `200 OK` — `SamplesResponse`, with an `X-Samples-Digest` header carrying the body's digest ([§3.3](#33-get-evalseval_iddatasetsdataset_namesamplesdigest)), so a client that keeps a copy knows which content it took without a second request. ```json { @@ -61,23 +88,68 @@ Get prompt samples for an eval/dataset. Returns the prompts a client should use |-------|------|-------------| | `id` | string | Deterministic sample ID (SHA256-based, 12 chars) | | `messages` | list[ChatMessage] | Chat messages to send to the model | +| `chat_template_kwargs` | object, optional | Arguments for the server's chat template, sent with the request (ASR `voxtral`: `{"transcription": true, "language": "en"}`). Omitted when unused | + +An ASR sample (here `voxtral` on `asr_fleurs_nl_nl`): + +```json +{ + "samples": [ + { + "id": "3f1c0d9a2b7e", + "messages": [ + {"role": "user", "content": [ + {"type": "input_audio", "input_audio": {"data": "", "format": "wav"}} + ]} + ], + "chat_template_kwargs": {"transcription": true, "language": "nl"} + } + ] +} +``` **Error Responses:** | Status | Condition | |--------|-----------| +| 400 | Unknown `prompt_profile`, or one given for a text eval; an ASR request without an ASR profile, or with `offset` or `limit` | | 404 | `eval_id` not found or dataset directory does not exist | -| 500 | Error loading samples from parquet | +| 500 | Error loading samples from parquet; an ASR dataset that fails its integrity checks (fingerprint, a sample out of position, a smoke build). A clip whose bytes no longer match its recorded hash is found while streaming, so it cuts the body off instead | -**Example:** +**Examples:** ```bash curl "http://localhost:8000/evals/math_500/datasets/2026.06.1/samples" +curl "http://localhost:8000/evals/mmstar/datasets/2026.10.1/samples?prompt_profile=internvl" +curl -o voxpopuli.json "http://localhost:8000/evals/asr_esb_voxpopuli_cleaned_aa/datasets/2026.10.1/samples?prompt_profile=voxtral" +``` + +--- + +### 3.3. `GET /evals/{eval_id}/datasets/{dataset_name}/samples/digest` + +The digest of what [`/samples`](#32-get-evalseval_iddatasetsdataset_namesamples) serves for the same path and `prompt_profile`, without the samples: a few bytes for a caller that only needs to know whether its copy is still current. It takes the same parameters and returns the same errors as `/samples`. + +**Response** `200 OK` — `SamplesDigestResponse` + +```json +{"digest": "sha256:1bcdef92af9cb06377f51f075c4fb1e4da9d1b32004c72922b2933cac15a2ea3", "sample_count": 5127} +``` + +| Field | Type | Description | +|-------|------|-------------| +| `digest` | string | `sha256:` and the hex SHA-256 of the served samples, each as its JSON with keys sorted, no whitespace and non-ASCII as UTF-8, followed by `\n`, in served order. Any change to what is served (an id, a prompt's wording, an image, the order) changes it; nothing else does, so it is stable across restarts and pinnable. A client can recompute it from a `/samples` body. | +| `sample_count` | integer | Number of samples served | + +The first request for a combination builds and hashes its samples once (3 s for POPE's 5127 samples, about 1 GB of JSON), then keeps only the digest and count; later requests are served from that in microseconds. For an ASR set the first request streams the set through once, decoding every clip, and learns the digest of every ASR profile in that one pass (each clip is encoded once per distinct cut), so the other five profiles' requests are served from the cache. With `PIPETTE_SCORES_PACKAGES` set, a set this server holds and its packages index lists is not built at all: the first request reads every profile's digest from the index ([packages.md](packages.md#3-index)). + +```bash +curl "http://localhost:8000/evals/pope/datasets/2026.10.1/samples/digest" ``` --- -### 3.3. `POST /score` +### 3.4. `POST /score` Score completions against ground truth. Returns per-sample scores enriched with the original prompt messages and completions so the caller can persist full audit records without a second round trip. @@ -87,8 +159,9 @@ Score completions against ground truth. Returns per-sample scores enriched with { "eval_id": "math_500", "dataset_name": "2026.06.1", + "enable_thinking": true, "completions": [ - {"id": "a1b2c3d4e5f6", "completion": "The answer is \\boxed{42}"} + {"id": "a1b2c3d4e5f6", "completion": "The answer is \\boxed{42}", "stop_reason": "eos"} ] } ``` @@ -97,13 +170,26 @@ Score completions against ground truth. Returns per-sample scores enriched with |-------|------|-------------| | `eval_id` | string | Eval identifier | | `dataset_name` | string | Dataset to score against | +| `enable_thinking` | bool, optional | The job's `enable_thinking` model flag. Omit it when the job does not set the flag. ASR evals run with thinking off: `true` is a 400 | +| `prompt_profile` | string, optional | The run's prompt profile. VLM evals: `scored_samples[*].messages` carry the prompts that profile produced, so stored results show what the model was sent. Grading is the same, except where the profile's VLMEvalKit wrapper cuts each reply down to its answer before the benchmark's scoring: `minicpm_v_cot` grades what MiniCPM-V-4.6's `extract_answer` (minicpm_v.py:1504-1529 at 8eca8e5) returns, the last stated `Answer:` even inside an unclosed reasoning block, on every benchmark but OCRBench; `kimi_vl` grades, on MMMU and MMStar, the last `\boxed{}` after Kimi-VL's thought ([scoring/profile_answers.py](../pipette_scores/scoring/profile_answers.py)). `scored_samples[*].completion` is the reply as sent either way. ASR evals: **required**; each completion is cleaned as that profile's source says before it is scored, and the messages are shown without their audio (each audio part emptied in place). An unknown profile, or one on a text eval, is a 400 | +| `samples_digest` | string, optional | The digest of the samples the run was given (`sha256:`, as [§3.3](#33-get-evalseval_iddatasetsdataset_namesamplesdigest) reports it). When given, scoring answers `409` if this server no longer serves those samples for the eval, dataset and profile, so answers are never graded and stored against prompts the model was not sent. Any other shape is a `422` | | `completions` | list[SampleCompletion] | Completions to score | | `completions[*].id` | string | Must match a sample ID from the samples endpoint | | `completions[*].completion` | string | Raw model output text | +| `completions[*].stop_reason` | string, optional | How the client's generation ended, as plan-types names it: `eos`, `truncated`, `doom_loop`, `failure` or `unknown`. Omit it when the client reported none | + +Unknown keys are ignored, so a caller may send more per-sample fields than these. + +Before grading, every eval rewrites reasoning markup that the thinking-tag rules do not know: Gemma 4 thought channels, Cohere North's thinking and text markers, and ERNIE's `` wrapper, and on the VLM evals Kimi-VL's `◁think▷` and `` (see [scoring](scoring.md#7-reasoning-markup-and-cut-off-reasoning)). Each completion is then graded on its text, as liquid_evals grades it. On the text evals neither `enable_thinking` nor `stop_reason` changes a verdict; together they select the completions that the strict view in `context` counts wrong. On the VLM evals they, and a `prompt_profile` that opens the reasoning, also decide how a few reasoning shapes are read (scoring.md §7, VLM evals). **Response** `200 OK` — `ScoreResponse` -`context` is a flat `string → JSON value` map of eval-specific aggregates (numbers typical, other scalars allowed). +`context` is a flat `string → JSON value` map of eval-specific aggregates (numbers typical, other scalars allowed). Every eval but the ASR evals (whose `context` is below) reports two keys for the strict view: + +- `_blanked_truncated`: the number of completions it counts wrong. These are the completions with `enable_thinking: true`, a `stop_reason` of `truncated` or `doom_loop`, no reasoning close and some text. +- `_pass_at_1_blanked`: the eval's pass@1 with those completions counted wrong. + +When either request field is missing, the count is 0 and the strict pass@1 equals the graded one. ```json { @@ -127,17 +213,21 @@ Aggregates (`total`, `correct`, accuracy) are derivable from `scored_samples` an | `runtime_version` | string | Value of `PIPETTE_SCORES_VERSION` env var (default `"dev"`) | | `scored_samples` | list[ScoredSample] | Per-sample results | | `scored_samples[*].id` | string | Matches the request `completions[*].id` | -| `scored_samples[*].messages` | list[ChatMessage] | Prompt from the dataset | +| `scored_samples[*].messages` | list[ChatMessage] | Prompt from the dataset, its text only: `image_url` parts are left out, as the caller holds the images (sending them back would copy a VLM corpus, about 1 GB for POPE, with every score). ASR: each `input_audio` part stays in its place with its `format` and an empty `data`, so the turn order is auditable | | `scored_samples[*].completion` | string | Echoed from the request | -| `scored_samples[*].is_correct` | bool | Whether the completion was scored correct | +| `scored_samples[*].is_correct` | bool | Whether the completion was scored correct (ASR: its unit scored with no error; not an ASR metric) | +| `scored_samples[*].measures` | object, optional | ASR only: the sample's part in its scoring unit (below). Omitted elsewhere | +| `scored_samples[*].chat_template_kwargs` | object, optional | The template arguments the sample was served with (ASR `voxtral`). Omitted when none | **Error Responses:** | Status | Condition | |--------|-----------| +| 400 | A completion id twice or not in the dataset; or, for a VLM or ASR eval, a sample with no completion or a completion with `stop_reason: failure`: a VLM or ASR eval is scored only with an answer to every sample (an ASR sample the row filter leaves out of the score included); a missing or unknown ASR `prompt_profile`; `enable_thinking: true` on an ASR eval | | 404 | `eval_id` not found or dataset directory does not exist | +| 409 | `samples_digest` given, and this server serves other samples for the eval, dataset and profile | | 422 | Malformed body (e.g. missing `eval_id` / `dataset_name`) | -| 500 | Error loading dataset, scoring failure, or scorer returned an id not present in request completions | +| 500 | Error loading dataset (an ASR dataset that fails its integrity checks included), scoring failure, or scorer returned an id not present in request completions | **Example:** @@ -152,3 +242,41 @@ curl -X POST "http://localhost:8000/score" \ ] }' ``` + +**ASR evals** (`asr_*`): a request is scored only with an answer to every +dataset sample (excluded ones included) and no `stop_reason: "failure"`; +anything else is the 400 above, never a result with the gap scored as an empty +transcript. `context` is the dataset's result and its counts: +`error_unit` (`"word"`: every ASR eval is scored by word error rate), `error_rate`, `result`, `hits`, `substitutions`, +`deletions`, `insertions`, `reference_units`, `units_scored`, +`samples_total`, `samples_excluded`, `stop_reasons`, `prompt_profile` and +`scoring`. `error_rate` and `result` are null only for a dataset with no +reference unit. Each `scored_samples[*]` adds `measures`, sum-safe (a unit's +counts and normalized texts on one sample; an Earnings22 call's on its first +chunk, the other chunks `{"unit": }`; `{"excluded": true}` for a sample +the row filter leaves out). Field by field: +[asr-evals.md §3](asr-evals.md#3-metrics). + +An abridged response on `asr_esb_voxpopuli_cleaned_aa` (illustrative values): + +```json +{ + "context": {"error_unit": "word", + "error_rate": 0.04, "result": 4.0, "hits": 965, + "substitutions": 20, "deletions": 15, "insertions": 5, "reference_units": 1000, + "units_scored": 50, "samples_total": 50, "samples_excluded": 0, + "stop_reasons": {"eos": 50}, + "prompt_profile": "qwen25_omni", + "scoring": {"normalizer": "open_asr_english", "aligner": "kaldialign_compound", "open_asr_exclusions": true}}, + "scored_samples": [ + {"id": "4d42027687cc", + "messages": [{"role": "system", "content": "You are a speech recognition model."}, + {"role": "user", "content": [{"type": "input_audio", "input_audio": {"data": "", "format": "wav"}}, + {"type": "text", "text": "Transcribe the English audio into text without any punctuation marks."}]}], + "completion": "CEFOPs role is to contribute to the shaping …", "is_correct": false, + "measures": {"unit": "4d42027687cc", "hits": 18, "substitutions": 1, "deletions": 1, "insertions": 0, + "reference_units": 20, "reference": "cedefop s role is to contribute to the shaping …", + "hypothesis": "cefops role is to contribute to the shaping …"}} + ] +} +``` diff --git a/packages/pipette-scores/docs/packages.md b/packages/pipette-scores/docs/packages.md new file mode 100644 index 0000000..08a9b25 --- /dev/null +++ b/packages/pipette-scores/docs/packages.md @@ -0,0 +1,55 @@ +# Eval packages + +`pipette-scores packages` packs every prompt set `/samples` serves for a VLM or ASR eval into the tar pipette-mgmt hands a device, so mgmt serves the tar without fetching or packing it. The text evals have no packages. + +## 1. Command + +```bash +pipette-scores packages --datasets s3:///scores-datasets --packages s3:///scores-packages +pipette-scores packages --datasets "$PIPETTE_SCORES_DATA_DIR" --packages ./packages --eval mme pope --version 2026.11.03-abc1234 +``` + +| Option | Description | +|--------|-------------| +| `--datasets` | `s3:///` holding `//`, each dataset downloaded to a temporary directory to be packed; or a data root holding `datasets///` (as `PIPETTE_SCORES_DATA_DIR`), read in place | +| `--packages` | `s3:///` or a local directory | +| `--eval` | Only these evals (default: every VLM and ASR eval) | +| `--version` | The scores release the index is written for (default: `PIPETTE_SCORES_VERSION`); refused unset or `dev` | + +A dataset whose index for that version exists is skipped. Any other is packed for every profile `/samples` takes: a VLM set for its own prompts and each prompt profile, one profile at a time; an ASR set for each ASR profile, in one pass over its audio (the pass the server's digest makes). Before a tar is stored, every sample is rebuilt from its line and media files, and the rebuilt samples must hash to the samples digest again. A tar already stored is not written again. Each package logs one line: eval, dataset, profile, identity, digest, bytes, and `uploaded` or `already present`. A dataset that fails is logged and the others are still packed; the run then exits non-zero. A run holds one dataset on disk at a time, with the media of its packages. + +## 2. Layout + +The store is write-once, like `scores-datasets/`: every write is conditional (S3 `If-None-Match: *`), and a 412 means another run wrote the key first. + +``` +/ + tars/.tar + index///.json +``` + +A package is an uncompressed ustar of `manifest.json`, `media/.` (each image or clip once, however many samples share it) and `samples.jsonl`, sorted by path, every header field fixed. Its bytes are the ones pipette-mgmt's dataset-cache build packed from `/samples` (`vlm-eval-corpus/1`, `asr-eval-corpus/1`), so a catalog pin keeps its digest; `tests/test_packages.py` pins a test vector mgmt shares. + +## 3. Index + +Written once per scores version and dataset, after all its packages are stored: + +```json +{ + "eval_id": "asr_fleurs_nl_nl", + "dataset_version": "2026.10.1", + "scores_version": "2026.11.03-abc1234", + "profiles": [ + { + "prompt_profile": "gemma4", + "samples_digest": "sha256:804f31798e0a7ef3213062619c57d9846960cef1ce271b1f863518c61320cba0", + "sample_count": 364, + "package": {"format": "asr-eval-corpus/1", "digest": "dc3c5314d753b2cfd04cd7160d0edfaa5605e31e70958e4ba4b439f8b8f29642", "bytes": 223719936, "media": 364} + } + ] +} +``` + +`prompt_profile` is `null` for a VLM set's own prompts. A catalog entry is pinned from an index entry: `[dataset]` `format`, `adapter = "evals-server@1"`, `digest`, `bytes`; `[dataset.upstream]` `source = "evals_server"`, `eval_id`, `dataset_version`, `prompt_profile`, and `identity`, the `samples_digest` without `sha256:`. + +With `PIPETTE_SCORES_PACKAGES` set to the store, the server reads `index///.json` on the first digest request for a set it holds and takes every profile's digest from it; a set without an index file has its digest computed as before ([httpapi.md §3.3](httpapi.md#33-get-evalseval_iddatasetsdataset_namesamplesdigest)). A dataset version missing from the server's data directory is not found, even when the index lists it. diff --git a/packages/pipette-scores/docs/scoring.md b/packages/pipette-scores/docs/scoring.md index ac77fc7..285be5c 100644 --- a/packages/pipette-scores/docs/scoring.md +++ b/packages/pipette-scores/docs/scoring.md @@ -4,7 +4,7 @@ Each eval has a dedicated scorer in `pipette_scores/scoring/`. The common flow i 1. **Dispatch** — `scoring.score()` routes to the per-eval `score()` function based on `eval_id`. 2. **Build lookup** — the scorer indexes dataset samples by ID for fast matching. -3. **Evaluate** — each completion is compared to its ground-truth sample, producing a `SampleScore(id, is_correct)`. +3. **Evaluate** — each completion's reasoning markup is rewritten (see [§7](#7-reasoning-markup-and-cut-off-reasoning)), then it is compared to its ground-truth sample, producing a `SampleScore(id, is_correct)`. 4. **Enrich** — the API layer joins scores with the original prompts and completions and returns the full `ScoreResponse` (see `httpapi.md`). A completion whose `id` does not match any dataset sample will raise a `KeyError`. Clients should only submit IDs returned by the samples endpoint. @@ -30,11 +30,13 @@ For evals served with `metadata.repeats = N`, the samples endpoint expands each ## 2. ifbench -Dataset: [ifbench](datasets.md#5-ifbench) | Module: `pipette_scores/scoring/ifbench/` (scorer + bundled upstream adapter) +Dataset: [ifbench](datasets.md#5-ifbench) | Module: `pipette_scores/scoring/ifbench/` (scorer + adapter for the vendored checkers) -Precise instruction following on out-of-distribution constraints, scored with the upstream IFBench **loose** checker. +Precise instruction following on out-of-distribution constraints, scored with the IFBench **loose** checker. -Thinking tags are stripped and degenerate (repetition-collapse) output is rejected before scoring. Each completion is built into an IFBench `InputExample` and run through `test_instruction_following_loose`; the sample is correct only if **every** attached constraint passes under at least one of the upstream's 8 response transformations. Served with `metadata.repeats = 5`, reported as pass@1. +The checkers are allenai/IFBench @ `1a902f9`, the revision liquid_evals scores its `ifbench_aa` task with, vendored at `vendor/ifbench-checkers`. Four local changes are listed in that directory's README. They keep scoring offline and deterministic, and count a check that raises as not followed for that response; in loose mode the other response variants are still tried. None of them changes a verdict on an input where liquid_evals gives a stable one. + +Reasoning markup is rewritten ([§7](#7-reasoning-markup-and-cut-off-reasoning)) and thinking tags are stripped before scoring. There is no repetition filter here; the client stops runaway repetition, and the scorer evaluates the text as submitted. Each completion is built into an IFBench `InputExample` and run through `test_instruction_following_loose`; the sample is correct only if **every** attached constraint passes under at least one of the upstream's 8 response transformations. Served with `metadata.repeats = 5`, reported as pass@1. --- @@ -44,7 +46,7 @@ Dataset: [ifstruct](datasets.md#6-ifstruct) | Module: `pipette_scores/scoring/if Structured format (JSON/YAML) generation, validated against the per-sample schema by the vendored `ifstruct` validator (`validate_response`). -For each completion the validator checks: output-format parse (JSON or YAML), the fenced-code-block and no-commentary requirements when set, top-level structure (bare list vs wrapped object) and wrapper-key name, all fields against the JSON schema, and the top-level item count. The sample is correct only when there are no validation errors. Reasoning tags are **not** stripped, so visible scratch-work fails the no-commentary check. Served once per prompt (no repeats). +For each completion the validator checks: output-format parse (JSON or YAML), the fenced-code-block and no-commentary requirements when set, top-level structure (bare list vs wrapped object) and wrapper-key name, all fields against the JSON schema, and the top-level item count. The sample is correct only when there are no validation errors. The completion first goes through the markup rewrite in [§7](#7-reasoning-markup-and-cut-off-reasoning). Before these checks the validator then removes thinking blocks, once, as liquid_evals does: `…`, `[THINK]…[/THINK]`, orphan opening or closing tags, and a GPT-OSS `assistantfinal` preamble. Any other visible scratch-work makes an unfenced response fail to parse. With a fenced block, it fails only samples that set `require_no_commentary`. Served once per prompt (no repeats). --- @@ -54,14 +56,126 @@ Dataset: [gpqa_diamond](datasets.md#7-gpqa_diamond) | Module: `pipette_scores/sc Graduate-level science multiple choice (A–D), Artificial Analysis methodology. -Thinking tags are stripped, then the chosen letter is extracted with the AA MCQ extractor (`score_mcq(..., valid_options="ABCD")` — an `Answer: X` pattern with markdown-tolerant fallbacks, taking the last match) and compared to the correct option. This is **generative + regex extraction**; extraction is restricted to A-D, so an unparseable response counts as incorrect. Served with `metadata.repeats = 5`, reported as pass@1. +Reasoning markup is rewritten ([§7](#7-reasoning-markup-and-cut-off-reasoning)) and thinking tags are stripped, then the chosen letter is extracted with the AA MCQ extractor (`score_mcq(..., valid_options="ABCD")` — an `Answer: X` pattern with markdown-tolerant fallbacks, taking the last match) and compared to the correct option. This is **generative + regex extraction**; extraction is restricted to A-D, so an unparseable response counts as incorrect. Served with `metadata.repeats = 5`, reported as pass@1. --- ## 5. math_500 -Dataset: [math_500](datasets.md#8-math_500) | Module: `pipette_scores/scoring/math_500/` (scorer + `math_generic`/`prm800k` verifier) +Dataset: [math_500](datasets.md#8-math_500) | Module: `pipette_scores/scoring/math_500/` (scorer + `math_generic`/`prm800k` verifier + vendored `liquid_evals_parser`) Competition mathematics; the model reasons step by step and gives its final answer in `\boxed{}`. -Graded with `score_math_generic` from `pipette_scores/scoring/math_500/math_generic.py`: thinking tags are stripped, the boxed answer is extracted, then graded by the **PRM800K** symbolic grader (sympy equivalence — so `1/2` matches `0.5`, or reordered expressions match) with a **robust-boxed** numeric-normalization fallback. The raw completion is passed through; the verifier does its own extraction. Served with `metadata.repeats = 5`, reported as pass@1. +The dataset version picks the grader through `metadata.grader`. Either way, reasoning markup is rewritten ([§7](#7-reasoning-markup-and-cut-off-reasoning)) and thinking tags are stripped first; the grader does its own answer extraction from what is left. + +- **`2026.06.1`, `math_generic`** (no `grader` key): `score_math_generic` from `pipette_scores/scoring/math_500/math_generic.py`. The boxed answer is extracted, then graded by the **PRM800K** symbolic grader (sympy equivalence — so `1/2` matches `0.5`, or reordered expressions match) with a **robust-boxed** numeric-normalization fallback. Served with `metadata.repeats = 5`. The context counts each grading path as `math_500_method/{prm800k,robust_boxed,no_match,failed}`. +- **`2026.10.1`, `liquid_evals_math500`**: liquid_evals' `extract_math_answer` and `verify_math_answer`, vendored verbatim from `liquid_evals/parser.py` @ `d5534cf` as `pipette_scores/scoring/math_500/liquid_evals_parser.py`, called as `tasks/math500.py` calls them. `extract_math_answer` takes the last `\boxed{}` (brace-counted, so nested braces and boxes keep the outer content), else a `###`/`####` line, `ANSWER:`, "answer is …", "final answer" patterns, else the last number. `verify_math_answer` accepts an exact match, else tries `check_math_result` (Hendrycks' mathd normalization, or PRM800K-style sympy equivalence) on the extracted answer, then on the whole response with a leading "final answer:"/"answer:"/`###` removed, then on each of the response's last 150 lines. Sympy is skipped for expressions over 3000 characters, with more than two unknown letters, or with `^{`, `^(` or multi-digit exponents, and LaTeX with more than 100 macros is not parsed; there is no time limit. Served with `metadata.repeats = 5`. The context counts the pattern the answer came from as `math_500_extraction/{boxed,hash,answer_colon,answer_is,other,numeric,none}`; `math_500_unparsed_rate` is the `none` share. + +--- + +## 6. Parity with liquid_evals + +The internal eval scheduler runs these evals through [Liquid4All/liquid_evals](https://github.com/Liquid4All/liquid_evals). Another internal harness has keys with the same names; those are a different implementation and are not reproduced here. Each scorer here reproduces the matching liquid_evals task, so the same completion gets the same verdict on both sides, apart from the differences listed below: + +| Eval | liquid_evals task | Same on both sides | +|------|-------------------|--------------------| +| `ifbench` | `ifbench_aa` | Prompts, loose prompt-level verdict, checker revision (allenai `1a902f9`; local patches in `vendor/ifbench-checkers/README.md`), nltk 3.9.4, thinking-tag removal | +| `ifstruct` | `ifstruct` | Test set (byte-identical), validator verdicts, thinking-tag removal | +| `gpqa_diamond` | `gpqa_diamond_aa` | Prompt and option shuffle, AA answer extraction | +| `math_500` `2026.06.1` | `math500_std` | `MathGenericFormatter` prompt, gold answers, `score_math_generic` without the LLM fallback | +| `math_500` `2026.10.1` | `math500` | Prompt, gold answers (`extract_math_answer(solution)`, identical to `answer`), `extract_math_answer` + `verify_math_answer` code, sympy 1.14.0 and pylatexenc 2.10 | + +Differences that remain by design: + +- **Generation is client-side.** The service serves prompts only. With no overrides, liquid_evals (in-process vLLM) samples at temperature 0.6, top_p 1.0 and top_k -1, with a fresh random seed per generate call. vLLM's repetition guard is on: a 2–128-token pattern repeated 10 times stops generation. Up to 32768 new tokens are allowed within a 32768-token context, capped to the model's base context. It applies the model's chat template with no system prompt. Pipette's clients set their own policy (temperature 0.6, no fixed seed) and stop runaway repetition with their own detector. +- **Repeats.** `math500_std` generates once per problem, while `math_500` `2026.06.1` is served with 5 repeats. The expected pass@1 is the same, and the variance is lower. `math500` also runs once per problem, but the model-card number is the mean of 5 `math500` runs, each with a fresh seed. `2026.10.1` serves those 5 passes as 5 repeats in one cell, and pass@1 over the 2500 attempts equals the mean of the 5 per-pass scores. +- **Crashes.** Where a liquid_evals checker raises and the whole run gets no score, the scorer here usually returns a verdict. For example, a JSON integer literal over 4300 digits fails the ifstruct sample, and a raising IFBench check counts as not followed for that response. Pathologically nested JSON (RecursionError) still fails on both sides; here `/score` returns HTTP 500 for the batch. +- **IFBench key 286.** liquid_evals reads that row's `n_start=0` as unset and picks a random span start on every call, so a correct answer passes about 1 time in 42. Here it always passes, which can raise ifbench pass@1 by up to 0.33 pp (5 of 1500 attempts). +- **Denominators and dataset pins.** ifstruct's pass rate here counts only the completions submitted. liquid_evals counts every sample, so a missing completion is a fail there. The ifbench, gpqa_diamond and math_500 pass@1 keys here already count every sample. liquid_evals reads its Hugging Face datasets without a revision pin, while the datasets here are pinned by content hash. They are identical today. +- **Reasoning markup.** liquid_evals strips only ``, `[THINK]` and gpt-oss channels. The rewrite in [§7](#7-reasoning-markup-and-cut-off-reasoning) also reads Gemma 4, Cohere North and ERNIE markup. A completion without that markup gets the same verdict on both sides, whatever its stop reason, including one cut off inside its reasoning. +- **Scale.** Scores here are 0–1 fractions; the internal eval scheduler shows them ×100. + +--- + +## 7. Reasoning markup and cut-off reasoning + +Module: `pipette_scores/scoring/completion_text.py` (`grading_text`) + +Every scorer grades `grading_text`'s output instead of the raw completion. The thinking-tag rules then run as before: `remove_thinking_tags` for ifbench, gpqa_diamond and math_500, and the validator's own copy for ifstruct, which therefore strips once. + +**Markup rewrite** (`normalize_reasoning_markup`). Some models mark their reasoning or their answer with tokens the thinking-tag rules do not know. These are rewritten first: + +| Model | Markup in the completion | Rewrite | +|-------|--------------------------|---------| +| Gemma 4 (llama.cpp) | `<\|channel>thought…`, then the answer | Each channel becomes `…`. An unclosed channel becomes an open ``, which the orphan-open rule removes to the end. A `` with no opener is dropped. This reads the markup as the Gemma 4 chat template's `strip_thinking` does | +| Cohere North (vLLM, or llama.cpp with the markers rendered) | `<\|START_THINKING\|>`, `<\|END_THINKING\|>` | ``, `` | +| Cohere North | `<\|START_TEXT\|>…<\|END_TEXT\|>` around the answer | Dropped | +| ERNIE-4.5-Thinking (also behind vLLM's `ernie45` parser) | `…` around the answer | Dropped | + +An answer wrapper's opening tag is dropped only where a section starts: at the top of the completion, or right where its reasoning closes, after the first `` (or the first `[/THINK]` when there is no ``). Its closing tag is dropped only at the very end. A wrapper nested in another is dropped too. The same tags anywhere else are answer text and stay. A completion without this markup is unchanged. + +Gemma 4 on vLLM without `--reasoning-parser gemma4` loses both channel tokens, which are special tokens, so its completions arrive as `thought\n` with no delimiter left to cut on. + +**Cut-off reasoning.** A completion is graded on its rewritten text whatever its `stop_reason`, as liquid_evals grades every response through `remove_thinking_tags`, with no stop-reason rule. For a completion cut off before its reasoning closed, that means: + +- When the chat template opens the reasoning block in the prompt, the completion carries no tag at all and is graded whole: an answer the reasoning already states counts. +- A completion that opened `` or `[THINK]` itself loses everything from the tag on, by the orphan-open rule; what is graded is the text before the tag, usually nothing. After the rewrite this also covers an unclosed Gemma 4 thought channel and North's `<|START_THINKING|>`. +- gpt-oss cut inside its analysis channel has no `assistantfinal` transition and is graded whole. +- A completion whose reasoning closed is graded on the answer after the close, even if the answer was cut. + +**Strict view.** Scorer `2026.10.12-552094f` graded such completions as empty instead. Each eval still reports that view in the response context: + +- `_blanked_truncated` counts the completions it blanks. A completion is counted when the request carries `enable_thinking: true`, its `stop_reason` is `truncated` (the output-token cap or a full context) or `doom_loop` (a repetition stop: vLLM's `repetition_detection`, or an older client's detector), its rewritten text has no reasoning close (no ``, `[/THINK]` or gpt-oss `assistantfinal` transition), and that text is not empty. A missing `stop_reason`, or `eos`, `failure` or `unknown`, is never counted. +- `_pass_at_1_blanked` is the eval's pass@1 with those completions counted wrong: the headline that scorer reported. For ifstruct, which is served once per prompt, it is `pass_rate` with those completions counted wrong. Every eval fails an empty completion, so counting them wrong is the same as grading them empty. + +Content that a server-side reasoning parser already split (vLLM with `--reasoning-parser`, or gpt-oss's Harmony final channel) is the answer alone with no close. With thinking on, a cut answer there is counted even if its reasoning finished, so the strict view understates those cells. + +### VLM evals + +Module: `split_reasoning`, `normalize_vlm_reasoning_markup`. The VLM scorers (mme, mmstar, ocrbench, mmmu_val, realworldqa, mmbench_en_dev, pope) read each completion with `split_reasoning`. The text evals (ifbench, gpqa_diamond, math_500, ifstruct) keep `grading_text` and the rules above. + +**Markup rewrite** (`normalize_vlm_reasoning_markup`): the table above, plus: + +| Model | Markup in the completion | Rewrite | +|-------|--------------------------|---------| +| Ministral-3-Reasoning | `…` after its `[/THINK]` | ``, `` | +| Kimi-VL-A3B-Thinking-2506 | `◁think▷…◁/think▷`, plain text in its vocabulary, so the server prints it | ``, ``. A thought that never closes is removed to the end, as Kimi's own `extract_summary` returns "" for it, unless the model stopped with a complete `…` in it (see `unclosed_stop` below) | + +**Reasoning, however it was opened.** A VLM scorer reads a completion once, and gets the text to grade, a status and the strict-view flag. Reasoning counts as **opened** when any of these holds: its opener is in the text (``, `[THINK]`, and after the rewrite `◁think▷`, ``, North, Gemma channels); the run's prompt profile opens it for the model (`REASONING_OPENERS` in `prompt_profiles.py`: `phi4_rv_think` by an assistant prefill, `nemotron_think` by its `/think` system message and template, `kimi_vl` and `mimo_vl` because the model always thinks; not `internvl_r1`, whose R1 prompt has the model write `` itself); `enable_thinking` is true; or a close (``, `[/THINK]`, the gpt-oss `assistantfinal` transition) shows it was opened out of sight, as with a hidden control token such as Ministral's `[THINK]`, or a template's `\n`. The run does not have to have recorded `enable_thinking`: a client refuses it for a template with no switch. + +| Status | Meaning | +|--------|---------| +| `closed` | a close was read, and the opener was known, or is in the text with no `` or `[/THINK]` before it | +| `orphan_close` | a close was read, the opener was not known, and no opener is in the text or a `` or `[/THINK]` comes before the first one | +| `unclosed_cut` | opened, no close, generation cut (`truncated` or `doom_loop`) | +| `unclosed_stop` | opened, no close, the model stopped on its own | +| `no_reasoning` | no sign of reasoning | + +The context counts them as `reasoning_status`, a dict with all five keys. A profile whose wrapper reduces the reply to its answer (`minicpm_v_cot`, [httpapi](httpapi.md)) is read after that reduction, as main's strict view reads it: a `minicpm_v_cot` reply cut inside its thought has become its answer, so it counts as `unclosed_cut` and blanked only through `enable_thinking`, which its runs record. With a server-side reasoning parser the content is the answer alone, so a run that declared `enable_thinking` reads `unclosed_stop`; the count flags it and changes no grade. + +**Text graded.** The grading is liquid_evals', as above: a completion with no tag is graded whole, whatever its status. Complete blocks are removed (a block ends at the first close after its opener, and an opener inside it is part of it); the text through the first close that has no opener is reasoning (the first such ``, else the first such `[/THINK]`, as `remove_thinking_tags` cuts), and a later stray close stays in the text; an opener left at the end removes the rest; then the gpt-oss transition and trailing harmony markers go. The two tag families pair with each other (`[THINK]…`). Beyond that: + +- **Answer before close.** A reply that opens with a one-word answer, closes a block it never opened, and goes on to reasoning that gives no answer (`yes[/THINK][THINK]I need to determine…`, Ministral-3-Reasoning on yes/no questions) is graded on that word, when no opener was known. More than one word before the close is reasoning, as Ministral writes in `So the answer should be "yes".[/THINK]`, and such a reply is graded empty. Counted as `reasoning_answer_before_close`. The rules of liquid_evals grade such a reply empty. +- **`unclosed_stop`.** When the opener is known from the profile or in the text and the model stopped without closing, a complete `…` in the reply is graded (the last one); without it the reply is graded as above. +- **`` wrapper.** An `` that opens the text after a close and its closing tag are dropped, like the `` and North wrappers of §7's table. + +**Strict view.** `blanked_truncated` counts a completion whose reasoning was opened by its own opener, by the run's prompt profile, or by `enable_thinking: true`, whose `stop_reason` is `truncated` or `doom_loop`, whose rewritten text has no close, and whose text is not empty, whatever the row recorded: a reply that opened its own thought and was cut counts even under `enable_thinking: false`. A reply cut inside reasoning that nothing shows was opened (a hidden control token, a profile or `enable_thinking` that does not say so) cannot be told from an answer, and is graded and counted as one. + +## 8. ASR evals (Open ASR Leaderboard English, FLEURS) + +`scoring/asr/`: the Open ASR Leaderboard's English WER (its normalizer, row +filter, long-form merge and kaldialign alignment) and FLEURS scored with its +multilingual scoring (its multilingual normalizer, which spells numbers out in +the language so that digits and number words compare equal, its compound-pair +step and kaldialign) applied to FLEURS's own `transcription` references, with +the leaderboard's code vendored unchanged (`scoring/asr/NOTICE.md`). The scorer cleans each completion with the request's +prompt profile, normalizes and aligns it with the dataset's recipe, and reports +the corpus rate (summed edits over summed reference units). A result is an +answer to every sample, or no result: a run with a missing or failed sample is +refused (`/score` answers 400), never scored with the gap as an empty +transcript. The +strict view of §7 does not apply: ASR runs with thinking off. Parity with the +public reference implementations (the leaderboard's own code, with num2words +0.5.14, and `datasets` audio decoding) is tested in +`tests/test_asr_public_reference.py`; the suite headlines' definition is +`scoring/asr/suite.py`. Details: [asr-evals.md](asr-evals.md). diff --git a/packages/pipette-scores/pipette_scores/api/app.py b/packages/pipette-scores/pipette_scores/api/app.py index c30e758..f8eba29 100644 --- a/packages/pipette-scores/pipette_scores/api/app.py +++ b/packages/pipette-scores/pipette_scores/api/app.py @@ -1,26 +1,35 @@ import asyncio import functools +import hashlib import html +import json import logging import os +from concurrent.futures import ThreadPoolExecutor from collections import Counter +from collections.abc import Iterator from contextlib import asynccontextmanager from pathlib import Path -from fastapi import FastAPI, HTTPException, Response +from fastapi import FastAPI, HTTPException, Request, Response from fastapi.middleware.cors import CORSMiddleware -from fastapi.responses import HTMLResponse +from fastapi.responses import HTMLResponse, StreamingResponse -from pipette_scores import scoring +from pipette_scores import packages, scoring from pipette_scores import dataset_catalog -from pipette_scores.dataset_catalog import _DEFAULT_ROOT +from pipette_scores.dataset_catalog import _DEFAULT_ROOT, asr_prompt_profiles, prompt_profiles +from pipette_scores.scoring import profile_answers +from pipette_scores.hashing import canonical_sample, samples_digest from pipette_scores.types import ( + ASR_EVAL_IDS, + ChatMessage, DatasetSample, EvalId, EvalSample, ScoredSample, ScoreRequest, ScoreResponse, + SamplesDigestResponse, SamplesResponse, ) from pipette_scores.memory import rss_mb @@ -29,9 +38,20 @@ logger = logging.getLogger(__name__) _APP_VERSION = os.environ.get("PIPETTE_SCORES_VERSION", "dev") +# The eval packages' store (packages.py); its index for this version holds the +# sets' digests, so a restart need not recompute them. +_PACKAGES = ( + packages.open_store(os.environ["PIPETTE_SCORES_PACKAGES"]) if "PIPETTE_SCORES_PACKAGES" in os.environ else None +) _INDEX_HTML = (Path(__file__).parent / "static" / "index.html").read_text(encoding="utf-8") +# An ASR set's samples are streamed, never built whole: they carry their audio +# (SPGISpeech's are gigabytes). The body is `{"samples": [...]}` written one +# sample at a time from its canonical JSON, the bytes its digest hashes. +_ASR_BODY_OPEN = b'{"samples":[' +_ASR_BODY_CLOSE = b"]}" + @asynccontextmanager async def lifespan(_app: FastAPI): @@ -62,9 +82,97 @@ async def lifespan(_app: FastAPI): ) -@functools.cache -def _load_prompts_cached(eval_id: str, dataset_name: str) -> list[EvalSample]: - return dataset_catalog.load_prompt_samples(eval_id, dataset_name) +# Prompt sets a worker keeps: a VLM set holds its base64 images (POPE about +# 1.1 GB), and each (eval, dataset, profile) is a set of its own, so an unbounded +# cache grows with every profile served. The few most recent are kept; another +# is rebuilt from its parquet (seconds) when asked for again. +_PROMPT_SETS = 4 + + +@functools.lru_cache(maxsize=_PROMPT_SETS) +def _load_prompts_cached(eval_id: str, dataset_name: str, prompt_profile: str | None = None) -> list[EvalSample]: + # The uncached loader: the catalog's own cache would keep every set anyway. + return dataset_catalog.load_prompt_samples.__wrapped__(eval_id, dataset_name, prompt_profile=prompt_profile) + + +# Digest and sample count per (eval, dataset, profile): all the digest route +# keeps resident. A cache that pins a copy of each set probes every pinned set +# every minute, and holding their samples instead would keep every one of them +# in memory for good (POPE's alone are a gigabyte). Datasets are immutable +# within a process (a new build ships under a new name, and a deploy restarts), +# so a digest, once known, stays true. +_DIGESTS: dict[tuple[str, str, str | None], tuple[str, int]] = {} +# The digest of each list /samples served, by the list's identity: the cached +# list is the same object on every request, so it is hashed once. +_SERVED: dict[tuple[str, str, str | None], tuple[int, str]] = {} +# Cold builds run here, one at a time: each holds a whole sample set while it +# is hashed, after a deploy every pinned set is cold at once, and waiting on a +# thread of its own keeps them from holding the threads /score runs on. A +# second request for a set queued behind its build finds the digest done. +_DIGEST_BUILDER = ThreadPoolExecutor(max_workers=1, thread_name_prefix="samples-digest") + + +def _build_prompts(eval_id: str, dataset_name: str, prompt_profile: str | None) -> list[EvalSample]: + """The samples ``/samples`` serves, built afresh outside every cache.""" + return dataset_catalog.load_prompt_samples.__wrapped__(eval_id, dataset_name, prompt_profile=prompt_profile) + + +def _samples_digest(eval_id: str, dataset_name: str, prompt_profile: str | None) -> tuple[str, int]: + """The digest and count of a set, built once per process and dropped once + hashed. Runs on ``_DIGEST_BUILDER``. An ASR set is streamed through once for + every profile at a time (dataset_catalog.asr_prompt_digests), so the + probes for its other profiles find their digests done. A set this server + holds and the packages index lists for this version + (``PIPETTE_SCORES_PACKAGES``) is not built: the index gives the digest of + each of its profiles.""" + key = (eval_id, dataset_name, prompt_profile) + # Only a VLM or ASR set on disk: the index also lists versions uploaded since this server synced its datasets. + if ( + key not in _DIGESTS + and _PACKAGES is not None + and packages.profiles(eval_id) + and dataset_catalog._dataset_dir(eval_id, dataset_name).exists() + ): + index = _PACKAGES.read(packages.index_key(_APP_VERSION, eval_id, dataset_name)) + if index is not None: + for entry in json.loads(index)["profiles"]: + _DIGESTS.setdefault( + (eval_id, dataset_name, entry["prompt_profile"]), (entry["samples_digest"], entry["sample_count"]) + ) + if (known := _DIGESTS.get(key)) is None and eval_id in ASR_EVAL_IDS: + for profile, digest in dataset_catalog.asr_prompt_digests(EvalId(eval_id), dataset_name).items(): + _DIGESTS.setdefault((eval_id, dataset_name, profile), digest) + return _DIGESTS[key] + if known is None: + built = _build_prompts(eval_id, dataset_name, prompt_profile) + # A /samples request may have stored the digest meanwhile; it hashed + # the body it served, so it stands. + known = _DIGESTS.setdefault(key, (samples_digest(built), len(built))) + del built + return known + + +def _served_digest(eval_id: str, dataset_name: str, prompt_profile: str | None, samples: list[EvalSample]) -> str: + """The digest of the body ``/samples`` is about to send, hashed from that + very list (once per list); the digest route then reports it too.""" + key = (eval_id, dataset_name, prompt_profile) + served = _SERVED.get(key) + if served is not None and served[0] == id(samples): + return served[1] + digest = samples_digest(samples) + _SERVED[key] = (id(samples), digest) + known = _DIGESTS.get(key) + if known is not None and known[0] != digest: + logger.warning( + "samples changed under a cached digest eval=%s dataset=%s profile=%s was=%s now=%s", + eval_id, + dataset_name, + prompt_profile, + known[0], + digest, + ) + _DIGESTS[key] = (digest, len(samples)) + return digest @functools.cache @@ -79,15 +187,72 @@ def _validate_eval_id(eval_id: str) -> None: raise HTTPException(status_code=404, detail=f"Eval not found: {eval_id}") +def _validate_prompt_profile(eval_id: str, prompt_profile: str | None) -> None: + if eval_id in ASR_EVAL_IDS: + # An ASR dataset holds no prompt: the profile is the request. + known = ", ".join(asr_prompt_profiles.PROFILES) + if prompt_profile is None: + raise HTTPException(status_code=400, detail=f"{eval_id} needs a prompt_profile (known: {known})") + if prompt_profile not in asr_prompt_profiles.PROFILES: + raise HTTPException( + status_code=400, detail=f"Unknown ASR prompt profile: {prompt_profile} (known: {known})" + ) + return + if prompt_profile is None: + return + if prompt_profile not in prompt_profiles.PROFILES: + known = ", ".join(prompt_profiles.PROFILES) + raise HTTPException(status_code=400, detail=f"Unknown prompt profile: {prompt_profile} (known: {known})") + if eval_id not in prompt_profiles.BENCHMARKS: + raise HTTPException(status_code=400, detail=f"{eval_id} has no prompt profiles") + + +def _refuse_paging(eval_id: str, request: Request) -> None: + """ASR samples are not paged; ``offset``/``limit`` are refused rather than + ignored, so a caller expecting a page does not silently get the whole set.""" + if eval_id in ASR_EVAL_IDS and {"offset", "limit"} & set(request.query_params): + raise HTTPException( + status_code=400, + detail=f"{eval_id}'s samples are not paged: GET /samples streams the whole set, with its digest", + ) + + +def _without_images(message: ChatMessage) -> ChatMessage: + """The message as a scored sample carries it: its text without its images. + The caller holds the corpus they came from; sending them back with every + score would copy it whole (about 1 GB for POPE).""" + if isinstance(message.content, str): + return message + return message.model_copy(update={"content": [part for part in message.content if part.get("type") != "image_url"]}) + + +def _without_audio(message: ChatMessage) -> ChatMessage: + """An ASR message as a scored sample carries it: its text, and each audio + part kept as a placeholder with its format and no data, so the turn order + stays auditable. The caller holds the clips (gigabytes for some sets).""" + if isinstance(message.content, str): + return message + return message.model_copy( + update={ + "content": [ + {**part, "input_audio": {**part.get("input_audio", {}), "data": ""}} + if part.get("type") == "input_audio" + else part + for part in message.content + ] + } + ) + + def _format_id_error(label: str, ids: list[str], max_show: int = 10) -> str: head = ", ".join(ids[:max_show]) suffix = f" (and {len(ids) - max_show} more)" if len(ids) > max_show else "" return f"{label}: {head}{suffix}" -def _load_prompts_or_error(eval_id: str, dataset_name: str) -> list[EvalSample]: +def _load_prompts_or_error(eval_id: str, dataset_name: str, prompt_profile: str | None = None) -> list[EvalSample]: try: - return _load_prompts_cached(eval_id, dataset_name) + return _load_prompts_cached(eval_id, dataset_name, prompt_profile) except FileNotFoundError as e: raise HTTPException(status_code=404, detail=str(e)) except Exception as e: @@ -105,10 +270,96 @@ def _load_dataset_or_error(eval_id: str, dataset_name: str) -> list[DatasetSampl raise HTTPException(status_code=500, detail=f"Error loading dataset: {e}") +async def _samples_digest_or_error(eval_id: str, dataset_name: str, prompt_profile: str | None) -> tuple[str, int]: + if (known := _DIGESTS.get((eval_id, dataset_name, prompt_profile))) is not None: + return known + try: + # Off the event loop: the first digest of a large VLM set hashes ~1 GB, + # of an ASR set every clip of it. + loop = asyncio.get_running_loop() + return await loop.run_in_executor(_DIGEST_BUILDER, _samples_digest, eval_id, dataset_name, prompt_profile) + except FileNotFoundError as e: + raise HTTPException(status_code=404, detail=str(e)) + except Exception as e: + logger.exception("digesting samples failed eval=%s dataset=%s", eval_id, dataset_name) + raise HTTPException(status_code=500, detail=f"Error digesting samples: {e}") + + +def _asr_body(eval_id: str, dataset_name: str, prompt_profile: str, digest: str, count: int) -> Iterator[bytes]: + """The streamed ``/samples`` body of an ASR set, re-hashed as it is written. + Its ``X-Samples-Digest`` went out before the first byte, so a body that does + not hash to it is cut off before its closing bracket: a client then holds + unparseable JSON, never a complete body under the wrong digest.""" + hasher = hashlib.sha256() + served = 0 + yield _ASR_BODY_OPEN + for sample in dataset_catalog.iter_asr_prompt_samples(EvalId(eval_id), dataset_name, prompt_profile): + line = canonical_sample(sample) + hasher.update(line) + hasher.update(b"\n") + yield line if served == 0 else b"," + line + served += 1 + if (f"sha256:{hasher.hexdigest()}", served) != (digest, count): + logger.error( + "streamed samples do not match their digest eval=%s dataset=%s profile=%s sent=%s/%d streamed=%s/%d", + eval_id, + dataset_name, + prompt_profile, + digest, + count, + f"sha256:{hasher.hexdigest()}", + served, + ) + raise RuntimeError(f"{eval_id}/{dataset_name} streamed other samples than its digest {digest}") + yield _ASR_BODY_CLOSE + + @app.get("/evals/{eval_id}/datasets/{dataset_name}/samples") -async def get_samples_endpoint(eval_id: str, dataset_name: str) -> SamplesResponse: +async def get_samples_endpoint( + eval_id: str, dataset_name: str, request: Request, response: Response, prompt_profile: str | None = None +) -> SamplesResponse: + """``prompt_profile`` swaps the dataset's prompt for a model family's + VLMEvalKit prompt (dataset_catalog/prompt_profiles.py), or, for an ASR eval, + names the family whose request each clip becomes (required). The + ``X-Samples-Digest`` header is the body's digest, so a client that keeps a + copy knows which content it took without a second request. + + An ASR set comes streamed, one sample at a time, after its digest is known + (from the digest route's cache, or one pass over the set): its samples carry + their audio, so they are never held whole.""" + _validate_eval_id(eval_id) + _validate_prompt_profile(eval_id, prompt_profile) + _refuse_paging(eval_id, request) + if eval_id in ASR_EVAL_IDS: + assert prompt_profile is not None # _validate_prompt_profile requires it for ASR + digest, count = await _samples_digest_or_error(eval_id, dataset_name, prompt_profile) + return StreamingResponse( # type: ignore[return-value] + _asr_body(eval_id, dataset_name, prompt_profile, digest, count), + media_type="application/json", + headers={"X-Samples-Digest": digest}, + ) + samples = _load_prompts_or_error(eval_id, dataset_name, prompt_profile) + try: + digest = await asyncio.to_thread(_served_digest, eval_id, dataset_name, prompt_profile, samples) + except Exception as e: + logger.exception("digesting samples failed eval=%s dataset=%s", eval_id, dataset_name) + raise HTTPException(status_code=500, detail=f"Error digesting samples: {e}") + response.headers["X-Samples-Digest"] = digest + return SamplesResponse(samples=samples) + + +@app.get("/evals/{eval_id}/datasets/{dataset_name}/samples/digest") +async def get_samples_digest_endpoint( + eval_id: str, dataset_name: str, request: Request, prompt_profile: str | None = None +) -> SamplesDigestResponse: + """The digest of what ``/samples`` serves for the same request, without + the samples: a few bytes for a caller that only checks its copy is current. + The samples are built to be hashed and not kept.""" _validate_eval_id(eval_id) - return SamplesResponse(samples=_load_prompts_or_error(eval_id, dataset_name)) + _validate_prompt_profile(eval_id, prompt_profile) + _refuse_paging(eval_id, request) + digest, count = await _samples_digest_or_error(eval_id, dataset_name, prompt_profile) + return SamplesDigestResponse(digest=digest, sample_count=count) @app.post("/score") @@ -127,6 +378,22 @@ async def score_endpoint(request: ScoreRequest) -> ScoreResponse: ) _validate_eval_id(eval_id) + _validate_prompt_profile(eval_id, request.prompt_profile) + if eval_id in ASR_EVAL_IDS and request.enable_thinking: + raise HTTPException( + status_code=400, + detail=f"{eval_id} runs with thinking off: enable_thinking=true is not comparable", + ) + if request.samples_digest is not None: + served, _ = await _samples_digest_or_error(eval_id, dataset_name, request.prompt_profile) + if served != request.samples_digest: + raise HTTPException( + status_code=409, + detail=( + f"the run was given samples {request.samples_digest}, but this server serves " + f"{served} for them now: its answers are not to these prompts" + ), + ) # Reject duplicate ids up front — no dataset needed for this check, and # the per-id dicts below would silently drop all but the last. @@ -150,17 +417,48 @@ async def score_endpoint(request: ScoreRequest) -> ScoreResponse: detail=_format_id_error("completions reference ids not in dataset", unknown_ids), ) + # A VLM or ASR eval is scored only on one answer to each of its samples: + # its headline is defined over the whole dataset, and a run that lacks an + # answer is not a result, however its gaps would score. pipette-mgmt + # refuses such a run first; this holds the line for any caller. (An ASR + # sample the leaderboard's row filter leaves out of the score still needs + # its answer.) + kind = "a VLM eval" if dataset_catalog.is_vlm_eval(eval_id) else "an ASR eval" if eval_id in ASR_EVAL_IDS else None + if kind is not None: + unanswered = sorted(known_ids - {c.id for c in request.completions}) + failed = sorted(c.id for c in request.completions if c.stop_reason == "failure") + if unanswered or failed: + gaps = [ + _format_id_error(label, ids) for label, ids in (("unanswered", unanswered), ("failed", failed)) if ids + ] + raise HTTPException( + status_code=400, + detail=f"{kind} is scored only with an answer to every sample; {'; '.join(gaps)}", + ) + try: # Off-load CPU-bound scoring to a worker thread so the event loop # remains free to service other requests on this uvicorn worker. - scores, context = await asyncio.to_thread(scoring.score, eval_id, request.completions, samples) + # A profile whose VLMEvalKit wrapper cuts each reply down to its answer + # is scored on what the wrapper hands on (scoring/profile_answers.py); + # the per-sample rows below keep the reply as sent. + graded = profile_answers.apply(request.prompt_profile, eval_id, request.completions) + scores, context = await asyncio.to_thread( + scoring.score, + EvalId(eval_id), + graded, + samples, + enable_thinking=request.enable_thinking, + prompt_profile=request.prompt_profile, + ) except Exception as e: logger.exception("scoring failed eval=%s dataset=%s", eval_id, dataset_name) raise HTTPException(status_code=500, detail=f"Error scoring: {e}") # Prompts are only needed for per-sample enrichment, so load them after - # scoring — saves a parquet read on the fail-fast paths above. - prompts = _load_prompts_or_error(eval_id, dataset_name) + # scoring — saves a parquet read on the fail-fast paths above. With the + # run's prompt profile, so each sample shows the prompt the model was sent. + prompts = _load_prompts_or_error(eval_id, dataset_name, request.prompt_profile) prompts_by_id = {p.id: p for p in prompts} completions_by_id = {c.id: c for c in request.completions} @@ -174,12 +472,16 @@ async def score_endpoint(request: ScoreRequest) -> ScoreResponse: detail=f"Scorer returned unknown ids: {hallucinated_ids}", ) + # The prompts' text only: the caller holds the media. + text_only = _without_audio if eval_id in ASR_EVAL_IDS else _without_images scored_samples = [ ScoredSample( id=s.id, - messages=prompts_by_id[s.id].messages, + messages=[text_only(m) for m in prompts_by_id[s.id].messages], completion=completions_by_id[s.id].completion, is_correct=s.is_correct, + measures=s.measures, + chat_template_kwargs=prompts_by_id[s.id].chat_template_kwargs, ) for s in scores ] diff --git a/packages/pipette-scores/pipette_scores/cli.py b/packages/pipette-scores/pipette_scores/cli.py index 1f132e4..3d24157 100644 --- a/packages/pipette-scores/pipette_scores/cli.py +++ b/packages/pipette-scores/pipette_scores/cli.py @@ -3,13 +3,19 @@ import argparse import json import logging +import os import pathlib import sys +import tempfile from tabulate import tabulate -from pipette_scores import scoring, dataset_catalog -from pipette_scores.types import EvalId, SampleCompletion +from collections import Counter + +from pipette_scores import packages, scoring, dataset_catalog +from pipette_scores.dataset_catalog import prompt_profiles +from pipette_scores.scoring import profile_answers +from pipette_scores.types import ASR_EVAL_IDS, EvalId, SampleCompletion logger = logging.getLogger(__name__) @@ -24,7 +30,10 @@ def cmd_list(args: argparse.Namespace) -> None: def cmd_prompts(args: argparse.Namespace) -> None: eval_id = EvalId(args.eval) - samples = dataset_catalog.load_prompt_samples(eval_id, args.dataset, root_dir=args.data_dir) + # ASR: the profile's requests without their audio (the server streams those). + samples = dataset_catalog.load_prompt_samples( + eval_id, args.dataset, root_dir=args.data_dir, prompt_profile=args.prompt_profile + ) if args.limit: samples = samples[: args.limit] @@ -49,7 +58,7 @@ def _load_completions_jsonl(path: pathlib.Path) -> list[SampleCompletion]: if not line: continue obj = json.loads(line) - completions.append(SampleCompletion(id=obj["id"], completion=obj["completion"])) + completions.append(SampleCompletion.model_validate(obj)) return completions @@ -57,7 +66,7 @@ def _load_payload_json(path: pathlib.Path) -> list[SampleCompletion]: """Load completions from a payload.json file.""" with open(path) as f: data = json.load(f) - return [SampleCompletion(id=c["id"], completion=c["completion"]) for c in data["completions"]] + return [SampleCompletion.model_validate(c) for c in data["completions"]] def _load_completions(path: pathlib.Path) -> list[SampleCompletion]: @@ -86,8 +95,42 @@ def cmd_score(args: argparse.Namespace) -> None: # Load ground truth samples samples = dataset_catalog.load_eval_samples(eval_id, args.dataset, root_dir=args.data_dir) + # The checks /score makes before scoring: one completion per id, each a dataset sample. + known = {s.id for s in samples} + if duplicates := sorted(i for i, n in Counter(c.id for c in completions).items() if n > 1): + raise SystemExit(f"duplicate completion ids: {', '.join(duplicates[:10])}") + if unknown := sorted({c.id for c in completions} - known): + raise SystemExit(f"completions reference ids not in {eval_id}/{args.dataset}: {', '.join(unknown[:10])}") + if eval_id in ASR_EVAL_IDS and args.prompt_profile is None: + raise SystemExit(f"{eval_id} needs --prompt-profile (the family whose answers these are)") + if eval_id not in ASR_EVAL_IDS and args.prompt_profile is not None: + if args.prompt_profile not in prompt_profiles.PROFILES: + raise SystemExit( + f"Unknown prompt profile: {args.prompt_profile} (known: {', '.join(prompt_profiles.PROFILES)})" + ) + if eval_id not in prompt_profiles.BENCHMARKS: + raise SystemExit(f"{eval_id} has no prompt profiles") + if eval_id in ASR_EVAL_IDS: + # A result is an answer to every sample, or no result. + unanswered = sorted(known - {c.id for c in completions}) + failed = sorted(c.id for c in completions if c.stop_reason == "failure") + if unanswered or failed: + gaps = [ + f"{label}: {', '.join(ids[:10])}" + for label, ids in (("unanswered", unanswered), ("failed", failed)) + if ids + ] + raise SystemExit(f"an ASR eval is scored only with an answer to every sample; {'; '.join(gaps)}") + logger.info("Scoring %d completions for %s/%s", len(completions), eval_id, args.dataset) - scores, context = scoring.score(eval_id, completions, samples) + # As /score: grade the answer the profile's wrapper hands on. + scores, context = scoring.score( + eval_id, + profile_answers.apply(args.prompt_profile, eval_id, completions), + samples, + enable_thinking=args.enable_thinking, + prompt_profile=args.prompt_profile, + ) total = len(scores) correct = sum(1 for v in scores if v.is_correct) @@ -95,13 +138,24 @@ def cmd_score(args: argparse.Namespace) -> None: # Summary table print() - print( - tabulate( - [[eval_id.value, args.dataset, total, correct, f"{accuracy:.2%}"]], - headers=["EVAL", "DATASET", "TOTAL", "CORRECT", "ACCURACY"], - tablefmt="simple", + if eval_id in ASR_EVAL_IDS: + # The corpus error rate; is_correct ("no error in its unit") is no ASR metric. + result = "n/a" if context["result"] is None else f"{context['result']:.4f}%" + print( + tabulate( + [[eval_id.value, args.dataset, context["samples_total"], context["units_scored"], result]], + headers=["EVAL", "DATASET", "SAMPLES", "UNITS", "WER"], + tablefmt="simple", + ) + ) + else: + print( + tabulate( + [[eval_id.value, args.dataset, total, correct, f"{accuracy:.2%}"]], + headers=["EVAL", "DATASET", "TOTAL", "CORRECT", "ACCURACY"], + tablefmt="simple", + ) ) - ) # Eval-specific context table if context: @@ -135,6 +189,39 @@ def cmd_score(args: argparse.Namespace) -> None: print(f"\nFull results written to {args.output}", file=sys.stderr) +def cmd_packages(args: argparse.Namespace) -> None: + # The index is keyed by the release the scores server runs, which looks it up under its own version. + if not args.version or args.version == "dev": + raise SystemExit("packages needs --version or PIPETTE_SCORES_VERSION: the scores release, not dev") + store = packages.open_store(args.packages) + remote = packages.open_store(args.datasets) if args.datasets.startswith("s3://") else None + if remote is None and not pathlib.Path(args.datasets, "datasets").is_dir(): + raise SystemExit(f"{args.datasets} has no datasets/ directory") + failed = [] + for eval_id in args.eval or [e.value for e in EvalId if packages.profiles(e)]: + # Each dataset version, with the keys to download for it (none: a local one is read in place). + if remote is None: + versions = {name: [] for name in dataset_catalog.list_dataset_names(eval_id, root_dir=args.datasets)} + else: + versions = {} + for key in remote.list(f"{eval_id}/"): + versions.setdefault(key.split("/")[1], []).append(key) + for name, keys in sorted(versions.items()): + if store.read(packages.index_key(args.version, eval_id, name)) is not None: + logger.info("eval=%s dataset=%s indexed for %s already, skipped", eval_id, name, args.version) + continue + try: + with tempfile.TemporaryDirectory(prefix="datasets-") as tmp: + for key in keys: + remote.download(key, pathlib.Path(tmp, "datasets", key)) + packages.build(eval_id, name, args.datasets if remote is None else tmp, store, args.version) + except Exception: + logger.exception("eval=%s dataset=%s failed", eval_id, name) + failed.append(f"{eval_id}/{name}") + if failed: + raise SystemExit(f"{len(failed)} datasets failed: {', '.join(failed)}") + + def main() -> None: parser = argparse.ArgumentParser( prog="pipette-scores", @@ -152,6 +239,11 @@ def main() -> None: p_prompts.add_argument("--eval", required=True, help="Eval ID (e.g. ifbench, ifstruct)") p_prompts.add_argument("--dataset", required=True, help="Dataset name (e.g. default)") p_prompts.add_argument("--limit", type=int, default=None, help="Max number of samples to export") + p_prompts.add_argument( + "--prompt-profile", + default=None, + help="VLM: a model family's prompts (optional). ASR: required; the requests without their audio", + ) p_prompts.add_argument("-o", "--output", default=None, help="Output file (default: stdout)") # score @@ -161,7 +253,7 @@ def main() -> None: formatter_class=argparse.RawDescriptionHelpFormatter, epilog="""\ supported input formats: - JSONL file one {"id": "...", "completion": "..."} per line + JSONL file one {"id": "...", "completion": "...", "stop_reason"?: "..."} per line payload.json {"completions": [...]} directory looks for payload.json first, then *.jsonl shards @@ -169,6 +261,7 @@ def main() -> None: %(prog)s --eval ifbench --dataset default -f completions.jsonl %(prog)s --eval ifstruct --dataset edge_2026.03.2 -f results/payload.json %(prog)s --eval ifbench --dataset default -f results/shards_dir/ + %(prog)s --eval asr_fleurs_nl_nl --dataset 2026.10.1 --prompt-profile voxtral -f completions.jsonl """, ) p_score.add_argument("--eval", required=True, help="Eval ID (e.g. ifbench, ifstruct)") @@ -176,11 +269,40 @@ def main() -> None: p_score.add_argument("-f", "--file", required=True, help="JSONL file, payload.json, or directory") p_score.add_argument("-o", "--output", default=None, help="Write full JSON results to file") p_score.add_argument("--no-samples", action="store_true", help="Hide per-sample results table") + p_score.add_argument( + "--prompt-profile", + default=None, + help="The run's prompt profile, as /score takes it (required for ASR evals: it cleans each answer)", + ) + p_score.add_argument( + "--enable-thinking", + action=argparse.BooleanOptionalAction, + default=None, + help="The job's enable_thinking model flag, as /score takes it (default: unset)", + ) + + # packages + p_packages = sub.add_parser( + "packages", + help="Pack every VLM and ASR prompt set into the eval packages mgmt serves (docs/packages.md)", + ) + p_packages.add_argument( + "--datasets", required=True, help="s3:/// of //, or a data dir with datasets/" + ) + p_packages.add_argument("--packages", required=True, help="s3:/// or a directory to store into") + p_packages.add_argument( + "--eval", nargs="+", choices=[e.value for e in EvalId if packages.profiles(e)], help="Only these evals" + ) + p_packages.add_argument( + "--version", + default=os.environ.get("PIPETTE_SCORES_VERSION"), + help="The scores release the index is written for (default: PIPETTE_SCORES_VERSION)", + ) args = parser.parse_args() logging.basicConfig(level=logging.INFO, format="%(name)s %(levelname)s %(message)s", stream=sys.stderr) - {"list": cmd_list, "prompts": cmd_prompts, "score": cmd_score}[args.command](args) + {"list": cmd_list, "prompts": cmd_prompts, "score": cmd_score, "packages": cmd_packages}[args.command](args) if __name__ == "__main__": diff --git a/packages/pipette-scores/pipette_scores/dataset_catalog/__init__.py b/packages/pipette-scores/pipette_scores/dataset_catalog/__init__.py index 22a121c..782f875 100644 --- a/packages/pipette-scores/pipette_scores/dataset_catalog/__init__.py +++ b/packages/pipette-scores/pipette_scores/dataset_catalog/__init__.py @@ -5,14 +5,41 @@ import logging import os import pathlib +from collections.abc import Iterator import pyarrow as pa import pyarrow.parquet as pq -from pipette_scores.dataset_catalog import gpqa_diamond, ifbench, ifstruct, math_500 -from pipette_scores.types import DatasetSample, EvalId, EvalSample +from pipette_scores.dataset_catalog import ( + _asr_common, + _vlm_common, + asr_prompt_profiles, + gpqa_diamond, + ifbench, + ifstruct, + math_500, + prompt_profiles, +) +from pipette_scores.types import ASR_EVAL_IDS, DatasetSample, EvalId, EvalSample from pipette_scores.parquet_io import read_parquet +# All VLM evals share one sample shape (VlmSample) and one loader (_vlm_common). +_VLM_EVALS = { + EvalId.MME, + EvalId.MMSTAR, + EvalId.OCRBENCH, + EvalId.MMMU_VAL, + EvalId.REALWORLDQA, + EvalId.MMBENCH_EN_DEV, + EvalId.POPE, +} + + +def is_vlm_eval(eval_id: str) -> bool: + """Whether ``eval_id`` names one of the VLM evals.""" + return eval_id in _VLM_EVALS + + # ifstruct stores raw rows as JSONL (schema has nested dicts and a Union[int, list[int]] field # that does not round-trip cleanly through parquet); everything else uses parquet. _JSONL_EVALS = {EvalId.IFSTRUCT} @@ -64,6 +91,10 @@ def load_eval_samples( d = _dataset_dir(eval_id, dataset_name, root_dir) if not d.exists(): raise FileNotFoundError(f"Dataset '{dataset_name}' not found for {eval_id}: {d}") + if eval_id in _VLM_EVALS: + return _vlm_common.load_eval_samples(d) + if eval_id in ASR_EVAL_IDS: + return _asr_common.load_eval_samples(d) return { EvalId.IFBENCH: ifbench.load_eval_samples, EvalId.IFSTRUCT: ifstruct.load_eval_samples, @@ -77,10 +108,32 @@ def load_prompt_samples( eval_id: EvalId, dataset_name: str, root_dir: pathlib.Path | str | None = None, + prompt_profile: str | None = None, ) -> list[EvalSample]: + """``prompt_profile`` names a model-specific prompt (prompt_profiles.PROFILES); + only the VLM evals have them. Unknown names raise ValueError. An ASR eval's + prompts here carry no audio, each audio part a placeholder with its format + and no data (see iter_asr_prompt_samples for the requests).""" + if eval_id in ASR_EVAL_IDS: + if prompt_profile is None: + raise ValueError(f"{eval_id} needs an ASR prompt profile ({', '.join(asr_prompt_profiles.PROFILES)})") + return _asr_common.load_prompt_text_samples( + EvalId(eval_id), _dataset_dir(eval_id, dataset_name, root_dir), prompt_profile + ) + if prompt_profile is not None and eval_id not in prompt_profiles.BENCHMARKS: + raise ValueError(f"{eval_id} has no prompt profiles; they apply to the VLM evals only") + build = prompt_profiles.profile(prompt_profile) if prompt_profile is not None else None d = _dataset_dir(eval_id, dataset_name, root_dir) if not d.exists(): raise FileNotFoundError(f"Dataset '{dataset_name}' not found for {eval_id}: {d}") + if eval_id in _VLM_EVALS: + return _vlm_common.load_prompt_samples( + d, + interleave_images=eval_id == EvalId.MMMU_VAL, + prompt=functools.partial(build, EvalId(eval_id)) if build else None, + system=prompt_profiles.SYSTEM_PROMPTS.get(prompt_profile) if prompt_profile else None, + prefill=prompt_profiles.PREFILLS.get(prompt_profile) if prompt_profile else None, + ) return { EvalId.IFBENCH: ifbench.load_prompt_samples, EvalId.IFSTRUCT: ifstruct.load_prompt_samples, @@ -89,6 +142,32 @@ def load_prompt_samples( }[eval_id](d) +def iter_asr_prompt_samples( + eval_id: EvalId, + dataset_name: str, + prompt_profile: str, + root_dir: pathlib.Path | str | None = None, +) -> Iterator[EvalSample]: + """An ASR eval's requests, with their audio, one at a time in served order + (_asr_common.iter_prompt_samples). Not cached: a set's audio is gigabytes. + The dataset is found and verified before this returns.""" + d = _dataset_dir(eval_id, dataset_name, root_dir) + if not d.exists(): + raise FileNotFoundError(f"Dataset '{dataset_name}' not found for {eval_id}: {d}") + return _asr_common.iter_prompt_samples(EvalId(eval_id), d, prompt_profile) + + +def asr_prompt_digests( + eval_id: EvalId, dataset_name: str, root_dir: pathlib.Path | str | None = None +) -> dict[str, tuple[str, int]]: + """Every ASR profile's samples digest and count for one dataset, in one + pass (_asr_common.prompt_digests).""" + d = _dataset_dir(eval_id, dataset_name, root_dir) + if not d.exists(): + raise FileNotFoundError(f"Dataset '{dataset_name}' not found for {eval_id}: {d}") + return _asr_common.prompt_digests(EvalId(eval_id), d) + + def load_raw_rows(eval_id: EvalId, dataset_name: str, split: str) -> list[dict]: """Load raw rows for a named dataset (parquet, or jsonl for ifstruct).""" ext = "jsonl" if eval_id in _JSONL_EVALS else "parquet" diff --git a/packages/pipette-scores/pipette_scores/dataset_catalog/_asr_audio.py b/packages/pipette-scores/pipette_scores/dataset_catalog/_asr_audio.py new file mode 100644 index 0000000..486e0b4 --- /dev/null +++ b/packages/pipette-scores/pipette_scores/dataset_catalog/_asr_audio.py @@ -0,0 +1,166 @@ +"""A clip as every model receives it: decoded, 16 kHz mono, then the profile's cut and WAV. + +A clip the source ships at 16 kHz mono is stored as shipped; any other +(MonsoonASR's 44.1 kHz, most of URGENT24's 16 to 48 kHz) is brought to 16 kHz +mono once, by the builder (_asr_build), the same way for every prompt profile, as the Open ASR +Leaderboard prepares audio for every model it evaluates: +``normalizer/data_utils.prepare_data`` casts the audio column to +``datasets.Audio(sampling_rate=16000)`` (huggingface/open_asr_leaderboard +@ 62a457c5c32d0de4df77e9470308aae7d45b055f). How that cast decodes depends on +the installed `datasets`: up to 3.x it decodes with soundfile and resamples with +librosa, from 4.0 it decodes and resamples with torchcodec (FFmpeg). The +leaderboard pins one version in one place only, its API-provider image +(``api/Dockerfile``: ``datasets==2.19.0``, "to avoid torchcodec dependency"); +its GPU model jobs install `datasets` unpinned (``requirements/requirements_jobs.txt``) +and so, on a current install, decode through torchcodec. pipette follows the +API image's path, `datasets` 2.19.0 with soundfile and librosa, because it is the +leaderboard's only pinned one and is reproducible from pinned Python packages +without FFmpeg. Only the clips that are resampled can differ between the two +paths: MonsoonASR's and URGENT24's non-16 kHz clips (docs/asr-evals.md §4). +`datasets` 2.19.0 ``features/audio.py`` ``Audio.decode_example`` does, in order: + + array, sampling_rate = sf.read(file) # float64 + array = array.T + if self.mono: # default True + array = librosa.to_mono(array) # mean over channels + if self.sampling_rate and self.sampling_rate != sampling_rate: + array = librosa.resample(array, orig_sr=sampling_rate, target_sr=self.sampling_rate) + +``librosa.resample``'s default ``res_type="soxr_hq"`` (librosa 0.10 to 1.0) runs +``soxr.resample(y, orig_sr, target_sr, quality="soxr_hq")`` and then +``util.fix_length`` to ``ceil(n * target_sr / orig_sr)`` samples (zero-padded or +cut), in the input's dtype. `resample_16k_mono` is that sequence without the +librosa dependency; tests/test_asr_public_reference.py checks it sample for +sample against `datasets` 2.19.0 with librosa itself, on the same machine: +soxr's high quality runs libsoxr's single-precision engine, which rounds +differently per CPU architecture (a few float32 ulps between macOS arm64, Linux +x86_64 and Linux aarch64), so a resampled clip's samples are exact per +architecture, not across architectures. That is why the builder, not the +server, resamples. + +After that, a profile may cut the clip to its model's audio limit (Gemma 4: 30 s, +see asr_prompt_profiles.py) and the result is written as a WAV that carries +those samples exactly (`wav`): 16-bit PCM when every sample is a 16-bit value +(k / 32768 for an integer k in [-32768, 32767], as a 16 kHz mono 16-bit source +decodes, and as some float sources hold), 32-bit float otherwise (a resampled or +downmixed clip, a float source with finer values). Both decode to the same +float32 samples: soundfile reads PCM_16 as k / 32768, and llama.cpp's audio +loader (miniaudio, ``ma_pcm_s16_to_f32``) as k * 0.000030517578125, the same +power of two, so the PCM16 WAV is lossless and half the bytes. The WAV is +written here, not by libsndfile, whose float WAVs carry a PEAK chunk with a +timestamp: the served bytes are the same on every call (`encode`). +""" + +from __future__ import annotations + +import base64 +import io +import struct +from typing import Any + +TARGET_RATE = 16000 + + +def resample_16k_mono(audio: bytes) -> Any: + """The clip as ``datasets.Audio(sampling_rate=16000)`` (2.19.0) decodes it: + float64, the channels averaged, then resampled with soxr HQ to + ``ceil(n * 16000 / rate)`` samples when not already at 16 kHz.""" + import math + + import numpy as np + import soundfile as sf + + array, rate = sf.read(io.BytesIO(audio)) + array = array.T + if array.ndim > 1: + # librosa.to_mono: np.mean over every axis but the last. + array = np.mean(array, axis=tuple(range(array.ndim - 1))) + if rate == TARGET_RATE: + return array + import soxr + + out = soxr.resample(array, rate, TARGET_RATE, quality="soxr_hq") + size = int(math.ceil(array.shape[-1] * (float(TARGET_RATE) / rate))) + # librosa.util.fix_length: cut, or pad with zeros at the end. + if out.shape[-1] > size: + out = out[:size] + elif out.shape[-1] < size: + out = np.pad(out, (0, size - out.shape[-1]), mode="constant") + return np.asarray(out, dtype=array.dtype) + + +def float_wav(samples: Any, rate: int = TARGET_RATE) -> bytes: + """Mono ``samples`` as a 32-bit IEEE-float WAV: the RIFF header, a 16-byte + ``fmt `` chunk (format 3), a ``fact`` chunk (the frame count) and ``data``, + the layout libsndfile writes for ``subtype="FLOAT"`` without its PEAK chunk, + so the bytes depend on the samples alone. The samples are converted to + little-endian float32 as libsndfile converts them (round to nearest).""" + import numpy as np + + data = np.ascontiguousarray(np.asarray(samples).reshape(-1), dtype=" Any | None: + """``samples`` as the 16-bit integers they are, when each is exactly k / 32768 + for an integer k in [-32768, 32767]; None when any is not (a 16-bit WAV would + then round it).""" + import numpy as np + + scaled = np.asarray(samples, dtype=np.float64).reshape(-1) * PCM16_SCALE + if scaled.size and not ( + np.array_equal(scaled, np.rint(scaled)) and scaled.min() >= -PCM16_SCALE and scaled.max() <= PCM16_SCALE - 1 + ): + return None + return scaled.astype(" bytes: + """Mono 16-bit integer ``values`` as a PCM WAV: the RIFF header, a 16-byte + ``fmt `` chunk (format 1) and ``data``, the layout libsndfile writes for + ``subtype="PCM_16"``.""" + import numpy as np + + data = np.ascontiguousarray(np.asarray(values).reshape(-1), dtype=" bytes: + """Mono ``samples`` as the smallest WAV that carries them exactly: 16-bit + PCM when every sample is a 16-bit value (`pcm16_values`), 32-bit float + otherwise. Either decodes to ``samples`` as float32.""" + values = pcm16_values(samples) + return float_wav(samples, rate) if values is None else pcm16_wav(values, rate) + + +def to_16k_mono(audio: bytes, *, max_samples: int | None = None) -> Any: + """The clip's samples as served: 16 kHz mono (`resample_16k_mono`), cut to + ``max_samples`` when given.""" + mono = resample_16k_mono(audio) + return mono if max_samples is None else mono[:max_samples] + + +def encode_samples(samples: Any) -> str: + """16 kHz mono samples as the served base64 WAV (`wav`).""" + return base64.b64encode(wav(samples)).decode("ascii") + + +def encode(audio: bytes, *, max_samples: int | None = None) -> str: + """The clip as base64 WAV: 16 kHz mono, cut to ``max_samples`` when given + (`to_16k_mono`), written as 16-bit PCM where that is exact and as 32-bit + float otherwise (`wav`). The same clip and cut always give the same bytes.""" + return encode_samples(to_16k_mono(audio, max_samples=max_samples)) diff --git a/packages/pipette-scores/pipette_scores/dataset_catalog/_asr_build.py b/packages/pipette-scores/pipette_scores/dataset_catalog/_asr_build.py new file mode 100644 index 0000000..56dae30 --- /dev/null +++ b/packages/pipette-scores/pipette_scores/dataset_catalog/_asr_build.py @@ -0,0 +1,385 @@ +"""Build the ASR eval datasets from their pinned sources (build-only: needs the `build` extra). + +Each dataset is read as the Open ASR Leaderboard loads it +(huggingface/open_asr_leaderboard ``normalizer/data_utils.load_data`` and +``load_chunked_data``), at the catalog's pinned revision and in the split's row +order, so a sample's position (``doc_id``) is its row in the source split: +`datasets.load_dataset(repo, subset, split=..., revision=...)` for the +leaderboard and FLEURS sets, and for chunked Earnings22 the chunk manifest, its +WAVs and the parent set's call transcripts. The audio column is read with +`Audio(decode=False)`. A clip already at 16 kHz mono is kept as the source +ships it. Any other (MonsoonASR's 44.1 kHz, most of URGENT24) is brought to +16 kHz mono here, once, as the leaderboard's ``prepare_data`` does before +inference (_asr_audio's ``resample_16k_mono``), and stored as the WAV serving +would write: serving resamples nothing, so a set's served samples, and the +corpus pins taken from them, are the same on every CPU architecture (soxr's +resampler rounds per architecture). Each such row's ``raw_doc`` records the +source's rate and channels. + +Written to ``///``: ``test.parquet`` (one row per clip) +and ``metadata.json`` (source pins, the scoring recipe, the fingerprint, the +audio summary and the license). Rows are streamed to the parquet one row group +at a time, so a build holds one row group of audio, not the dataset's. A full +build of the canonical version must match the catalog's ``expected_samples`` and +``expected_source_sha256`` (when filled) before the file lands. A ``--limit`` +smoke build may not take the canonical version, and is never served. The output +may not be inside this repository's checkout (the sets are not redistributable). +""" + +from __future__ import annotations + +import collections +import datetime +import hashlib +import io +import json +import logging +import pathlib +from collections.abc import Iterable, Iterator +from typing import Any, cast + +import pyarrow as pa +import pyarrow.parquet as pq + +from pipette_scores.dataset_catalog import _asr_audio, asr_datasets +from pipette_scores.dataset_catalog._asr_common import content_sha256, language_fields, source_sha256 +from pipette_scores.dataset_catalog.asr_datasets import AsrDataset +from pipette_scores.hashing import short_hash + +logger = logging.getLogger(__name__) + +# Rows per parquet row group: the samples stream reads one group at a time. +ROW_GROUP_SIZE = 64 + +# Where a built set may be kept, as each metadata.json records it. It is not +# part of any fingerprint, so rewording it moves no pin. +REDISTRIBUTION = ( + "internal use only: keep a built set only in the organization's private storage, for internal, " + "non-commercial benchmarking; never commit it to a public repository, or redistribute it in any " + "form" +) + +# This repository's checkout, in the workspace layout +# (/packages/pipette-scores/pipette_scores/dataset_catalog/_asr_build.py). +_CHECKOUT = pathlib.Path(__file__).resolve().parents[4] + +_SCHEMA = pa.schema( + [ + pa.field("sample_id", pa.string()), + pa.field("doc_id", pa.int64()), + pa.field("reference", pa.string()), + pa.field("language", pa.string()), + pa.field("session", pa.string()), + pa.field("chunk_index", pa.int64()), + pa.field("audio", pa.binary()), + pa.field("audio_format", pa.string()), + pa.field("audio_sha256", pa.string()), + pa.field("sample_rate", pa.int32()), + pa.field("channels", pa.int32()), + pa.field("num_frames", pa.int64()), + pa.field("raw_doc", pa.string()), + ] +) + + +def _audio_format(path: Any) -> str: + """A name for the clip's format: its file extension when alphanumeric, else wav.""" + if isinstance(path, str) and path: + ext = pathlib.PurePath(path).suffix.lower().lstrip(".") + if ext.isalnum(): + return ext + return "wav" + + +def _hf_rows(dataset: AsrDataset) -> Iterator[dict[str, Any]]: + from datasets import Audio, load_dataset + + data = load_dataset(dataset.repo, dataset.subset, split=dataset.split, revision=dataset.revision) + data = data.cast_column("audio", Audio(decode=False)) + items = cast(Iterable[dict[str, Any]], data) + for item in items: + audio = item["audio"] + audio_bytes = audio.get("bytes") + if audio_bytes is None: + audio_bytes = pathlib.Path(audio["path"]).read_bytes() + raw = item.get(dataset.reference_column) + yield { + "audio": audio_bytes, + "audio_format": _audio_format(audio.get("path")), + "reference": str(raw).strip() if raw is not None else "", + "session": None, + "chunk_index": None, + "raw_doc": {k: v for k, v in item.items() if k != "audio"}, + } + + +def _jsonl(repo: str, revision: str, filename: str) -> list[dict[str, Any]]: + from huggingface_hub import hf_hub_download + + path = hf_hub_download(repo, filename, repo_type="dataset", revision=revision) + return [json.loads(line) for line in pathlib.Path(path).read_text().splitlines() if line] + + +def _earnings22_rows() -> Iterator[dict[str, Any]]: + """As ``load_chunked_data`` attaches them: each chunk carries its call's full + transcript (the parent set's ``transcript``, read through ``get_text``), the + call (``parent_id``) and its ``chunk_index`` in the call.""" + from huggingface_hub import snapshot_download + + parents = _jsonl( + asr_datasets.EARNINGS22_PARENTS_REPO, + asr_datasets.EARNINGS22_PARENTS_REVISION, + asr_datasets.EARNINGS22_PARENTS_FILE, + ) + references = {row["id"]: row["transcript"] for row in parents} + chunks = _jsonl( + asr_datasets.EARNINGS22_CHUNKS_REPO, + asr_datasets.EARNINGS22_CHUNKS_REVISION, + asr_datasets.EARNINGS22_CHUNKS_FILE, + ) + audio_root = pathlib.Path( + snapshot_download( + asr_datasets.EARNINGS22_CHUNKS_REPO, + repo_type="dataset", + revision=asr_datasets.EARNINGS22_CHUNKS_REVISION, + allow_patterns=["audio/*.wav"], + ) + ) + counts = collections.Counter(chunk["parent_id"] for chunk in chunks) + if len(chunks) != 341 or set(counts) != set(references) or len(references) != 6: + raise SystemExit("Cleaned Earnings22 must contain 341 chunks of six calls.") + for chunk in chunks: + yield { + "audio": (audio_root / "audio" / chunk["file_name"]).read_bytes(), + "audio_format": "wav", + "reference": references[chunk["parent_id"]], + "session": chunk["parent_id"], + "chunk_index": int(chunk["chunk_index"]), + "raw_doc": {**chunk, "parent_chunk_count": counts[chunk["parent_id"]]}, + } + + +def source_rows(dataset: AsrDataset) -> Iterator[dict[str, Any]]: + if dataset.eval_id == asr_datasets.EvalId.ASR_ESB_EARNINGS22_CLEANED_AA_CHUNKED: + return _earnings22_rows() + return _hf_rows(dataset) + + +def _resolved_data_files(dataset: AsrDataset) -> dict[str, Any]: + """The source files `datasets` resolves the load to, and the `datasets` + version that resolved them.""" + import datasets + + if dataset.eval_id == asr_datasets.EvalId.ASR_ESB_EARNINGS22_CLEANED_AA_CHUNKED: + return {} + builder = datasets.load_dataset_builder(dataset.repo, dataset.subset, revision=dataset.revision) + files = builder.config.data_files or {} + return { + "data_files": [str(f) for f in files.get(dataset.split, [])], + "datasets_version": datasets.__version__, + } + + +def _json_safe(value: Any) -> Any: + if isinstance(value, dict): + return {k: _json_safe(v) for k, v in value.items()} + if isinstance(value, list | tuple): + return [_json_safe(v) for v in value] + if isinstance(value, bytes): + return None + if isinstance(value, float) and value != value: # NaN + return None + return value + + +def _prepared(dataset: AsrDataset, rows: Iterable[dict[str, Any]], limit: int | None) -> Iterator[dict[str, Any]]: + """Rows ready to write, with each clip's header read (soundfile must read it).""" + import soundfile as sf + + lang = language_fields(dataset.language) + for position, row in enumerate(rows): + if limit is not None and position >= limit: + break + audio, audio_format = row["audio"], row["audio_format"] + info = sf.info(io.BytesIO(audio)) + raw_doc = {**_json_safe(row["raw_doc"]), **lang, "audio_subtype": info.subtype} + if info.samplerate != _asr_audio.TARGET_RATE or info.channels != 1: + # The source row's columns, for _asr_common.source_sha256 (the catalog's pin). + raw_doc |= { + "source_audio_sha256": hashlib.sha256(audio).hexdigest(), + "source_audio_format": audio_format, + "source_sample_rate": int(info.samplerate), + "source_channels": int(info.channels), + "source_num_frames": int(info.frames), + } + audio, audio_format = _asr_audio.wav(_asr_audio.resample_16k_mono(audio)), "wav" + info = sf.info(io.BytesIO(audio)) + yield { + "sample_id": short_hash(f"{dataset.eval_id.value}|{position}"), + "doc_id": position, + "reference": row["reference"], + "language": dataset.language, + "session": row["session"], + "chunk_index": row["chunk_index"], + "audio": audio, + "audio_format": audio_format, + "audio_sha256": hashlib.sha256(audio).hexdigest(), + "sample_rate": int(info.samplerate), + "channels": int(info.channels), + "num_frames": int(info.frames), + "raw_doc": json.dumps(raw_doc, ensure_ascii=False, sort_keys=True), + "_container": str(info.format), + # The stored clip's; raw_doc's audio_subtype is the source's (and hashed, so it stays). + "_subtype": str(info.subtype), + } + + +def _stream(path: pathlib.Path, rows: Iterable[dict[str, Any]]) -> list[dict[str, Any]]: + """Write ``rows`` to ``path`` one row group at a time; returns them without + their audio (the audio's size kept as ``_audio_bytes``).""" + kept: list[dict[str, Any]] = [] + batch: list[dict[str, Any]] = [] + with pq.ParquetWriter(path, _SCHEMA, compression="zstd") as writer: + + def flush() -> None: + writer.write_table(pa.Table.from_pylist(batch, schema=_SCHEMA), row_group_size=ROW_GROUP_SIZE) + batch.clear() + + for row in rows: + batch.append({k: v for k, v in row.items() if not k.startswith("_")}) + kept.append({**{k: v for k, v in row.items() if k != "audio"}, "_audio_bytes": len(row["audio"])}) + if len(batch) == ROW_GROUP_SIZE: + flush() + if batch: + flush() + return kept + + +def audio_summary(rows: list[dict[str, Any]]) -> dict[str, Any]: + subtypes = collections.Counter(r["_subtype"] for r in rows) + durations = [r["num_frames"] / r["sample_rate"] for r in rows] + return { + # As stored: 16 kHz, a clip the source ships otherwise resampled at build. + "sample_rates": dict(sorted(collections.Counter(r["sample_rate"] for r in rows).items())), + "source_sample_rates": dict( + sorted( + collections.Counter( + json.loads(r["raw_doc"]).get("source_sample_rate", r["sample_rate"]) for r in rows + ).items() + ) + ), + "channels": dict(sorted(collections.Counter(r["channels"] for r in rows).items())), + # The container soundfile reads, e.g. WAV or FLAC (the rows' audio_format + # is the clip's file extension). + "formats": dict(sorted(collections.Counter(r["_container"] for r in rows).items())), + "subtypes": dict(sorted(subtypes.items())), + "total_seconds": round(sum(durations), 3), + "max_seconds": round(max(durations, default=0.0), 3), + "clips_over_30_s": sum(d > 30.0 for d in durations), + "audio_bytes": sum(r["_audio_bytes"] for r in rows), + } + + +def write_metadata( + out_dir: pathlib.Path, + dataset: AsrDataset, + version: str, + rows: list[dict[str, Any]], + limit: int | None, + resolved: dict[str, Any] | None = None, +) -> dict[str, Any]: + source: dict[str, Any] = { + "repo": dataset.repo, + "revision": dataset.revision, + "subset": dataset.subset, + "split": dataset.split, + "reference_column": dataset.reference_column, + **(resolved or {}), + } + if dataset.eval_id == asr_datasets.EvalId.ASR_ESB_EARNINGS22_CLEANED_AA_CHUNKED: + source |= { + "manifest": asr_datasets.EARNINGS22_CHUNKS_FILE, + "reference_repo": asr_datasets.EARNINGS22_PARENTS_REPO, + "reference_revision": asr_datasets.EARNINGS22_PARENTS_REVISION, + "reference_manifest": asr_datasets.EARNINGS22_PARENTS_FILE, + } + metadata = { + "eval_id": dataset.eval_id.value, + "dataset": version, + "family": dataset.family, + "source": source, + "num_samples": len(rows), + # Audio-inclusive fingerprint (the audio through each row's audio_sha256), + # recomputed at load from the columns; see _asr_common. + "content_sha256": content_sha256(rows), + # The catalog's pin: the rows as the source shipped them (_asr_common.source_sha256). + "source_sha256": source_sha256(rows), + "scoring": dataset.scoring, + "audio": audio_summary(rows), + "built_at": datetime.datetime.now(datetime.timezone.utc).isoformat(timespec="seconds"), + # Not None for a smoke build, which is never served. + "limit": limit, + "license": dataset.license, + "license_notes": dataset.license_notes, + "redistribution": REDISTRIBUTION, + } + (out_dir / "metadata.json").write_text(json.dumps(metadata, indent=2) + "\n") + return metadata + + +def refuse_inside_checkout(out_root: pathlib.Path, checkout: pathlib.Path = _CHECKOUT) -> None: + """Refuse an output under this repository's checkout: nothing ignores every + path there, and the sets must never be committed or baked into an image.""" + if not (checkout / ".git").exists(): # an installed wheel: no checkout to protect + return + resolved = out_root.resolve() + if resolved == checkout or checkout in resolved.parents: + raise SystemExit( + f"refusing to build ASR datasets into {resolved}: it is inside the repository checkout " + f"{checkout}. Build outside it (e.g. --out /data/pipette-scores/datasets) and point " + "PIPETTE_SCORES_DATA_DIR at its parent." + ) + + +def build( + dataset: AsrDataset, + out_root: pathlib.Path, + version: str = asr_datasets.DATASET_VERSION, + *, + limit: int | None = None, + rows: Iterable[dict[str, Any]] | None = None, +) -> dict[str, Any]: + """Build one dataset into ``///``; returns its + metadata. ``rows`` replaces the pinned source (tests).""" + if limit is not None and version == asr_datasets.DATASET_VERSION: + raise SystemExit(f"a --limit build may not take the canonical version {version}; pass another --version") + refuse_inside_checkout(out_root) + target = out_root / dataset.eval_id.value / version + target.mkdir(parents=True, exist_ok=True) + partial = target / "test.parquet.partial" + from_source = rows is None + try: + kept = _stream(partial, _prepared(dataset, source_rows(dataset) if rows is None else rows, limit)) + # The catalog's count and fingerprint pin the canonical version, as the + # loader checks them (another version name is a test or a new build). + if limit is None and version == asr_datasets.DATASET_VERSION: + if dataset.expected_samples is not None and len(kept) != dataset.expected_samples: + raise SystemExit( + f"expected {dataset.expected_samples} {dataset.eval_id} clips, got {len(kept)}: " + "wrong or changed source?" + ) + digest = source_sha256(kept) + if dataset.expected_source_sha256 and digest != dataset.expected_source_sha256: + raise SystemExit( + f"{dataset.eval_id}: source_sha256 {digest} != expected {dataset.expected_source_sha256}: " + "the source or the row formatting changed; review before updating the pin." + ) + resolved = _resolved_data_files(dataset) if from_source else None + except BaseException: + partial.unlink(missing_ok=True) + raise + path = target / "test.parquet" + partial.replace(path) + metadata = write_metadata(target, dataset, version, kept, limit, resolved) + logger.info("Wrote %d clips to %s (%.1f MB)", len(kept), path, path.stat().st_size / 1e6) + return metadata diff --git a/packages/pipette-scores/pipette_scores/dataset_catalog/_asr_common.py b/packages/pipette-scores/pipette_scores/dataset_catalog/_asr_common.py new file mode 100644 index 0000000..4dccf40 --- /dev/null +++ b/packages/pipette-scores/pipette_scores/dataset_catalog/_asr_common.py @@ -0,0 +1,444 @@ +"""Serve and score the ASR eval datasets (`test.parquet` + `metadata.json`). + +A set too large for one git-LFS object (SPGISpeech, GigaSpeech) is stored as +ordered parts instead of `test.parquet`: `test-00000-of-0000N.parquet`, … , +holding the rows in order (`parquet_files`; `split_parts` writes them). The parts +are read as one table, and the fingerprint below does not depend on where the +rows are cut, so a split set keeps the build's `metadata.json` unchanged. + +The scorer reads every column but the audio; the samples stream and their +digest read the audio one row group at a time (the builder writes small row +groups) and hold one sample's request at a time, so no path holds a whole set's +audio in memory. The audio is stored at 16 kHz mono (the builder resamples a +clip the source ships otherwise, and the loader refuses any other row), and +each served clip is decoded the same way for every profile (_asr_audio). + +Integrity, checked before a dataset is scored or served (`verified_rows`): + +- ``content_sha256`` (required) is the audio-inclusive fingerprint, as VL's + field of the same name: every column, the audio through its ``audio_sha256``. + Since the audio digest is a column, it is recomputed at load without reading + any audio, and each served clip is re-hashed against that column, so the + stream carries exactly the fingerprinted bytes. +- ``source_sha256`` (`source_sha256`) is the same fingerprint over the rows as + the source shipped them: a clip the builder resampled counts with its source + bytes' digest, rate, channels, length and format (kept in its ``raw_doc``). + It does not depend on the resampling, which rounds per CPU architecture, so + any machine reproduces it. For the canonical dataset version it must equal + the catalog's ``expected_source_sha256`` (when filled). +- Order: row ``i`` has ``doc_id == i`` and ``sample_id == short_hash("|i")``, + so a sample's id is its position in the source split. +- The recipe: metadata.json's ``scoring`` and every row's ``language`` equal the + catalog's (`asr_datasets.py`), the clip count its ``expected_samples``. +- A ``--limit`` smoke build (metadata ``limit`` set) is never served or scored. +""" + +from __future__ import annotations + +import functools +import hashlib +import json +import pathlib +import re +from collections.abc import Iterator, Sequence +from typing import Any + +import pyarrow.parquet as pq + +from pipette_scores.dataset_catalog import _asr_audio +from pipette_scores.hashing import canonical_sample, short_hash +from pipette_scores.types import AsrSample, ChatMessage, EvalId, EvalSample + +# Every column but the audio bytes, whose sha256 is a column of its own. +_TEXT_COLUMNS = ( + "sample_id", + "doc_id", + "reference", + "language", + "session", + "chunk_index", + "audio_format", + "audio_sha256", + "sample_rate", + "channels", + "num_frames", + "raw_doc", +) + + +class AsrDatasetError(ValueError): + """A materialized ASR dataset that must not be scored or served.""" + + +SINGLE_FILE = "test.parquet" +_PART = re.compile(r"test-(\d{5})-of-(\d{5})\.parquet") + + +def part_name(index: int, count: int) -> str: + return f"test-{index:05d}-of-{count:05d}.parquet" + + +def parquet_files(dataset_dir: pathlib.Path) -> list[pathlib.Path]: + """The dataset's parquet files in row order: ``test.parquet``, or every part + ``test--of-.parquet`` for i = 0 … n-1. Refuses both layouts at once, a + part missing, a part of another count, or a stray ``test-*.parquet``: a dataset + must be read whole or not at all (its count and fingerprint checks would catch + a missing part too; this names the problem).""" + single = dataset_dir / SINGLE_FILE + named = sorted(p.name for p in dataset_dir.glob("test-*.parquet")) + if not named: + if not single.is_file(): + raise FileNotFoundError(f"{dataset_dir} has no {SINGLE_FILE} or test--of-.parquet parts") + return [single] + + def refuse(problem: str) -> AsrDatasetError: + return AsrDatasetError(f"ASR integrity check failed for {dataset_dir}: {problem}") + + if single.exists(): + raise refuse(f"both {SINGLE_FILE} and parts ({', '.join(named)}): keep one layout") + matches = [_PART.fullmatch(n) for n in named] + if not all(matches): + raise refuse(f"{', '.join(n for n, m in zip(named, matches) if not m)} is not a test--of-.parquet part") + count = int(matches[0].group(2)) # type: ignore[union-attr] + expected = [part_name(i, count) for i in range(count)] + if count < 1 or named != expected: + raise refuse( + f"parts {', '.join(named)} are not test--of-{count:05d}.parquet for every i from 0 to {count - 1}" + ) + return [dataset_dir / n for n in expected] + + +def _pack(sizes: Sequence[int], budget: int) -> list[list[int]]: + """Row-group indices in order, a new run started where the next would pass ``budget``.""" + runs: list[list[int]] = [[]] + used = 0 + for index, size in enumerate(sizes): + if runs[-1] and used + size > budget: + runs.append([]) + used = 0 + runs[-1].append(index) + used += size + return runs + + +def split_parts(source: pathlib.Path, out_dir: pathlib.Path, max_part_bytes: int) -> list[pathlib.Path]: + """Write ``source`` (a dataset's ``test.parquet``) into ``out_dir`` as the + fewest parts of at most ``max_part_bytes`` each, cut between row groups, the + rows, row groups, schema and compression unchanged and in order; a file within + the bound is rewritten as one ``test.parquet``. Returns the files written. Copy + ``metadata.json`` beside them as it is: ``content_sha256`` is a function of + the rows, not of the files, so it holds for the parts.""" + if out_dir.resolve() == source.parent.resolve(): + raise ValueError(f"write the parts of {source} into another directory, not beside it") + file = pq.ParquetFile(source) + meta = file.metadata + sizes = [ + sum(group.column(c).total_compressed_size for c in range(group.num_columns)) + for group in (meta.row_group(i) for i in range(meta.num_row_groups)) + ] + if not sizes: + raise ValueError(f"{source} has no row groups") + # Each part also carries a footer, at most the whole file's. + budget = max_part_bytes - (source.stat().st_size - sum(sizes)) + if max(sizes) > budget: + raise ValueError(f"{source} cannot be cut between row groups into parts of {max_part_bytes} bytes") + # The fewest parts (filling each in order until the next row group would pass + # the budget), then the most even cut into that many: the least budget that + # still needs no more parts. + count = len(_pack(sizes, budget)) + low, high = max(sizes), budget + while low < high: + middle = (low + high) // 2 + low, high = (low, middle) if len(_pack(sizes, middle)) <= count else (middle + 1, high) + groups = _pack(sizes, low) + names = [SINGLE_FILE] if len(groups) == 1 else [part_name(i, len(groups)) for i in range(len(groups))] + compression = meta.row_group(0).column(0).compression.lower() + out_dir.mkdir(parents=True, exist_ok=True) + partials: list[pathlib.Path] = [] + try: + for name, group in zip(names, groups): + partial = out_dir / f"{name}.partial" + partials.append(partial) + with pq.ParquetWriter(partial, file.schema_arrow, compression=compression) as writer: + for index in group: + table = file.read_row_group(index) + writer.write_table(table, row_group_size=table.num_rows) + if partial.stat().st_size > max_part_bytes: + raise ValueError(f"{name} came out at {partial.stat().st_size} bytes, over {max_part_bytes}") + except BaseException: + for partial in partials: + partial.unlink(missing_ok=True) + raise + final = [out_dir / name for name in names] + for partial, path in zip(partials, final): + partial.replace(path) + return final + + +def language_fields(alpha_3: str) -> dict[str, Any]: + """The language facts the prompt profiles need, fixed at build time (pycountry + is build-only): the language's English name (pycountry's ISO 639-3 reference + name), and the ISO 639-1 code pycountry gives (Voxtral's language hint), None + when there is none.""" + import pycountry + + code = alpha_3.strip().lower() + language = pycountry.languages.get(alpha_3=code) + if language is None: + raise ValueError(f"Unknown ISO 639-3 language code: {alpha_3!r}") + alpha_2 = getattr(language, "alpha_2", None) + return {"language_name": str(language.name), "language_alpha_2": alpha_2 if isinstance(alpha_2, str) else None} + + +def content_sha256(rows: Sequence[dict[str, Any]]) -> str: + """The dataset's audio-inclusive fingerprint: every column, the audio through + its ``audio_sha256``. Order-independent like VL's (the order is pinned by + ``doc_id``, a hashed column checked against the position); the builder + computes ``audio_sha256`` from the bytes, the loader recomputes this from + the columns without reading audio.""" + canonical = "\n".join( + sorted("\x1f".join("" if r.get(c) is None else str(r[c]) for c in _TEXT_COLUMNS) for r in rows) + ) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +# A resampled clip's source, as the builder records it in raw_doc, by the column it stands for. +SOURCE_KEYS = { + "source_audio_sha256": "audio_sha256", + "source_audio_format": "audio_format", + "source_sample_rate": "sample_rate", + "source_channels": "channels", + "source_num_frames": "num_frames", +} + + +def source_row(row: dict[str, Any]) -> dict[str, Any]: + """The row as it would be stored unresampled: a resampled clip's source + fields back in their columns and out of ``raw_doc``.""" + doc = json.loads(row["raw_doc"]) + if "source_audio_sha256" not in doc: + return row + source = {column: doc.pop(key) for key, column in SOURCE_KEYS.items()} + return {**row, **source, "raw_doc": json.dumps(doc, ensure_ascii=False, sort_keys=True)} + + +def source_sha256(rows: Sequence[dict[str, Any]]) -> str: + """`content_sha256` of the rows as the source shipped them (`source_row`): + the catalog's pin, the same on every CPU architecture.""" + return content_sha256([source_row(r) for r in rows]) + + +def read_metadata(dataset_dir: pathlib.Path) -> dict[str, Any]: + return json.loads((dataset_dir / "metadata.json").read_text()) + + +def _check_dataset(dataset_dir: pathlib.Path, metadata: dict[str, Any], rows: list[dict[str, Any]]) -> None: + from pipette_scores.dataset_catalog import asr_datasets + + def refuse(problem: str) -> AsrDatasetError: + return AsrDatasetError(f"ASR integrity check failed for {dataset_dir}: {problem}") + + eval_id_value = metadata.get("eval_id") + if eval_id_value != dataset_dir.parent.name: + raise refuse(f"metadata.json names eval {eval_id_value!r}, the dataset sits under {dataset_dir.parent.name!r}") + try: + dataset = asr_datasets.ASR_DATASETS[EvalId(eval_id_value)] + except (KeyError, ValueError): + raise refuse(f"{eval_id_value!r} is not an ASR eval") from None + if metadata.get("limit") is not None: + raise refuse(f"a --limit {metadata['limit']} smoke build is never served or scored") + expected = metadata.get("content_sha256") + if not expected: + raise refuse("metadata.json has no content_sha256") + actual = content_sha256(rows) + if actual != expected: + raise refuse( + f"content_sha256 {actual} != {expected} from metadata.json (each served clip is " + "re-hashed against its audio_sha256; verify_asr_datasets.py checks them all offline)" + ) + canonical = dataset_dir.name == asr_datasets.DATASET_VERSION + if canonical and dataset.expected_source_sha256: + source = source_sha256(rows) + if source != dataset.expected_source_sha256: + raise refuse( + f"source_sha256 {source} is not the catalog's expected_source_sha256 " + f"{dataset.expected_source_sha256} for {asr_datasets.DATASET_VERSION}" + ) + if canonical and dataset.expected_samples is not None and len(rows) != dataset.expected_samples: + raise refuse(f"{len(rows)} clips, the catalog expects {dataset.expected_samples}") + if metadata.get("num_samples") != len(rows): + raise refuse(f"metadata.json says {metadata.get('num_samples')} clips, the parquet holds {len(rows)}") + if metadata.get("scoring") != dataset.scoring: + raise refuse(f"scoring recipe {metadata.get('scoring')} is not the catalog's {dataset.scoring}") + for position, row in enumerate(rows): + if row["doc_id"] != position or row["sample_id"] != short_hash(f"{eval_id_value}|{position}"): + raise refuse( + f"row {position} is doc_id {row['doc_id']} / sample {row['sample_id']}: a sample's id must be " + "its position in the source split" + ) + if row["language"] != dataset.language: + raise refuse( + f"row {position} is in {row['language']!r}, the catalog's {dataset.eval_id} is {dataset.language!r}" + ) + # Serving must not resample: soxr rounds per CPU architecture. + if (row["sample_rate"], row["channels"]) != (_asr_audio.TARGET_RATE, 1): + raise refuse( + f"row {position} is {row['sample_rate']} Hz x {row['channels']}: stored clips are 16 kHz mono " + "(a build before resampling moved to the builder; rebuild it)" + ) + + +@functools.lru_cache(maxsize=64) +def _verified(dataset_dir: pathlib.Path) -> tuple[dict[str, Any], tuple[dict[str, Any], ...]]: + rows = [ + row + for path in parquet_files(dataset_dir) + for row in pq.read_table(path, columns=list(_TEXT_COLUMNS)).to_pylist() + ] + metadata = read_metadata(dataset_dir) + _check_dataset(dataset_dir, metadata, rows) + return metadata, tuple(rows) + + +def verified_rows(dataset_dir: pathlib.Path) -> tuple[dict[str, Any], ...]: + """Every row without its audio, once the dataset passes the checks above + (memoized per directory: datasets are immutable build artifacts).""" + return _verified(dataset_dir.resolve())[1] + + +def load_eval_samples(dataset_dir: pathlib.Path) -> list[AsrSample]: + metadata, rows = _verified(dataset_dir.resolve()) + scoring = metadata["scoring"] + return [ + AsrSample( + id=row["sample_id"], + reference=row["reference"], + language=row["language"], + session=row["session"], + chunk_index=row["chunk_index"], + scoring=scoring, + raw_doc=json.loads(row["raw_doc"]), + ) + for row in rows + ] + + +def num_samples(dataset_dir: pathlib.Path) -> int: + return len(verified_rows(dataset_dir)) + + +def _audio_from(dataset_dir: pathlib.Path, rows: Sequence[dict[str, Any]]) -> Iterator[bytes]: + """The audio of every row, in order, one row group read at a time (across + the parts of a split set, in order), each clip re-hashed against its row's + ``audio_sha256``.""" + position = 0 + for path in parquet_files(dataset_dir): + file = pq.ParquetFile(path) + for index in range(file.metadata.num_row_groups): + clips = file.read_row_group(index, columns=["audio"]).column("audio").to_pylist() + for audio in clips: + if hashlib.sha256(audio).hexdigest() != rows[position]["audio_sha256"]: + raise AsrDatasetError( + f"ASR integrity check failed for {dataset_dir}: row {position}'s audio does not match its " + "audio_sha256" + ) + yield audio + position += 1 + + +def encode_for(audio: bytes, profile: Any) -> str: + """The clip as ``profile`` sends it: 16 kHz mono WAV (16-bit PCM where that + is exact, else 32-bit float), cut to the profile's sample cap if it has one + (_asr_audio.encode).""" + return _asr_audio.encode(audio, max_samples=profile.audio_max_samples) + + +def _clip(eval_id: EvalId, row: dict[str, Any]) -> Any: + from pipette_scores.dataset_catalog import asr_prompt_profiles + + doc = json.loads(row["raw_doc"]) + return asr_prompt_profiles.AsrClip( + eval_id=eval_id, + language=row["language"], + language_name=doc["language_name"], + language_alpha_2=doc.get("language_alpha_2"), + ) + + +def _request(eval_id: EvalId, row: dict[str, Any], profile_name: str, audio_b64: str | None) -> EvalSample: + """One row as the profile's request; ``audio_b64=None`` keeps the audio part + as a placeholder, its format and no data (the prompt a scored sample is + shown with: the turn order stays visible, the clip is the caller's).""" + from pipette_scores.dataset_catalog import asr_prompt_profiles + + profile = asr_prompt_profiles.profile(profile_name) + clip = _clip(eval_id, row) + audio = {"type": "input_audio", "input_audio": {"data": audio_b64 or "", "format": "wav"}} + messages = profile.messages(clip, audio) + return EvalSample( + id=row["sample_id"], + messages=[ChatMessage(role=m["role"], content=m["content"]) for m in messages], + chat_template_kwargs=profile.chat_template_kwargs(clip), + ) + + +def iter_prompt_samples(eval_id: EvalId, dataset_dir: pathlib.Path, profile_name: str) -> Iterator[EvalSample]: + """The profile's requests, with their audio, for the whole set, in order, + one at a time: memory holds one row group's source audio and one + request, never the set. The dataset is verified before the first request + (`verified_rows`), and each clip is re-hashed as it is read.""" + from pipette_scores.dataset_catalog import asr_prompt_profiles + + profile = asr_prompt_profiles.profile(profile_name) + dataset_dir = dataset_dir.resolve() + rows = verified_rows(dataset_dir) + + def requests() -> Iterator[EvalSample]: + for position, audio in enumerate(_audio_from(dataset_dir, rows)): + yield _request(eval_id, rows[position], profile_name, encode_for(audio, profile)) + + return requests() + + +def iter_profile_requests(eval_id: EvalId, dataset_dir: pathlib.Path) -> Iterator[dict[str, EvalSample]]: + """Each row's request for every profile, by profile name, in served order and + in one pass over the audio: each clip is decoded once, and encoded once per + distinct cut (a profile's cap only matters for a clip longer than it). Each + request is the one `iter_prompt_samples` serves for that profile.""" + from pipette_scores.dataset_catalog import asr_prompt_profiles + + dataset_dir = dataset_dir.resolve() + rows = verified_rows(dataset_dir) + for position, audio in enumerate(_audio_from(dataset_dir, rows)): + mono = _asr_audio.to_16k_mono(audio) + encoded: dict[int | None, str] = {} + requests = {} + for name, profile in asr_prompt_profiles.PROFILES.items(): + cap = profile.audio_max_samples + cut = cap if cap is not None and len(mono) > cap else None + if cut not in encoded: + encoded[cut] = _asr_audio.encode_samples(mono if cut is None else mono[:cut]) + requests[name] = _request(eval_id, rows[position], name, encoded[cut]) + yield requests + + +def prompt_digests(eval_id: EvalId, dataset_dir: pathlib.Path) -> dict[str, tuple[str, int]]: + """Every profile's ``samples_digest`` of `iter_prompt_samples`, and the + sample count, in one pass over the audio (`iter_profile_requests`), each + request hashed into its profile's digest. A server asked for one profile's + digest learns them all for the price of one pass.""" + from pipette_scores.dataset_catalog import asr_prompt_profiles + + hashers = {name: hashlib.sha256() for name in asr_prompt_profiles.PROFILES} + for requests in iter_profile_requests(eval_id, dataset_dir): + for name, request in requests.items(): + hashers[name].update(canonical_sample(request)) + hashers[name].update(b"\n") + count = len(verified_rows(dataset_dir)) + return {name: (f"sha256:{h.hexdigest()}", count) for name, h in hashers.items()} + + +def load_prompt_text_samples(eval_id: EvalId, dataset_dir: pathlib.Path, profile_name: str) -> list[EvalSample]: + """Every request with its audio parts emptied (format kept, no data): what a + scored sample shows as its prompt.""" + return [_request(eval_id, row, profile_name, None) for row in verified_rows(dataset_dir)] diff --git a/packages/pipette-scores/pipette_scores/dataset_catalog/_vlm_build.py b/packages/pipette-scores/pipette_scores/dataset_catalog/_vlm_build.py new file mode 100644 index 0000000..efbd89b --- /dev/null +++ b/packages/pipette-scores/pipette_scores/dataset_catalog/_vlm_build.py @@ -0,0 +1,581 @@ +"""Shared build machinery for the 7 VLM benchmark dataset builder scripts. + +The one-shot builders in ``packages/pipette-scores/scripts/build__dataset.py`` +(mme, mmstar, ocrbench, mmmu_val, realworldqa, mmbench_en_dev, pope) declare a +:class:`BenchmarkSpec` (pinned VLMEvalKit TSV URL + MD5, prompt style, license +facts) and delegate here. Like the other third-party datasets (ifbench, +gpqa_diamond, math_500), the output is **materialized locally and never +committed** — a license review concluded none of the 7 benchmarks may be +redistributed in transformed form (see docs/dataset-licenses.md). + +Small pieces of VLMEvalKit logic are vendored (with source citations) instead +of importing the ``vlmeval`` package: + +- TSV parsing: ``vlmeval/smp/file.py`` ``load_tsv`` (pd.read_csv, sep='\\t') +- image pointer dedup: ``vlmeval/dataset/image_base.py`` ``ImageBaseDataset.__init__`` +- list-in-a-string parse: ``vlmeval/smp/misc.py`` ``toliststr`` (ast.literal_eval + instead of eval) +- MCQ prompt: ``vlmeval/dataset/image_mcq.py`` ``ImageMCQDataset.build_prompt`` + +Output contract (read by ``pipette_scores.dataset_catalog._vlm_common``): +``datasets///test.parquet`` with columns ``sample_id``, +``doc_id``, ``prompt``, ``answer``, ``num_images``, ``image_0..image_N`` +(raw encoded image bytes), ``raw_doc`` (JSON) — plus ``metadata.json``. +""" + +from __future__ import annotations + +import argparse +import ast +import base64 +import dataclasses +import datetime +import hashlib +import io +import json +import logging +import os +import pathlib +import string +import tempfile +from typing import Any, Callable + +import pandas as pd +import pyarrow as pa +import pyarrow.parquet as pq + +from pipette_scores.dataset_catalog._vlm_common import metadata_content_sha256 + +from pipette_scores.hashing import short_hash + +logger = logging.getLogger(__name__) + +# VLMEvalKit commit the TSV URL/MD5 pins and vendored logic were taken from. +VLMEVALKIT_COMMIT = "8eca8e5" + +# Default semantic dataset version (year.month.patch); override with --version. +DEFAULT_DATASET_VERSION = "2026.10.1" + +# Re-encode settings used only when --max-image-px triggers a resize +# (matches the legacy edge-evals extractor: JPEG quality 85, LANCZOS). +JPEG_QUALITY = 85 + +_DEFAULT_CACHE_DIR = pathlib.Path( + os.environ.get( + "PIPETTE_VLM_TSV_CACHE", + pathlib.Path.home() / ".cache" / "pipette-scores" / "vlmevalkit", + ) +) + + +@dataclasses.dataclass(frozen=True) +class BenchmarkSpec: + """Everything a per-benchmark builder script declares.""" + + eval_id: str + vlmevalkit_name: str # VLMEvalKit DATASET_URL key, e.g. 'MMBench_DEV_EN_V11' + url: str + md5: str + # 'mcq' -> VLMEvalKit ImageMCQDataset.build_prompt text (Hint/Question/Options) + # 'bare' -> the raw `question` column verbatim (MME / POPE / OCRBench) + prompt_style: str + license: str + license_notes: str + # Sample count of the full build; asserted unless --limit is used. + # None = not yet known (filled after the first build). + expected_samples: int | None = None + # Fingerprint of the full original-resolution build ('' until filled after + # the first build); asserted only for full, non-resized builds. + expected_content_sha256: str = "" + # Optional row filter applied before --limit (e.g. MMMU: keep VAL split). + filter_fn: Callable[[pd.DataFrame], pd.DataFrame] | None = None + # Extra lines appended to the post-build license notice (e.g. the + # RealWorldQA CC-BY-ND no-redistribution warning). + extra_notice: str | None = None + + +# --------------------------------------------------------------------------- +# Download + integrity +# --------------------------------------------------------------------------- + + +def _file_md5(path: pathlib.Path) -> str: + h = hashlib.md5() + with path.open("rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + + +def download_tsv( + url: str, + md5: str, + cache_dir: pathlib.Path | None = None, + insecure: bool = False, +) -> pathlib.Path: + """Download (or reuse a cached copy of) a pinned upstream TSV, verifying MD5. + + ``insecure=True`` disables TLS certificate verification only — the + opencompass.openxlab.space leaf certificate has been observed expired. + Integrity is still enforced via the pinned MD5, which comes from + VLMEvalKit's DATASET_MD5 tables (commit 8eca8e5), not from the server. + """ + import requests + + cache_dir = pathlib.Path(cache_dir) if cache_dir else _DEFAULT_CACHE_DIR + cache_dir.mkdir(parents=True, exist_ok=True) + target = cache_dir / url.rsplit("/", 1)[-1] + + if target.exists(): + if _file_md5(target) == md5: + logger.info("Using cached %s (md5 ok)", target) + return target + logger.warning("Cached %s has wrong md5, re-downloading", target) + target.unlink() + + logger.info("Downloading %s -> %s", url, target) + if insecure: + import urllib3 + + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + try: + with requests.get(url, stream=True, timeout=120, verify=not insecure) as resp: + resp.raise_for_status() + fd, tmp_name = tempfile.mkstemp(dir=cache_dir, suffix=".part") + try: + with os.fdopen(fd, "wb") as f: + for chunk in resp.iter_content(chunk_size=1 << 20): + f.write(chunk) + os.replace(tmp_name, target) + except BaseException: + pathlib.Path(tmp_name).unlink(missing_ok=True) + raise + except requests.exceptions.SSLError as e: + raise RuntimeError( + f"TLS verification failed for {url}: {e}\n" + "The opencompass.openxlab.space certificate has been observed expired. " + "Re-run with --insecure to skip TLS verification; file integrity is " + "still enforced against the MD5 pinned from VLMEvalKit." + ) from e + + got = _file_md5(target) + if got != md5: + target.unlink() + raise RuntimeError(f"MD5 mismatch for {url}: expected {md5}, got {got} — refusing to build from it") + return target + + +def load_tsv(path: pathlib.Path) -> pd.DataFrame: + # Parity with VLMEvalKit `vlmeval/smp/file.py` load_tsv: only an empty cell is + # missing, so an option whose text is "None" or "NA" is kept. + return pd.read_csv(path, sep="\t", keep_default_na=False, na_values=[""]) + + +# --------------------------------------------------------------------------- +# Image handling +# --------------------------------------------------------------------------- + + +def resolve_image_pointers(df: pd.DataFrame) -> pd.DataFrame: + """Resolve the TSV space-saving convention where an ``image`` cell holds + another row's index instead of base64 (cells <= 64 chars are pointers) — + e.g. MMBench circular-rotation rows point at their base question's image. + + Vendored from VLMEvalKit ``vlmeval/dataset/image_base.py`` + ``ImageBaseDataset.__init__`` (commit 8eca8e5). + """ + if "image" not in df.columns: + return df + indices = [str(x) for x in df["index"]] + image_map = {idx: str(img) for idx, img in zip(indices, df["image"])} + for k in image_map: + if len(image_map[k]) <= 64: + idx = image_map[k] + if idx not in image_map or len(image_map[idx]) <= 64: + raise ValueError(f"image pointer {k!r} -> {idx!r} does not resolve to a base64 image") + image_map[k] = image_map[idx] + df = df.copy() + df["image"] = [image_map[k] for k in indices] + return df + + +def _to_list_str(s: Any) -> list[str]: + # Vendored from VLMEvalKit `vlmeval/smp/misc.py` toliststr, with + # ast.literal_eval instead of eval (the cell is untrusted input). + if isinstance(s, str) and s.startswith("[") and s.endswith("]"): + return [str(x) for x in ast.literal_eval(s)] + if isinstance(s, str): + return [s] + if isinstance(s, list): + return [str(x) for x in s] + raise TypeError(f"cannot interpret image cell of type {type(s)}") + + +def images_from_cell(cell: Any) -> list[bytes]: + """Decode the ``image`` cell (one base64 string, or a stringified list of + them for multi-image rows) into raw image bytes, byte-exact.""" + if cell is None or (isinstance(cell, float) and pd.isna(cell)): + return [] + return [base64.b64decode(b64) for b64 in _to_list_str(cell)] + + +def sniff_image_format(data: bytes) -> str: + if data.startswith(b"\xff\xd8\xff"): + return "jpeg" + if data.startswith(b"\x89PNG\r\n\x1a\n"): + return "png" + if data.startswith((b"GIF87a", b"GIF89a")): + return "gif" + if data.startswith(b"RIFF") and data[8:12] == b"WEBP": + return "webp" + if data.startswith(b"BM"): + return "bmp" + return "unknown" + + +def maybe_resize(data: bytes, max_image_px: int | None) -> bytes: + """Default: return the original bytes untouched (accuracy parity with + VLMEvalKit). With ``max_image_px`` set, images whose longest side exceeds + the cap are downscaled (LANCZOS) and re-encoded as JPEG q=85; smaller + images keep their original bytes.""" + if max_image_px is None: + return data + from PIL import Image + + with Image.open(io.BytesIO(data)) as img: + w, h = img.size + if max(w, h) <= max_image_px: + return data + scale = max_image_px / max(w, h) + resized = img.convert("RGB").resize((max(1, int(w * scale)), max(1, int(h * scale))), Image.LANCZOS) + buf = io.BytesIO() + resized.save(buf, format="JPEG", quality=JPEG_QUALITY) + return buf.getvalue() + + +# --------------------------------------------------------------------------- +# Prompt building +# --------------------------------------------------------------------------- + + +def build_mcq_prompt(line: dict) -> str: + """Text part of VLMEvalKit's ``ImageMCQDataset.build_prompt`` + (``vlmeval/dataset/image_mcq.py`` @ 8eca8e5), kept byte-identical: + + [Hint: {hint}\\n] # only if present + Question: {question}\\n + Options:\\nA. {A}\\nB. {B}\\n... # letters present in the row + Please select the correct answer from the options above. \\n + """ + question = line["question"] + options = {cand: line[cand] for cand in string.ascii_uppercase if cand in line and not pd.isna(line[cand])} + options_prompt = "Options:\n" + for key, item in options.items(): + options_prompt += f"{key}. {item}\n" + hint = line["hint"] if ("hint" in line and not pd.isna(line["hint"])) else None + prompt = "" + if hint is not None: + prompt += f"Hint: {hint}\n" + prompt += f"Question: {question}\n" + if len(options): + prompt += options_prompt + prompt += "Please select the correct answer from the options above. \n" + return prompt + + +def build_bare_prompt(line: dict) -> str: + """MME / POPE / OCRBench: the raw ``question`` column, verbatim.""" + return str(line["question"]) + + +# --------------------------------------------------------------------------- +# Row assembly +# --------------------------------------------------------------------------- + + +def _json_safe(v: Any) -> Any: + """Make a pandas cell JSON-serializable (numpy scalars -> python, NaN -> None).""" + if v is None: + return None + if isinstance(v, float) and pd.isna(v): + return None + if hasattr(v, "item"): # numpy scalar + return v.item() + if isinstance(v, (bool, int, float, str)): + return v + return str(v) + + +def row_raw_doc(line: dict) -> dict: + """All non-image TSV columns, JSON-safe. ``image_path`` is kept (MME's + acc_plus pairing key); the base64 ``image`` column is dropped — the bytes + are stored in the parquet image columns.""" + return {k: _json_safe(v) for k, v in line.items() if k != "image"} + + +def build_rows( + df: pd.DataFrame, + spec: BenchmarkSpec, + max_image_px: int | None, +) -> tuple[list[dict], dict[str, int]]: + """Build parquet-ready rows. Returns (rows, image format counts). + + Each row dict holds ``_images`` (list of bytes) which ``write_dataset`` + spreads into ``image_0..image_N`` columns. + """ + prompt_fn = {"mcq": build_mcq_prompt, "bare": build_bare_prompt}[spec.prompt_style] + rows: list[dict] = [] + format_counts: dict[str, int] = {} + for pos, (_, series) in enumerate(df.iterrows()): + line = series.to_dict() + images = [maybe_resize(b, max_image_px) for b in images_from_cell(line.get("image"))] + for b in images: + fmt = sniff_image_format(b) + format_counts[fmt] = format_counts.get(fmt, 0) + 1 + index = line.get("index", pos) + answer = line.get("answer") + rows.append( + { + "sample_id": short_hash(f"{spec.eval_id}|{index}"), + "doc_id": int(index) if str(index).lstrip("-").isdigit() else pos, + "prompt": prompt_fn(line), + "answer": json.dumps(_json_safe(answer)) if not pd.isna(answer) else None, + "num_images": len(images), + "_images": images, + "raw_doc": json.dumps(row_raw_doc(line), ensure_ascii=False), + } + ) + return rows, format_counts + + +def content_sha256(rows: list[dict]) -> str: + """Fingerprint of the dataset's logical content (same idea as the + math_500 builder): hashes the canonical rows — prompt, answer, raw_doc and + the image bytes' digests — not the parquet bytes, which aren't stable + across writer versions.""" + canonical = "\n".join( + sorted( + "\x1f".join( + [ + r["sample_id"], + r["prompt"], + r["answer"] or "", + r["raw_doc"], + *[hashlib.sha256(img).hexdigest() for img in r["_images"]], + ] + ) + for r in rows + ) + ) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +# --------------------------------------------------------------------------- +# Output +# --------------------------------------------------------------------------- + + +def write_dataset(out_dir: pathlib.Path, rows: list[dict]) -> pathlib.Path: + """Write ``test.parquet`` with an explicit schema: image columns are + binary even when sparsely populated (multi-image benchmarks).""" + max_images = max((r["num_images"] for r in rows), default=0) + n_img_cols = max(max_images, 1) + + fields = [ + pa.field("sample_id", pa.string()), + pa.field("doc_id", pa.int64()), + pa.field("prompt", pa.string()), + pa.field("answer", pa.string()), + pa.field("num_images", pa.int32()), + *[pa.field(f"image_{i}", pa.binary()) for i in range(n_img_cols)], + pa.field("raw_doc", pa.string()), + ] + columns: dict[str, list] = {f.name: [] for f in fields} + for r in rows: + images = r["_images"] + for key in ("sample_id", "doc_id", "prompt", "answer", "num_images", "raw_doc"): + columns[key].append(r[key]) + for i in range(n_img_cols): + columns[f"image_{i}"].append(images[i] if i < len(images) else None) + + out_dir.mkdir(parents=True, exist_ok=True) + path = out_dir / "test.parquet" + table = pa.Table.from_pydict(columns, schema=pa.schema(fields)) + pq.write_table(table, path, compression="zstd") + return path + + +def write_metadata( + out_dir: pathlib.Path, + spec: BenchmarkSpec, + version: str, + num_samples: int, + digest: str, + metadata_digest: str, + format_counts: dict[str, int], + max_image_px: int | None, + limit: int | None, +) -> None: + metadata = { + "eval_id": spec.eval_id, + "dataset": version, + "source": { + "provider": f"VLMEvalKit pinned TSV (commit {VLMEVALKIT_COMMIT})", + "vlmevalkit_dataset": spec.vlmevalkit_name, + "url": spec.url, + "md5": spec.md5, + }, + "num_samples": num_samples, + # Image-inclusive digest (offline verify_vlm_datasets.py) + cheap + # image-free digest the serving loader re-verifies at load. + "content_sha256": digest, + "metadata_content_sha256": metadata_digest, + "built_at": datetime.datetime.now(datetime.timezone.utc).isoformat(timespec="seconds"), + # None = original resolution/bytes (accuracy parity with VLMEvalKit). + "max_image_px": max_image_px, + "jpeg_quality": JPEG_QUALITY if max_image_px is not None else None, + "image_formats": dict(sorted(format_counts.items())), + "limit": limit, + "license": spec.license, + "license_notes": spec.license_notes, + "redistribution": ( + "internal use only: keep it solely in the organization's private storage; " + "never commit it to a public repo or redistribute it in any form " + "(see docs/dataset-licenses.md)" + ), + } + (out_dir / "metadata.json").write_text(json.dumps(metadata, indent=2) + "\n") + + +def print_license_notice(spec: BenchmarkSpec, out_dir: pathlib.Path, num_samples: int) -> None: + bar = "=" * 78 + lines = [ + bar, + f"Built {spec.eval_id} ({num_samples} samples) -> {out_dir}", + "", + f"LICENSE: {spec.license}", + spec.license_notes, + "", + "This dataset was built locally for evaluation use. Do NOT commit it to a", + "repository, upload it, or redistribute it in any form. If a pipette-mgmt", + "server will serve these samples (base64 images) to clients outside your", + "organization, that is distribution too — get legal sign-off first.", + ] + if spec.extra_notice: + lines += ["", spec.extra_notice] + lines.append(bar) + print("\n".join(lines), flush=True) + + +# --------------------------------------------------------------------------- +# Build + CLI entrypoint (shared by the 7 scripts) +# --------------------------------------------------------------------------- + + +def build( + spec: BenchmarkSpec, + out_dir: pathlib.Path, + version: str = DEFAULT_DATASET_VERSION, + *, + limit: int | None = None, + max_image_px: int | None = None, + insecure: bool = False, + cache_dir: pathlib.Path | None = None, +) -> int: + """Run one benchmark's build into ``//``. Returns the + number of samples written.""" + tsv_path = download_tsv(spec.url, spec.md5, cache_dir=cache_dir, insecure=insecure) + df = load_tsv(tsv_path) + df = resolve_image_pointers(df) + if spec.filter_fn is not None: + df = spec.filter_fn(df) + if limit is not None: + df = df.head(limit) + logger.warning("--limit %d set: building a truncated dataset (smoke-test only)", limit) + elif spec.expected_samples is not None and len(df) != spec.expected_samples: + raise SystemExit(f"expected {spec.expected_samples} {spec.eval_id} rows, got {len(df)} — wrong/changed TSV?") + + rows, format_counts = build_rows(df, spec, max_image_px) + + digest = content_sha256(rows) + if limit is None and max_image_px is None and spec.expected_content_sha256: + if digest != spec.expected_content_sha256: + raise SystemExit( + f"content hash {digest} != expected {spec.expected_content_sha256} — upstream " + f"data or the row formatting changed; review before updating the pin." + ) + + target_dir = out_dir / version + path = write_dataset(target_dir, rows) + write_metadata( + target_dir, + spec, + version, + len(rows), + digest, + metadata_content_sha256(rows), + format_counts, + max_image_px, + limit, + ) + logger.info("Wrote %d rows to %s (%.1f MB)", len(rows), path, path.stat().st_size / 1e6) + print_license_notice(spec, target_dir, len(rows)) + return len(rows) + + +def run_cli(spec: BenchmarkSpec, description: str | None = None) -> None: + """Shared argparse entrypoint for the per-benchmark builder scripts.""" + parser = argparse.ArgumentParser(description=description) + parser.add_argument( + "--version", + default=DEFAULT_DATASET_VERSION, + help=f"Dataset version dir name (default: {DEFAULT_DATASET_VERSION})", + ) + parser.add_argument( + "--out-dir", + type=pathlib.Path, + default=pathlib.Path("datasets") / spec.eval_id, + help=f"datasets/{spec.eval_id} root (version subdir is created under it)", + ) + parser.add_argument( + "--limit", + type=int, + default=None, + help="Build just the first N rows (smoke-testing only; not a valid eval dataset)", + ) + parser.add_argument( + "--max-image-px", + type=int, + default=None, + help=( + "Downscale images whose longest side exceeds N px (JPEG re-encode). " + "Default keeps original bytes for accuracy parity with VLMEvalKit; " + "use e.g. 512 to shrink edge payloads." + ), + ) + parser.add_argument( + "--insecure", + action="store_true", + help=( + "Skip TLS certificate verification for the upstream TSV download " + "(opencompass.openxlab.space has served an expired cert). File " + "integrity is still enforced via the pinned MD5." + ), + ) + parser.add_argument( + "--cache-dir", + type=pathlib.Path, + default=None, + help="Where downloaded TSVs are cached (default: ~/.cache/pipette-scores/vlmevalkit)", + ) + args = parser.parse_args() + logging.basicConfig(level=logging.INFO) + build( + spec, + args.out_dir, + args.version, + limit=args.limit, + max_image_px=args.max_image_px, + insecure=args.insecure, + cache_dir=args.cache_dir, + ) diff --git a/packages/pipette-scores/pipette_scores/dataset_catalog/_vlm_common.py b/packages/pipette-scores/pipette_scores/dataset_catalog/_vlm_common.py new file mode 100755 index 0000000..5722a73 --- /dev/null +++ b/packages/pipette-scores/pipette_scores/dataset_catalog/_vlm_common.py @@ -0,0 +1,193 @@ +import base64 +import hashlib +import json +import pathlib +from collections.abc import Callable + +from pipette_scores.parquet_io import read_parquet +from pipette_scores.types import ChatMessage, EvalSample, VlmSample + + +def metadata_content_sha256(rows: list[dict]) -> str: + """Cheap, **image-free** integrity digest over the non-image columns + (``sample_id``, ``prompt``, ``answer``, ``raw_doc``). + + Recomputable at load without decoding images, so the serving path can + re-verify a materialized parquet against the fingerprint the builder + wrote to ``metadata.json`` — the VLM analogue of the text loaders' + ``content_sha256`` re-check. It deliberately does **not** hash image + bytes (that stays in the builder's image-inclusive ``content_sha256``, + verified offline by ``verify_vlm_datasets.py``); this narrows the + load-time gap to "no image-byte check" rather than "no check at all". + Order-independent (rows are sorted by id) so it's stable across writers. + """ + items = sorted( + ( + str(r["sample_id"]), + str(r["prompt"]), + "" if r.get("answer") is None else str(r["answer"]), + str(r["raw_doc"]), + ) + for r in rows + ) + canonical = "\n".join("\x1f".join(fields) for fields in items) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +def _read_verified(dataset_dir: pathlib.Path) -> list[dict]: + """Read ``test.parquet`` and re-verify its non-image digest against + ``metadata.json`` when the fingerprint is present. Absent fingerprint + (datasets built before this field) → no check, matching prior behavior. + """ + rows = read_parquet(dataset_dir / "test.parquet") + meta_path = dataset_dir / "metadata.json" + expected = None + if meta_path.is_file(): + expected = json.loads(meta_path.read_text()).get("metadata_content_sha256") + if expected: + actual = metadata_content_sha256(rows) + if actual != expected: + raise ValueError( + f"VLM integrity check failed for {dataset_dir}: metadata_content_sha256 " + f"{actual} != {expected} from metadata.json — the materialized dataset " + "doesn't match the committed fingerprint (image bytes are checked " + "separately by verify_vlm_datasets.py)." + ) + return rows + + +# Image bytes are stored exactly as encoded in the parquet (the builders keep +# the original upstream bytes by default), so the mime type is sniffed from +# the magic bytes rather than assumed. +_MAGIC_MIME = ( + (b"\xff\xd8\xff", "jpeg"), + (b"\x89PNG\r\n\x1a\n", "png"), + (b"GIF87a", "gif"), + (b"GIF89a", "gif"), + (b"BM", "bmp"), +) + + +def _image_format(image_bytes: bytes) -> str: + for magic, fmt in _MAGIC_MIME: + if image_bytes.startswith(magic): + return fmt + if image_bytes.startswith(b"RIFF") and image_bytes[8:12] == b"WEBP": + return "webp" + return "jpeg" # historical default + + +def _image_to_data_uri(image_bytes: bytes) -> str: + b64 = base64.b64encode(image_bytes).decode("ascii") + return f"data:image/{_image_format(image_bytes)};base64,{b64}" + + +def _images_from_row(row: dict) -> list[bytes]: + n = int(row.get("num_images", 0) or 0) + out = [] + for i in range(n): + b = row.get(f"image_{i}") + if b is not None: + out.append(b) + return out + + +def _interleaved_blocks(prompt: str, images: list[bytes]) -> list[dict] | None: + """Interleave images at ```` placeholders in the prompt. + + Port of VLMEvalKit ``MMMUDataset.split_MMMU`` (vlmeval/dataset/image_mcq.py + @ 8eca8e5): MMMU questions embed 1-based ```` references; the + referenced image is inserted at the placeholder position and the text kept + otherwise unchanged. Returns None when the prompt has no (well-formed) + placeholders so the caller falls back to the images-first layout. + """ + segs = prompt.split("'); we + # fall back to the default layout on anything malformed instead. + if len(seg) < 2 or not seg[0].isdigit() or seg[1] != ">": + return None + image_idx = int(seg[0]) - 1 + if not 0 <= image_idx < len(images): + return None + blocks.append( + { + "type": "image_url", + "image_url": {"url": _image_to_data_uri(images[image_idx])}, + } + ) + if seg[2:]: + blocks.append({"type": "text", "text": seg[2:]}) + return blocks + + +def _content_blocks(prompt: str, images: list[bytes], *, interleave: bool = False) -> list[dict]: + """Images first, then the prompt; ``interleave`` places them at their + ```` placeholders instead, as upstream does for MMMU only.""" + interleaved = _interleaved_blocks(prompt, images) if interleave else None + if interleaved is not None: + return interleaved + blocks: list[dict] = [ + { + "type": "image_url", + "image_url": {"url": _image_to_data_uri(img_bytes)}, + } + for img_bytes in images + ] + blocks.append({"type": "text", "text": prompt}) + return blocks + + +def load_eval_samples(dataset_dir: pathlib.Path) -> list[VlmSample]: + rows = _read_verified(dataset_dir) + return [ + VlmSample( + id=row["sample_id"], + prompt=row["prompt"], + answer=json.loads(row["answer"]) if row.get("answer") else None, + raw_doc=json.loads(row["raw_doc"]), + ) + for row in rows + ] + + +def load_prompt_samples( + dataset_dir: pathlib.Path, + *, + interleave_images: bool = False, + prompt: Callable[[dict, str], str] | None = None, + system: str | None = None, + prefill: str | None = None, +) -> list[EvalSample]: + """``prompt``, given a row's raw document and its stored prompt, returns + the text to send instead (a prompt profile); by default the stored prompt. + ``system``, if given, is sent as a system message before the user turn. + ``prefill``, if given, is sent as an assistant message after it: the start + of the reply, which the server continues.""" + rows = _read_verified(dataset_dir) + preamble = [ChatMessage(role="system", content=system)] if system else [] + postamble = [ChatMessage(role="assistant", content=prefill)] if prefill else [] + return [ + EvalSample( + id=row["sample_id"], + messages=[ + *preamble, + ChatMessage( + role="user", + content=_content_blocks( + prompt(json.loads(row["raw_doc"]), row["prompt"]) if prompt else row["prompt"], + _images_from_row(row), + interleave=interleave_images, + ), + ), + *postamble, + ], + ) + for row in rows + ] diff --git a/packages/pipette-scores/pipette_scores/dataset_catalog/asr_datasets.py b/packages/pipette-scores/pipette_scores/dataset_catalog/asr_datasets.py new file mode 100644 index 0000000..d033be2 --- /dev/null +++ b/packages/pipette-scores/pipette_scores/dataset_catalog/asr_datasets.py @@ -0,0 +1,290 @@ +"""The ASR eval datasets: the Open ASR Leaderboard's English sets, and FLEURS. + +One entry per EvalId: where each set's audio and reference come from, at which +pinned revision, how it is scored, how many clips it holds, and the fingerprint +of our full build. The builder (`_asr_build.py`) reads the sources from here and +writes the recipe into each dataset's metadata.json; the loader refuses a +dataset whose recipe, language or fingerprint differs from this catalog's. + +**Open ASR Leaderboard, English** (huggingface/open_asr_leaderboard; the +leaderboard Space hf-audio/open_asr_leaderboard). The nine public sets its +English average is taken over, as of its 09-10-2026 version ("Add URGENT24 to +avg English WER", Space ``init.py`` ``default_datasets``): AMI-Cleaned, +Earnings22-Cleaned-AA-chunked, Gigaspeech-Cleaned, LS Clean, LS Other, +SPGISpeech, URGENT24, Voice Arena Monsoon, Voxpopuli-AA-Cleaned, in that order +(the "public" columns of ``normalizer/eval_utils.score_results`` and the +``DATASET_CONFIGS`` of its job scripts, @ 62a457c5c32d0de4df77e9470308aae7d45b055f). +The Space's displayed average also folds in two private sets that no third +party can run, so the mean here is the public-set mean. The EvalIds keep the +``asr_esb_`` prefix of the leaderboard's lineage (the ESB benchmark, Gandhi et +al. 2022, arXiv 2210.13352; the dataset repo was ``esb-datasets-test-only-sorted``) +so that consumers' ids stay stable; the method is the leaderboard's, not ESB's +(ESB scored orthographic WER over other sets). + +**FLEURS** (google/fleurs, Conneau et al. 2022, arXiv 2205.12446): English and +the six languages of the Open ASR Leaderboard's multilingual FLEURS runs, scored +with the leaderboard's multilingual scoring (``score_results`` with +``multilingual=True`` and the language, @ 62a457c5c32d0de4df77e9470308aae7d45b055f: +its normalizer, which spells numbers out in the language so that digits and +number words compare equal, and its compound-aware kaldialign word error rate) +applied to FLEURS's own ``transcription`` references. The leaderboard's +published FLEURS rows use its own cleaned copy of the references instead, so +its numbers can differ slightly from these. + +The leaderboard loads its datasets unpinned; pinning each source at a revision +is pipette's addition, for reproducibility. +""" + +from __future__ import annotations + +import dataclasses + +from pipette_scores.scoring.asr.align import Aligner +from pipette_scores.scoring.asr.normalize import NormalizerRecipe +from pipette_scores.types import ASR_EVAL_ORDER, EvalId + +DATASET_VERSION = "2026.10.1" + +# hf-audio/open-asr-leaderboard at the revision that adds urgent2024 ("Add +# urgent2024 (#13)", 2026-10-07). The other configs read here are byte-identical +# to those of the earlier b6bdcd0beb34f8975dc659796176d88f43aff502 (that diff +# touches only README.md and adds urgent2024/ and urgent2024_clean/), so the +# fingerprints recorded at that revision hold. +OPEN_ASR_REPO = "hf-audio/open-asr-leaderboard" +OPEN_ASR_REVISION = "de79e57adf9983993bcc72586d7313460825f5eb" +# Voice Arena Monsoon: the leaderboard's VoiceArena/Monsoon_en_IN_test redirects +# to this repo. +MONSOON_REPO = "VoiceArena/MonsoonASR-Open-ASR-leaderboard-en-IN" +MONSOON_REVISION = "bc1da7b42ef6e2853123c97bf6d22067e4802d11" +# Earnings22-Cleaned-AA-chunked: the chunks, and their calls' transcripts from the +# parent set (normalizer/data_utils.CHUNKED_DATASETS, load_chunked_data). +EARNINGS22_CHUNKS_REPO = "ArtificialAnalysis/Earnings22-Cleaned-AA-chunked" +EARNINGS22_CHUNKS_REVISION = "27f7d0056d593231974a752ce03c1e965da69b3f" +EARNINGS22_CHUNKS_FILE = "earnings22_cleaned_aa_chunked_v1.jsonl" +EARNINGS22_PARENTS_REPO = "ArtificialAnalysis/Earnings22-Cleaned-AA" +EARNINGS22_PARENTS_REVISION = "5999936ee3d8d3cde160e3c65a8cc6ff5b7c5276" +EARNINGS22_PARENTS_FILE = "earnings22_cleaned_aa_v1.jsonl" +# google/fleurs main since 2026-05-15 (its parquet-data// files). +FLEURS_REPO = "google/fleurs" +FLEURS_REVISION = "70bb2e84b976b7e960aa89f1c648e09c59f894dd" + + +@dataclasses.dataclass(frozen=True) +class AsrDataset: + eval_id: EvalId + family: str # "open_asr_leaderboard" or "fleurs" + repo: str + revision: str + subset: str + split: str + reference_column: str + language: str # ISO 639-3 + normalizer: NormalizerRecipe + aligner: Aligner + open_asr_exclusions: bool + # Clips in the full build; asserted unless --limit. None = not yet known. + expected_samples: int | None + license: str + license_notes: str + # Fingerprint of the full build's source rows (_asr_common.source_sha256): + # the audio-inclusive `content_sha256` with each resampled clip counted as + # the source shipped it, so any machine reproduces it. Asserted by a full + # build and checked at load for the canonical DATASET_VERSION. "" = not yet + # filled (needs a full build). + expected_source_sha256: str = "" + + @property + def scoring(self) -> dict[str, str | bool]: + return { + "normalizer": self.normalizer.value, + "aligner": self.aligner.value, + "open_asr_exclusions": self.open_asr_exclusions, + } + + +# `source_sha256` of each dataset's full build (its metadata.json), recorded +# from our builds of this catalog's pinned sources. The input is pinned, not +# the output: MonsoonASR's and URGENT24's clips are resampled by the builder, +# and soxr rounds per CPU architecture, so their stored bytes (and +# `content_sha256`) differ by machine while their source does not. For the +# other sets the two fingerprints are equal. A dataset missing here has not +# been built yet; its build is not fingerprint-checked until it is filled. +EXPECTED_SOURCE_SHA256: dict[str, str] = { + "asr_esb_ami_cleaned": "797c85259c7fb0b48d480ee59b7dbb200e4980cec9d3fed12a5c2e86214ec60e", + "asr_esb_gigaspeech_cleaned": "2319583c8dee37975e4e1de4a9027c9c636d3b9f8245f54b9473930fd4671954", + "asr_esb_librispeech_test_other": "cc542f21a71a52f52a9e403435c6b7b97c6bc6dd035085162c3da18db23afb13", + "asr_esb_spgispeech": "59a74aa484125a451c685a07a2bc386cac898ecd65f616fa2d8a606c4ab65bf5", + "asr_esb_urgent2024": "d34af39602eb850899831527a95a52c139023f3e4f7ee1706245ac4ecff18c34", + "asr_esb_monsoon_en_in": "6f1dc75ba4d4987e2c769739e65f022908f9ed52647b0b3de83c5d66c22528cd", + "asr_fleurs_en_us": "12d7987ddab4b70be2224e227906cdcd8e691c60d1c4744fa518d3192b101c84", + "asr_fleurs_es_419": "99ce91c48f7985d36ba99c145b4f140298350a9d41868cc46d2c6b7cdd2e2ee6", + "asr_fleurs_pt_br": "0eaa5667353c164fd4550fa93eba0de570599d8757338fce9aebc99f8731d2ab", + "asr_fleurs_de_de": "a0c50e364388dcc00aee89b03ddb97b0f47fc8da295276ec67b1d17cb61f0f9a", + "asr_fleurs_fr_fr": "3f2b76bd2e42b418ec8ad086d85973b83ebd9e115fc81167f10d0af2c3a1db9d", + "asr_fleurs_it_it": "8130bebe617d1955cc23ed12463aacf39e5ef2b44fcbf39237e30a53c2bb964c", + "asr_esb_earnings22_cleaned_aa_chunked": "50d2ffe0cb3214a136f18b17ab27c884e9e147f94b2cf48008d42ac45cf78a4d", + "asr_esb_librispeech_test_clean": "a197151ac15bc4acca9b34b5dfb330d1034e1bfd93d0a022fc68ec9c29fd5508", + "asr_esb_voxpopuli_cleaned_aa": "0da49af1cff86560208d27bffbf948e54549d050600f715f29ab7c16f26fc19f", + "asr_fleurs_nl_nl": "cca0257e92a4ec142b7a57ec2e31e8a4fe0e68715cd91ca550fc3bf7ae94af23", +} + + +# Each set's license as its dataset card (or the leaderboard's About table, +# Space hf-audio/open_asr_leaderboard constants.py) states it. Every set is for +# internal evaluation only here: the built data is never redistributed. +_OPEN_ASR_LICENSES = { + "ami_cleaned": ("CC-BY-4.0", "AMI Meeting Corpus; attribution required. Internal evaluation use only."), + "gigaspeech_cleaned": ( + "Apache-2.0 (gated: SpeechColab terms)", + "GigaSpeech; access is gated behind the SpeechColab terms of use " + "(huggingface.co/datasets/speechcolab/gigaspeech). Internal evaluation use only.", + ), + "librispeech": ( + "CC-BY-4.0", + "LibriSpeech (LibriVox public-domain recordings); attribution required. Internal evaluation use only.", + ), + "spgispeech": ( + "Kensho SPGISpeech user agreement (non-commercial)", + "SPGISpeech may be used for non-commercial research only. Internal evaluation use only.", + ), + "urgent2024": ( + "Mixed; non-commercial", + "URGENT 2024 Challenge speech (Common Voice 11.0 CC0, LibriTTS CC-BY-4.0, VCTK ODC-BY, Creative Commons " + "YouTube audio) mixed with noise and room responses including WHAM! (CC-BY-NC-4.0) and TUT Urban Acoustic " + "Scenes 2018 (non-commercial, research only), which make the noisy set research/non-commercial use only " + "(leaderboard About table; huggingface.co/datasets/urgent-challenge/urgent2024_official). Internal " + "evaluation use only.", + ), + "voxpopuli_cleaned_aa": ( + "Apache-2.0", + "VoxPopuli (European Parliament; CC0) with references cleaned by Artificial Analysis, released under " + "Apache-2.0 (ArtificialAnalysis/VoxPopuli-Cleaned-AA card). Internal evaluation use only.", + ), +} + + +def _open_asr(eval_id: EvalId, subset: str, split: str, expected: int | None) -> AsrDataset: + license_name, notes = _OPEN_ASR_LICENSES[subset] + return AsrDataset( + eval_id=eval_id, + family="open_asr_leaderboard", + repo=OPEN_ASR_REPO, + revision=OPEN_ASR_REVISION, + subset=subset, + split=split, + reference_column="text", + language="eng", + normalizer=NormalizerRecipe.OPEN_ASR_ENGLISH, + aligner=Aligner.KALDIALIGN_COMPOUND, + open_asr_exclusions=True, + expected_samples=expected, + license=license_name, + license_notes=notes, + expected_source_sha256=EXPECTED_SOURCE_SHA256.get(eval_id.value, ""), + ) + + +# The FLEURS locales evaluated: (google/fleurs config, its ISO 639-3 language as +# pycountry names it, clips in its test split at FLEURS_REVISION). English, and +# the languages the Open ASR Leaderboard evaluates on FLEURS (its +# api/run_api_ml.sh `DATASET_CONFIGS`, @ 62a457c5c32d0de4df77e9470308aae7d45b055f: +# de, fr, it, es, pt, nl, hy) without Armenian, in pipette's order. FLEURS ships +# Spanish and Portuguese as es_419 and pt_br only. +FLEURS_LOCALES: tuple[tuple[str, str, int], ...] = ( + ("en_us", "eng", 647), + ("es_419", "spa", 908), + ("pt_br", "por", 919), + ("de_de", "deu", 862), + ("fr_fr", "fra", 676), + ("it_it", "ita", 865), + ("nl_nl", "nld", 364), +) + + +def _fleurs(locale: str, language: str, expected: int) -> AsrDataset: + return AsrDataset( + eval_id=EvalId(f"asr_fleurs_{locale}"), + family="fleurs", + repo=FLEURS_REPO, + revision=FLEURS_REVISION, + subset=locale, + split="test", + # FLEURS's own normalized reference text (google/fleurs `transcription`). + reference_column="transcription", + language=language, + # score_results(multilingual=True, language=): eval_utils._normalize + # and _align_pair with normalize_compound_pairs. + normalizer=NormalizerRecipe.OPEN_ASR_MULTILINGUAL, + aligner=Aligner.KALDIALIGN_COMPOUND_PAIRS, + # Its multilingual runs drop only a reference that is empty or the ignore + # marker before normalization (run_eval_ml.py); no FLEURS reference is. + open_asr_exclusions=False, + expected_samples=expected, + license="CC-BY-4.0", + license_notes="FLEURS (Google); attribution required. Internal evaluation use only.", + expected_source_sha256=EXPECTED_SOURCE_SHA256.get(f"asr_fleurs_{locale}", ""), + ) + + +ASR_DATASETS: dict[EvalId, AsrDataset] = { + d.eval_id: d + for d in ( + # The Open ASR Leaderboard's English public sets, in its column order. + _open_asr(EvalId.ASR_ESB_AMI_CLEANED, "ami_cleaned", "test", 7805), + AsrDataset( + eval_id=EvalId.ASR_ESB_EARNINGS22_CLEANED_AA_CHUNKED, + family="open_asr_leaderboard", + repo=EARNINGS22_CHUNKS_REPO, + revision=EARNINGS22_CHUNKS_REVISION, + subset="default", + split="test", + reference_column="transcript", + language="eng", + normalizer=NormalizerRecipe.OPEN_ASR_ENGLISH, + aligner=Aligner.KALDIALIGN_COMPOUND, + open_asr_exclusions=True, + # 341 chunks of 6 calls, scored per call (eval_utils.merge_chunked_manifest). + expected_samples=341, + license="Apache-2.0", + license_notes=( + "Earnings-22 (Rev.com; CC-BY-SA-4.0) cleaned and chunked by Artificial Analysis, released under " + "Apache-2.0 (ArtificialAnalysis/Earnings22-Cleaned-AA and -chunked cards). Internal evaluation use only." + ), + expected_source_sha256=EXPECTED_SOURCE_SHA256.get(EvalId.ASR_ESB_EARNINGS22_CLEANED_AA_CHUNKED.value, ""), + ), + _open_asr(EvalId.ASR_ESB_GIGASPEECH_CLEANED, "gigaspeech_cleaned", "test", 18768), + _open_asr(EvalId.ASR_ESB_LIBRISPEECH_TEST_CLEAN, "librispeech", "test.clean", 2620), + _open_asr(EvalId.ASR_ESB_LIBRISPEECH_TEST_OTHER, "librispeech", "test.other", 2939), + _open_asr(EvalId.ASR_ESB_SPGISPEECH, "spgispeech", "test", 39341), + # 1,000 clips (the dataset card's split size); fingerprint pinned above. + _open_asr(EvalId.ASR_ESB_URGENT2024, "urgent2024", "test", 1000), + AsrDataset( + eval_id=EvalId.ASR_ESB_MONSOON_EN_IN, + family="open_asr_leaderboard", + repo=MONSOON_REPO, + revision=MONSOON_REVISION, + subset="default", + split="test", + reference_column="text", + language="eng", + normalizer=NormalizerRecipe.OPEN_ASR_ENGLISH, + aligner=Aligner.KALDIALIGN_COMPOUND, + open_asr_exclusions=True, + expected_samples=2102, + license="CC-BY-4.0", + license_notes=( + "Voice Arena Monsoon en-IN public test set (VoiceArena/MonsoonASR-Open-ASR-leaderboard-en-IN card); " + "attribution required. Internal evaluation use only." + ), + expected_source_sha256=EXPECTED_SOURCE_SHA256.get(EvalId.ASR_ESB_MONSOON_EN_IN.value, ""), + ), + _open_asr(EvalId.ASR_ESB_VOXPOPULI_CLEANED_AA, "voxpopuli_cleaned_aa", "test", 628), + *(_fleurs(locale, language, n) for locale, language, n in FLEURS_LOCALES), + ) +} + +if [e for e in ASR_EVAL_ORDER] != list(ASR_DATASETS): + raise RuntimeError("ASR_DATASETS must hold every asr_* EvalId, in EvalId order") + +OPEN_ASR_EVAL_IDS: tuple[EvalId, ...] = tuple(e for e, d in ASR_DATASETS.items() if d.family == "open_asr_leaderboard") +FLEURS_EVAL_IDS: tuple[EvalId, ...] = tuple(e for e, d in ASR_DATASETS.items() if d.family == "fleurs") diff --git a/packages/pipette-scores/pipette_scores/dataset_catalog/asr_prompt_profiles.py b/packages/pipette-scores/pipette_scores/dataset_catalog/asr_prompt_profiles.py new file mode 100644 index 0000000..33e3bbd --- /dev/null +++ b/packages/pipette-scores/pipette_scores/dataset_catalog/asr_prompt_profiles.py @@ -0,0 +1,346 @@ +"""How each model family is asked to transcribe a clip: the ASR prompt profiles. + +An ASR dataset holds the clip and its reference only; a profile turns a clip into +the request the model's own public usage guidance describes (its model card, +cookbook or official evaluation configuration): the turns and their order, the +prompt wording, the chat-template arguments, the model's audio length limit, +and the cleanup of the answer before scoring. ``AsrProfile.reference`` names +each source at a pinned revision. Where a source gives no wording for a +language, the profile extends it, and says so (``pipette's extension``); every +other step that is pipette's own is labelled the same way. + +Every profile receives the same audio: 16 kHz mono, resampled once as the Open +ASR Leaderboard prepares audio for every model (_asr_audio), sent as a WAV that +carries those samples exactly (16-bit PCM where they are 16-bit values, 32-bit +float otherwise). + +Generation settings are not here; the benchmark definition carries them. Nor is +what the server must provide (docs/asr-evals.md, "Server prerequisites"): +thinking off (no profile sends ``enable_thinking``), Voxtral's transcription +chat template, LFM3's turn format, and MiniCPM-o's ``use_tts_template`` switch. + +Profiles: + +- ``gemma4``: no system turn; the user turn is Google's ASR prompt, then the clip + (the card puts audio after the text), cut to 30 s. +- ``qwen25_omni``: system "You are a speech recognition model."; the user turn + is the clip, then the cookbook's prompt; the answer is the text after the last + "\\nassistant\\n", stripped. +- ``voxtral``: the clip alone in transcription mode; the language is named for + the nine languages Mistral evaluates Voxtral's transcription on, and left to + automatic detection otherwise. +- ``minicpm_o``: no system turn; the user turn is OpenBMB's evaluation prompt for + the dataset, written in the clip's language, followed by a newline, then the + clip; the answer is the text between ``<|tts_bos|>`` and ``<|tts_eos|>``, + stripped. +- ``lfm25_audio``: system "Perform ASR." (the card's fixed prompt, in every + language); the clip alone; a trailing ``<|im_end|>`` dropped. +- ``lfm3``: LFM3's native ASR chat format: system "Perform ASR in {language}.", + the clip alone; the answer as decoded. +""" + +from __future__ import annotations + +import dataclasses +from collections.abc import Callable +from typing import Any + +from pipette_scores.types import EvalId + + +@dataclasses.dataclass(frozen=True) +class AsrClip: + """What a profile reads from a dataset row.""" + + eval_id: EvalId + language: str # ISO 639-3 + language_name: str # English name (_asr_common.language_fields) + language_alpha_2: str | None + + +Content = list[dict[str, Any]] + + +def _no_kwargs(_clip: AsrClip) -> dict[str, Any] | None: + return None + + +def _as_sent(text: str) -> str: + return text + + +@dataclasses.dataclass(frozen=True) +class AsrProfile: + name: str + # The public sources this profile follows, at pinned revisions. + reference: str + # (clip, the clip's input_audio part) -> chat messages + messages: Callable[[AsrClip, dict[str, Any]], list[dict[str, Any]]] + # The model's audio length limit, in 16 kHz samples (None = the whole clip). + audio_max_samples: int | None = None + # Per-request chat-template arguments (merged by the client over the + # profile's server-level ones). + chat_template_kwargs: Callable[[AsrClip], dict[str, Any] | None] = _no_kwargs + # The cleanup of the model's answer before scoring. + postprocess: Callable[[str], str] = _as_sent + + +def _user(*content: dict[str, Any]) -> dict[str, Any]: + return {"role": "user", "content": list(content)} + + +def _text(text: str) -> dict[str, Any]: + return {"type": "text", "text": text} + + +# --- Gemma 4 ----------------------------------------------------------------- +# google/gemma-4-E2B-it model card, "Best Practices": §6 "Audio" gives the ASR +# prompt with a {LANGUAGE} placeholder, §4 puts audio after the text, §7 limits +# audio to 30 seconds. The placeholder takes the language's English name. +GEMMA4_ASR_PROMPT = ( + "Transcribe the following speech segment in {source_language} into " + "{source_language} text.\n\n" + "Follow these specific instructions for formatting the answer:\n" + "* Only output the transcription, with no newlines.\n" + "* When transcribing numbers, write the digits, i.e. write 1.7 and not " + "one point seven, and write 3 instead of three." +) +# 30 s at 16 kHz: the card's limit, and the max_length=480_000 (truncation on) +# with which transformers' Gemma4AudioFeatureExtractor cuts longer audio. +GEMMA4_MAX_SAMPLES = 480_000 + + +def _gemma4(clip: AsrClip, audio: dict[str, Any]) -> list[dict[str, Any]]: + return [_user(_text(GEMMA4_ASR_PROMPT.format(source_language=clip.language_name)), audio)] + + +# --- Qwen2.5-Omni ------------------------------------------------------------ +# QwenLM/Qwen2.5-Omni cookbooks/universal_audio_understanding.ipynb: system "You +# are a speech recognition model.", the audio then the prompt "Transcribe the +# English audio into text without any punctuation marks.". The cookbook words +# its other languages' examples slightly differently ("without including any"); +# pipette uses the English wording for every language (pipette's extension). +# The order is the cookbook's messages: the audio part, then the text part, as +# the current Qwen2.5-Omni chat template renders them (the outputs recorded in +# the cookbook's cells, from an earlier template, show the text first). +QWEN25_OMNI_SYSTEM = "You are a speech recognition model." +QWEN25_OMNI_ASR_PROMPT = "Transcribe the {source_language} audio into text without any punctuation marks." + + +def _qwen25_omni(clip: AsrClip, audio: dict[str, Any]) -> list[dict[str, Any]]: + return [ + {"role": "system", "content": QWEN25_OMNI_SYSTEM}, + _user(audio, _text(QWEN25_OMNI_ASR_PROMPT.format(source_language=clip.language_name))), + ] + + +def _qwen25_omni_answer(text: str) -> str: + # The cookbook decodes the whole sequence (batch_decode of generate's + # output), so its response is the chat transcript ending + # "assistant\n"; the answer is the text after the last + # "\nassistant\n", stripped. A server that returns the generated text only + # is unaffected. + return text.split("\nassistant\n")[-1].strip() + + +# --- Voxtral ----------------------------------------------------------------- +# mistralai/Voxtral-Mini-3B-2507 model card, "Transcription": the dedicated +# transcription mode (mistral-common TranscriptionRequest; transformers +# apply_transcription_request), with an optional ISO 639-1 `language` that +# "will improve language adherence" (mistral-common +# protocol/transcription/request.py); without one the model detects the +# language. The hint is given for the languages Mistral reports Voxtral's +# transcription on: FLEURS in the Voxtral technical report (arXiv 2507.13264, +# App. A.1, Table 4: Arabic, Dutch, English, French, German, Hindi, Italian, +# Portuguese, Spanish). +VOXTRAL_EXPLICIT_LANGUAGES = frozenset({"ar", "de", "en", "es", "fr", "hi", "it", "nl", "pt"}) + + +def _voxtral(_clip: AsrClip, audio: dict[str, Any]) -> list[dict[str, Any]]: + return [_user(audio)] + + +def _voxtral_kwargs(clip: AsrClip) -> dict[str, Any]: + alpha_2 = clip.language_alpha_2 + return { + "transcription": True, + "language": alpha_2 if alpha_2 in VOXTRAL_EXPLICIT_LANGUAGES else "", + } + + +# --- MiniCPM-o 4.5 ----------------------------------------------------------- +# OpenBMB's evaluation prompts: OmniEvalKit o_e_Kit/configs/generation_configs.json +# `asr_configs`, each "\n{audio}" (the prompt, a newline, then the clip). +MINICPM_GENERAL = "Please listen to the audio carefully and transcribe it with high precision." +MINICPM_LOWERCASE = ( + "Please listen to the audio carefully and transcribe it with high precision. " + "Please output in lower case without punctuation." +) +MINICPM_SPGISPEECH = "Please listen to the financial domain audio and transcribe it accurately." +# The FLEURS languages other than English. MiniCPM-o picks the language of its +# answer from the language of the instruction: the openbmb/MiniCPM-o-4_5 model +# card (@ d59117d19ddb8a28a62f1b5f4e3b34ac48d510a4, audio understanding) gives +# one English prompt for English ASR and for Chinese-to-English translation +# alike, and the MiniCPM-o 2.6 code (openbmb/MiniCPM-o-2_6 modeling_minicpmo.py +# @ 06849bfd36da94da1f5a88fa17e8ba63f08a4c4d, `get_sys_prompt`) says the model +# can "select the response language based on the question language". So an +# English instruction pulls the answer towards English. OpenBMB writes the +# instruction in the target language: OmniEvalKit's one European-language +# config, `commonvoice_fr`, is "Veuillez écouter attentivement l'audio et le +# transcrire avec précision.". Each prompt here is that instruction in the +# clip's language, then a clause asking for a verbatim transcript in that +# language and no translation. The French first sentence is OmniEvalKit's, +# verbatim; the German, Spanish, Portuguese, Italian and Dutch translations and +# every "do not translate" clause are pipette's extension. The scheme was chosen +# among six candidate prompts on the FLEURS validation split, by a rule fixed +# before any was run; the test split was not used (docs/asr-evals.md, "How +# MiniCPM-o's FLEURS prompts were chosen"). The non-French strings await review +# by native speakers. +MINICPM_FLEURS: dict[EvalId, str] = { + EvalId.ASR_FLEURS_ES_419: ( + "Por favor, escuche atentamente el audio y transcríbalo con gran precisión. " + "Transcríbalo palabra por palabra en el idioma original (español); no lo traduzca." + ), + EvalId.ASR_FLEURS_PT_BR: ( + "Por favor, ouça o áudio com atenção e transcreva-o com alta precisão. " + "Transcreva-o palavra por palavra no idioma original (português); não o traduza." + ), + EvalId.ASR_FLEURS_DE_DE: ( + "Bitte hören Sie sich die Audioaufnahme aufmerksam an und transkribieren Sie sie mit hoher Präzision. " + "Transkribieren Sie sie wortgetreu in der Originalsprache (Deutsch); übersetzen Sie sie nicht." + ), + EvalId.ASR_FLEURS_FR_FR: ( + "Veuillez écouter attentivement l'audio et le transcrire avec précision. " + "Transcrivez-le mot pour mot dans la langue d'origine (français) ; ne le traduisez pas." + ), + EvalId.ASR_FLEURS_IT_IT: ( + "Per favore, ascolta attentamente l'audio e trascrivilo con alta precisione. " + "Trascrivilo parola per parola nella lingua originale (italiano); non tradurlo." + ), + EvalId.ASR_FLEURS_NL_NL: ( + "Luister aandachtig naar de audio en transcribeer deze met hoge precisie. " + "Transcribeer het woordelijk in de oorspronkelijke taal (Nederlands); vertaal het niet." + ), +} +# OmniEvalKit's english configs: LibriSpeech, GigaSpeech and VoxPopuli take the +# lowercase prompt, SPGISpeech the financial one, the others (FLEURS English +# included: its `fleurs_en`, and its default) the general one. +_MINICPM_LOWERCASE_SETS = frozenset( + { + EvalId.ASR_ESB_GIGASPEECH_CLEANED, + EvalId.ASR_ESB_VOXPOPULI_CLEANED_AA, + EvalId.ASR_ESB_LIBRISPEECH_TEST_CLEAN, + EvalId.ASR_ESB_LIBRISPEECH_TEST_OTHER, + } +) + + +def minicpm_o_prompt(clip: AsrClip) -> str: + if clip.eval_id == EvalId.ASR_ESB_SPGISPEECH: + return MINICPM_SPGISPEECH + if clip.eval_id in _MINICPM_LOWERCASE_SETS: + return MINICPM_LOWERCASE + if clip.language == "eng": + return MINICPM_GENERAL + try: + return MINICPM_FLEURS[clip.eval_id] + except KeyError: + raise ValueError(f"minicpm_o has no {clip.language_name} prompt for {clip.eval_id.value}") from None + + +def _minicpm_o(clip: AsrClip, audio: dict[str, Any]) -> list[dict[str, Any]]: + # A chat-completions server concatenates a turn's content parts, so the + # newline OmniEvalKit puts between the prompt and {audio} rides on the text. + return [_user(_text(minicpm_o_prompt(clip) + "\n"), audio)] + + +def _minicpm_o_answer(text: str) -> str: + # openbmb/MiniCPM-o-4_5 modeling_minicpmo.py: the spoken answer is the text + # after the last <|tts_bos|> and before <|tts_eos|> (`_generate_mel_spec`), + # and chat() cuts its answer at <|tts_eos|>. The TTS chat template + # (use_tts_template) opens the assistant turn with <|tts_bos|>, which a + # server may return with the answer; absent, the split is a no-op. + return text.split("<|tts_bos|>")[-1].split("<|tts_eos|>")[0].strip() + + +# --- LFM2.5-Audio and LFM3 --------------------------------------------------- +# LiquidAI/LFM2.5-Audio-1.5B: the card points to the liquid-audio package for +# ASR, whose README ("ASR") uses sequential generation with "the fixed system +# prompt `Perform ASR.`" and the audio as the user turn. The card lists English +# as the supported language and gives no other prompt, so the profile sends +# that fixed prompt whatever the clip's language (a FLEURS cell in another +# language runs, and is marked as outside the card's declared languages). +LFM25_AUDIO_ASR_SYSTEM = "Perform ASR." +# LFM3's native ASR chat format (no public card yet). +LFM_ASR_SYSTEM = "Perform ASR in {source_language}." + + +def _lfm25_audio(_clip: AsrClip, audio: dict[str, Any]) -> list[dict[str, Any]]: + return [{"role": "system", "content": LFM25_AUDIO_ASR_SYSTEM}, _user(audio)] + + +def _lfm3(clip: AsrClip, audio: dict[str, Any]) -> list[dict[str, Any]]: + return [{"role": "system", "content": LFM_ASR_SYSTEM.format(source_language=clip.language_name)}, _user(audio)] + + +def _lfm25_audio_answer(text: str) -> str: + # The end-of-turn token, should a server return it with the text. + return text.removesuffix("<|im_end|>") + + +PROFILES: dict[str, AsrProfile] = { + p.name: p + for p in ( + AsrProfile( + "gemma4", + "google/gemma-4-E2B-it model card @ 3e22461f65e89153144f8adb70e3b8c2cc9845a7, Best Practices " + "§4 (audio after text), §6 (ASR prompt), §7 (30 s audio)", + _gemma4, + audio_max_samples=GEMMA4_MAX_SAMPLES, + ), + AsrProfile( + "qwen25_omni", + "QwenLM/Qwen2.5-Omni @ d8a31ca56c0456b6edfcbcbf4bdbb6ae2200ef42 " + "cookbooks/universal_audio_understanding.ipynb (speech recognition)", + _qwen25_omni, + postprocess=_qwen25_omni_answer, + ), + AsrProfile( + "voxtral", + "mistralai/Voxtral-Mini-3B-2507 model card @ 3060fe34b35ba5d44202ce9ff3c097642914f8f3 " + "(Transcription); mistral-common @ b54161726d8cde801c976a2bad48c23777551927 " + "protocol/transcription/request.py; Voxtral technical report arXiv 2507.13264 App. A.1 Table 4", + _voxtral, + chat_template_kwargs=_voxtral_kwargs, + ), + AsrProfile( + "minicpm_o", + "OpenBMB/OmniEvalKit @ 1adac0577258d539efc03f16d8a0f4b3f7df6c19 " + "o_e_Kit/configs/generation_configs.json (asr_configs); openbmb/MiniCPM-o-4_5 " + "@ d59117d19ddb8a28a62f1b5f4e3b34ac48d510a4 model card (audio understanding) and " + "modeling_minicpmo.py (chat, _generate_mel_spec); openbmb/MiniCPM-o-2_6 " + "@ 06849bfd36da94da1f5a88fa17e8ba63f08a4c4d modeling_minicpmo.py (get_sys_prompt)", + _minicpm_o, + postprocess=_minicpm_o_answer, + ), + AsrProfile( + "lfm25_audio", + "LiquidAI/LFM2.5-Audio-1.5B model card @ c362a0625dfe45aa588dce5f0ada28a7e5707628; " + "Liquid4All/liquid-audio @ 19e65845923a7f136442c95137884ec61eb386aa README (ASR)", + _lfm25_audio, + postprocess=_lfm25_audio_answer, + ), + AsrProfile( + "lfm3", + "LFM3's native ASR chat format", + _lfm3, + ), + ) +} + + +def profile(name: str) -> AsrProfile: + try: + return PROFILES[name] + except KeyError: + raise ValueError(f"Unknown ASR prompt profile: {name!r} (known: {', '.join(PROFILES)})") from None diff --git a/packages/pipette-scores/pipette_scores/dataset_catalog/math_500.py b/packages/pipette-scores/pipette_scores/dataset_catalog/math_500.py index f9aa8b3..5695494 100644 --- a/packages/pipette-scores/pipette_scores/dataset_catalog/math_500.py +++ b/packages/pipette-scores/pipette_scores/dataset_catalog/math_500.py @@ -3,7 +3,7 @@ import pathlib from pipette_scores.repeats import expand, read_repeats -from pipette_scores.types import ChatMessage, EvalSample, Math500Sample +from pipette_scores.types import ChatMessage, EvalSample, Math500Grader, Math500Sample from pipette_scores.hashing import short_hash from pipette_scores.parquet_io import read_parquet @@ -44,12 +44,32 @@ def _read_rows(dataset_dir: pathlib.Path) -> list[dict]: return rows +def read_grader(dataset_dir: pathlib.Path) -> Math500Grader: + """Read ``metadata.grader``: the grader this dataset version is scored with. + + A version without the key (2026.06.1) is scored with ``math_generic``, so + its results stay reproducible. An unknown value fails the load rather than + falling back to a grader the dataset was not built for. + """ + meta_path = dataset_dir / "metadata.json" + if not meta_path.exists(): + return Math500Grader.MATH_GENERIC + grader = json.loads(meta_path.read_text()).get("grader", Math500Grader.MATH_GENERIC.value) + try: + return Math500Grader(grader) + except ValueError: + known = ", ".join(g.value for g in Math500Grader) + raise ValueError(f"metadata.grader must be one of {known}, got {grader!r} in {meta_path}") from None + + def load_eval_samples(dataset_dir: pathlib.Path) -> list[Math500Sample]: + grader = read_grader(dataset_dir) samples = [ Math500Sample( id=short_hash(row["prompt"]), prompt=row["prompt"], answer=str(row["answer"]), + grader=grader, ) for row in _read_rows(dataset_dir) ] diff --git a/packages/pipette-scores/pipette_scores/dataset_catalog/prompt_profiles.py b/packages/pipette-scores/pipette_scores/dataset_catalog/prompt_profiles.py new file mode 100644 index 0000000..5ae1fda --- /dev/null +++ b/packages/pipette-scores/pipette_scores/dataset_catalog/prompt_profiles.py @@ -0,0 +1,337 @@ +"""Model-specific prompts, copied from VLMEvalKit or a model card. + +By default every model gets the same prompt for a benchmark: the dataset's own +VLMEvalKit prompt, stored with the dataset. A prompt profile replaces that text +with the prompt one model family is evaluated with — the one VLMEvalKit's +wrapper builds for it, or the one its model card asks for — so a run can +reproduce the numbers that family publishes. Only the text changes: the images +and where they go, the dataset and the scoring stay the same. + +A profile may also set a system message (SYSTEM_PROMPTS). That is how a family +whose thinking is switched on by a prompt rather than by its chat template is +run with thinking: InternVL3.5 thinks only under its R1 system prompt, and +Nemotron-Nano-v2-VL thinks when a system message says ``/think``. + +A profile may also end the conversation with an assistant message (PREFILLS), +which the server continues instead of starting a new turn. That is how a family +whose template has no thinking switch at all is forced to think: Phi-4-reasoning-vision +thinks when its reply is opened with ````. + +Every string here is copied either from VLMEvalKit at commit 8eca8e5, from the +file and lines named in each profile (paths are under vlmeval/vlm/), or from +the model card a profile names. To add one, write a function from (benchmark, +the dataset row's raw document, the default prompt) to the prompt text, and add +it to PROFILES. +""" + +import math +import re +import string +from collections.abc import Callable + +from pipette_scores.types import EvalId + +Profile = Callable[[EvalId, dict, str], str] + +MULTIPLE_CHOICE = frozenset({EvalId.MMSTAR, EvalId.REALWORLDQA, EvalId.MMBENCH_EN_DEV, EvalId.MMMU_VAL}) +# The benchmarks every profile covers: the VLM evals. +BENCHMARKS = MULTIPLE_CHOICE | {EvalId.MME, EvalId.POPE, EvalId.OCRBENCH} + +SHORT_ANSWER = "Answer the question using a single word or phrase." + + +def _present(value) -> bool: + """Upstream skips cells pandas reads as missing; the rows store them as null or NaN.""" + return value is not None and value != "" and not (isinstance(value, float) and math.isnan(value)) + + +def _hint(doc: dict) -> str | None: + return doc["hint"] if _present(doc.get("hint")) else None + + +def _options(doc: dict) -> dict[str, str]: + return {letter: doc[letter] for letter in string.ascii_uppercase if _present(doc.get(letter))} + + +def _letter_choice(doc: dict) -> str: + """A question answered with an option letter: internvl/utils.py:161-182, + which granite_vision/granite_vision.py:113-139 copies.""" + question = doc["question"] + if hint := _hint(doc): + question = f"{hint}\n{question}" + options = _options(doc) + for letter, option in options.items(): + question += f"\n{letter}. {option}" + chinese = re.search(r"[\u4e00-\u9fff]", question) is not None + if options: + return question + ( + "\n请直接回答选项字母。" + if chinese + else "\nAnswer with the option's letter from the given choices directly." + ) + return question + ("\n请直接回答问题。" if chinese else "\nAnswer the question directly.") + + +def internvl(benchmark: EvalId, doc: dict, default: str) -> str: + """InternVL3.5: internvl/internvl_chat.py:245-270.""" + if benchmark in MULTIPLE_CHOICE: + return _letter_choice(doc) + if benchmark == EvalId.MME: + return f"{doc['question']} {SHORT_ANSWER}" + if benchmark == EvalId.OCRBENCH: + return f"{doc['question']}\n{SHORT_ANSWER}" + return doc["question"] # POPE + + +def granite_vision(benchmark: EvalId, doc: dict, default: str) -> str: + """IBM's Granite Vision wrapper, granite_vision/granite_vision.py:82-139. + Written for Granite Vision 3.x; the closest upstream recipe for 4.1.""" + if benchmark in MULTIPLE_CHOICE: + return _letter_choice(doc) + if benchmark == EvalId.OCRBENCH: + return doc["question"] + "\nReply with only one word or a short phrase or a full address." + return default # MME, POPE + + +# minicpm_v.py:820-830 +MINICPM_LETTER = "\nAnswer with the option's letter from the given choices directly." +MINICPM_NO_OPTIONS = "Carefully read the following question. Answer the question directly." +MINICPM_CHOICE_COT = ( + "Carefully read the following multichoice question, solve it step by step and finally pick the option " + 'associated with the correct answer in the format of "Answer: selected option\n\n' +) +MINICPM_SHORT_COT = ( + "Read the following question carefully, solve it step by step, and then output the final answer in the " + 'format of "Answer: single number or single word or phrase".\n\n' +) +MINICPM_OCR_COT = "Carefully observe the image and answer the OCR-related questions below. \n\n" + + +def minicpm_v(benchmark: EvalId, doc: dict, default: str) -> str: + """MiniCPM-V answering directly: the prompts VLMEvalKit gives it on the + benchmarks it runs without step-by-step reasoning, minicpm_v.py:1110-1149.""" + if benchmark in MULTIPLE_CHOICE: + prompt = f"Hint: {hint}\n" if (hint := _hint(doc)) else "" + prompt += f"Question: {doc['question']}\n" + if options := _options(doc): + return prompt + "Options:\n" + "".join(f"{k}. {v}\n" for k, v in options.items()) + MINICPM_LETTER + # Only MMMU has questions without options; upstream puts this line first there. + return f"{MINICPM_NO_OPTIONS}\n{prompt}" + if benchmark == EvalId.OCRBENCH: + # Upstream sends this instruction as a text item before the image. + return f"{SHORT_ANSWER}\n{doc['question']}" + return doc["question"] # MME, POPE + + +def minicpm_v_cot(benchmark: EvalId, doc: dict, default: str) -> str: + """MiniCPM-V reasoning step by step to an "Answer: ..." line: how VLMEvalKit + prompts MiniCPM-V-4.6-Thinking on every benchmark and MiniCPM-V-4.5 on MMMU, + MMBench, MMStar and OCRBench, minicpm_v.py:1150-1189.""" + if benchmark in MULTIPLE_CHOICE: + prompt = f"Hint: {hint}\n" if (hint := _hint(doc)) else "" + prompt += f"{doc['question']}\n" + if options := _options(doc): + return MINICPM_CHOICE_COT + prompt + "".join(f"{k}. {v}\n" for k, v in options.items()) + return MINICPM_SHORT_COT + prompt + if benchmark == EvalId.OCRBENCH: + return MINICPM_OCR_COT + doc["question"] + return MINICPM_SHORT_COT + doc["question"] # MME, POPE + + +# liquid.py:13-48 +LFM_DIRECT_ANSWER = "\nPlease answer directly with only the final answer, do not give any explanation." +_YES_NO_WORD = "\nAnswer with a single word: Yes or No." +_YES_NO_ONLY = "\nOnly output Yes or No." +LFM_MME_BY_CATEGORY = { + "ocr": "{question}", + "artwork": "{question}" + _YES_NO_WORD, + "celebrity": "This is a face recognition question about the person shown. Decide whether the name in the " + "question matches the image. {question}" + _YES_NO_WORD, + "color": "Check the visible color of the relevant object in the image. {question}" + LFM_DIRECT_ANSWER, + "count": "{question}" + _YES_NO_ONLY, + "existence": "{question}", + "landmark": "{question}" + LFM_DIRECT_ANSWER, + "position": "{question}" + _YES_NO_ONLY, + "posters": "Look carefully at the image. {question}" + _YES_NO_WORD, + "scene": "Look carefully at the image. {question}" + _YES_NO_WORD, + "code_reasoning": "{question}" + _YES_NO_WORD, + "commonsense_reasoning": "Use the image and common sense. {question}" + _YES_NO_WORD, + "numerical_calculation": "Calculate carefully from the visible information. {question}" + _YES_NO_WORD, + "text_translation": "{question}" + LFM_DIRECT_ANSWER, +} + + +def lfm_vl(benchmark: EvalId, doc: dict, default: str) -> str: + """LFM2-VL and LFM2.5-VL-1.6B: the dataset's prompt, then LFM's direct-answer + instruction, on every benchmark (liquid.py:224-228, 266-268).""" + return default + LFM_DIRECT_ANSWER + + +def lfm2_5_vl_3b(benchmark: EvalId, doc: dict, default: str) -> str: + """LFM2.5-VL-3B: liquid.py:129-133, 152-157, 224-232.""" + if benchmark == EvalId.MME: + return LFM_MME_BY_CATEGORY[str(doc["category"]).lower()].format(question=doc["question"]) + if benchmark == EvalId.OCRBENCH: + return default + "\nPlease answer concisely with short words or phrases when possible." + if benchmark in (EvalId.MMMU_VAL, EvalId.POPE): + return default + return default + LFM_DIRECT_ANSWER # MMStar, RealWorldQA, MMBench + + +# The Qwen3.5 model cards' "Best Practices" for benchmarking, item 3 +# ("Standardize Output Format"), multiple-choice questions. +QWEN3_5_CHOICE = 'Please show your choice in the `answer` field with only the choice letter, e.g., `"answer": "C"`.' + + +def qwen3_5(benchmark: EvalId, doc: dict, default: str) -> str: + """Qwen3.5: the dataset's prompt, then the output format Qwen's model card + asks for on multiple-choice questions. VLMEvalKit runs Qwen3.5 on its + dataset prompts, so the other benchmarks keep them.""" + if benchmark in MULTIPLE_CHOICE and _options(doc): + return f"{default.rstrip()}\n{QWEN3_5_CHOICE}" + return default + + +# GLM-4.6V's VLMEvalKit wrapper, GLMThinking (cogvlm.py:55-242), has no custom prompt, so the +# dataset prompts stand, but it adds this as a text item of its own after POPE's +# question (cogvlm.py:131-132, 162-163). +GLM_V_YES_NO = "Please answer yes or no." + + +def glm_v(benchmark: EvalId, doc: dict, default: str) -> str: + """GLM-4.6V-Flash: the dataset's prompt, and on POPE VLMEvalKit's extra + yes/no line. The wrapper sends it as a second text item, which GLM's chat + template renders on a line of its own, so one string with a newline before + the line is the same prompt.""" + if benchmark == EvalId.POPE: + return f"{default}\n{GLM_V_YES_NO}" + return default + + +# Kimi-VL's VLMEvalKit wrapper, KimiVL.message_to_promptimg (kimi_vl.py:92-116), on MMMU and MMStar: it +# drops the dataset's closing line and appends this to every text item. +# The benchmarks the wrapper changes the prompt on, and reduces the reply to its boxed answer on. +KIMI_VL_BENCHMARKS = frozenset({EvalId.MMSTAR, EvalId.MMMU_VAL}) +KIMI_VL_DROPPED = "Please select the correct answer from the options above. \n" +KIMI_VL_BOXED_COT = ( + "Answer the preceding question. The last line of your response should follow this format: " + "'Answer: $\\boxed{LETTER}$' (without quotes), where LETTER is one of the options. " + "Think step by step logically, considering all relevant information before answering." +) + + +def kimi_vl(benchmark: EvalId, doc: dict, default: str) -> str: + """Kimi-VL-A3B-Thinking-2506: on MMMU and MMStar the dataset's closing + "Please select..." line is replaced by Kimi's boxed step-by-step instruction, + kimi_vl.py:92-116; the other benchmarks keep the dataset prompts. + + VLMEvalKit appends the instruction to each text item. MMMU's question reaches + it split at its ```` placeholders (split_MMMU), so every piece of + text carries the instruction, an empty one included; the same split is made + here, and the loader puts the images where the placeholders were. MMStar's + prompt is one text item.""" + if benchmark not in KIMI_VL_BENCHMARKS: + return default + default = default.replace(KIMI_VL_DROPPED, "") + if benchmark == EvalId.MMSTAR: + return default + KIMI_VL_BOXED_COT + head, *pieces = default.split("" and the text after it; one not shaped so is left for the + # loader's own fallback, which sends the prompt whole. + if not all(len(piece) >= 2 and piece[0].isdigit() and piece[1] == ">" for piece in pieces): + return default + KIMI_VL_BOXED_COT + return "".join([head + KIMI_VL_BOXED_COT, *(f" str: + """Nemotron-Nano-12B-v2-VL: the dataset's prompt (VLMEvalKit has no wrapper + for the model, so the dataset prompts stand), starting with a newline. The + model's chat template writes ```` and a newline before the question; + the image goes in a content part of its own, so the newline is the first + character of the text part. On MMMU the loader puts the images at their + ```` placeholders, so the newline opens the text before the first image.""" + return "\n" + default + + +def dataset_prompt(benchmark: EvalId, doc: dict, default: str) -> str: + """The dataset's prompts, for a profile that changes only the messages around them. + + Phi-4-reasoning-vision-15B: Microsoft ran the VLMEvalKit benchmarks "without any custom + user-prompting" (technical report, section 6), under the model's own system prompt and chat + template. Thinking is forced by PREFILLS. + + MiMo-VL-7B-RL-2508: what VLMEvalKit's Qwen2VLChat sends it (qwen2_vl/prompt.py:26-31, + use_custom_prompt is False); thinking is on by default, so there is nothing to switch. The + profile adds the system message and, as a REASONING_OPENERS entry, sets ``opens_reasoning``.""" + return default + + +# InternVL3.5's thinking mode: no chat-template switch, only this system +# prompt (internvl/internvl_chat.py:25-38, used by the -Thinking entries). +R1_SYSTEM_PROMPT = ( + "You are an AI assistant that rigorously follows this response protocol:\n\n" + "1. First, conduct a detailed analysis of the question. Consider different angles, potential solutions, and " + "reason through the problem step-by-step. Enclose this entire thinking process within and " + "tags.\n\n" + "2. After the thinking section, provide a clear, concise, and direct answer to the user's question. Separate " + "the answer from the think section with a newline.\n\n" + "Ensure that the thinking process is thorough but remains focused on the query. The final answer should be " + "standalone and not reference the thinking section." +) + +# Nemotron-Nano-12B-v2-VL's thinking switch: a ``/think`` flag in a system or user message, which the +# chat template strips from the text (the card's examples send ``/no_think`` as a system message, and +# the enable_thinking kwarg is not read). +NEMOTRON_THINK_FLAG = "/think" + +# MiMo-VL's system prompt as VLMEvalKit's lmdeploy path gives it (the qwen2d5-vl template's +# default, lmdeploy v0.9.2 model.py:1076-1080) and as Xiaomi's own eval harness does (lmms-eval +# mimo_vl_eval branch, models/mivllm.py:117-123). Without a system message the model's chat +# template sends "You are MiMo, an AI assistant developed by Xiaomi.". +MIMO_VL_SYSTEM_PROMPT = "You are a helpful assistant." + +# Phi-4-reasoning-vision's forced-thinking switch, model card section 2.4: end the generation prompt +# with ```` (```` forces the direct answer). +PHI4_RV_THINK_PREFILL = "" + +PROFILES: dict[str, Profile] = { + "internvl": internvl, + "internvl_r1": internvl, + "granite_vision": granite_vision, + "minicpm_v": minicpm_v, + "minicpm_v_cot": minicpm_v_cot, + "lfm_vl": lfm_vl, + "lfm2_5_vl_3b": lfm2_5_vl_3b, + "qwen3_5": qwen3_5, + "glm_v": glm_v, + "kimi_vl": kimi_vl, + "nemotron_think": nemotron_think, + "phi4_rv_think": dataset_prompt, + "mimo_vl": dataset_prompt, +} + + +# A system message sent before the user turn, for the profiles that have one. +SYSTEM_PROMPTS: dict[str, str] = { + "internvl_r1": R1_SYSTEM_PROMPT, + "nemotron_think": NEMOTRON_THINK_FLAG, + "mimo_vl": MIMO_VL_SYSTEM_PROMPT, +} + +# An assistant message that ends the conversation, for the profiles that have one: the server +# continues it, so the reply is what follows it and does not repeat it. This needs a runtime that +# continues a final assistant message (llama-server does by default); one that ends it as a +# finished turn sends the model no opener. +PREFILLS: dict[str, str] = {"phi4_rv_think": PHI4_RV_THINK_PREFILL} + +# The profiles whose model reasons from the first token of its reply, so a reply cut before the +# close shows no tag; the VLM scorers' strict view counts such a reply as thinking. internvl_r1 is +# not here: its R1 system prompt has the model write itself, in the reply. +REASONING_OPENERS = frozenset({"nemotron_think", "phi4_rv_think", "kimi_vl", "mimo_vl"}) + + +def profile(name: str) -> Profile: + try: + return PROFILES[name] + except KeyError: + raise ValueError(f"unknown prompt profile {name!r}; known: {', '.join(PROFILES)}") from None diff --git a/packages/pipette-scores/pipette_scores/hashing.py b/packages/pipette-scores/pipette_scores/hashing.py index fa134a8..d973251 100644 --- a/packages/pipette-scores/pipette_scores/hashing.py +++ b/packages/pipette-scores/pipette_scores/hashing.py @@ -1,7 +1,44 @@ """Stable content hashing for sample identity.""" +from __future__ import annotations + import hashlib +import json +from collections.abc import Iterable + +from pydantic import TypeAdapter + +from pipette_scores.types import EvalSample + +# A sample as `/samples` writes it: through the declared `EvalSample`, as the +# response model does, so a subclass's own fields are dropped here as there. +_AS_SERVED = TypeAdapter(EvalSample) def short_hash(text: str, length: int = 12) -> str: return hashlib.sha256(text.encode()).hexdigest()[:length] + + +def samples_digest(samples: Iterable[EvalSample]) -> str: + """``sha256:`` of prompt samples as ``/samples`` serves them. + + Each sample is hashed as its JSON with keys sorted, no whitespace and + non-ASCII written as UTF-8, followed by a newline, in served order. It + changes when anything served changes (an id, a prompt's wording, an image, + the order) and nothing else, so a pin taken from it holds across restarts. + Hashing one sample at a time keeps memory to one sample's JSON. + """ + h = hashlib.sha256() + for sample in samples: + h.update(canonical_sample(sample)) + h.update(b"\n") + return f"sha256:{h.hexdigest()}" + + +def canonical_sample(sample: EvalSample) -> bytes: + """One sample's canonical JSON, the bytes `samples_digest` hashes: keys + sorted, no whitespace, non-ASCII as UTF-8. It is also valid JSON for the + sample, so a streamed ``/samples`` body can be written from it.""" + return json.dumps( + _AS_SERVED.dump_python(sample, mode="json"), sort_keys=True, separators=(",", ":"), ensure_ascii=False + ).encode() diff --git a/packages/pipette-scores/pipette_scores/packages.py b/packages/pipette-scores/pipette_scores/packages.py new file mode 100644 index 0000000..8263d4a --- /dev/null +++ b/packages/pipette-scores/pipette_scores/packages.py @@ -0,0 +1,411 @@ +"""The eval packages: each prompt set ``/samples`` serves for a VLM or ASR eval, +packed into the tar mgmt hands a device, and the write-once store mgmt reads +them from (`pipette-scores packages`, docs/packages.md). + +A package is byte for byte the tar mgmt's dataset-cache build packed from +``/samples`` (mgmt ``dataset_cache/format.rs`` ``write_corpus``, plan-types +``vlm_eval_corpus.rs``), so a catalog pin names the same bytes whichever built +them: an uncompressed ustar of ``manifest.json``, every image or clip moved out +of its sample into ``media/.`` (stored once however many samples +share it), and ``samples.jsonl``, every header field fixed. + +The store, ``s3:///`` or a local directory, is written once: + +- ``tars/.tar`` +- ``index///.json``: every profile's + samples digest, sample count and package, written once all are stored. +""" + +from __future__ import annotations + +import base64 +import contextlib +import functools +import hashlib +import json +import logging +import os +import pathlib +import re +import tempfile +from collections.abc import Iterable, Iterator +from typing import Any + +from pipette_scores import dataset_catalog +from pipette_scores.dataset_catalog import _asr_common, asr_prompt_profiles, prompt_profiles +from pipette_scores.hashing import _AS_SERVED, canonical_sample +from pipette_scores.types import ASR_EVAL_IDS, EvalId, EvalSample + +logger = logging.getLogger(__name__) + +VLM_FORMAT = "vlm-eval-corpus/1" +ASR_FORMAT = "asr-eval-corpus/1" +ADAPTER = "evals-server@1" + +_BLOCK = 512 +_PART_BYTES = 64 << 20 +# An image's file extension is its subtype (jpg for jpeg), as mgmt's `extension` takes it. +_SUBTYPE = re.compile(r"[a-z0-9]{1,16}") + + +def profiles(eval_id: str) -> list[str | None]: + """The prompt profiles ``/samples`` takes for ``eval_id`` (api/app.py + `_validate_prompt_profile`), None for a VLM set's own prompts. A text eval + has none, and so no package.""" + if dataset_catalog.is_vlm_eval(eval_id): + return [None, *(prompt_profiles.PROFILES if eval_id in prompt_profiles.BENCHMARKS else ())] + if eval_id in ASR_EVAL_IDS: + return list(asr_prompt_profiles.PROFILES) + return [] + + +def index_key(scores_version: str, eval_id: str, dataset_version: str) -> str: + return f"index/{scores_version}/{eval_id}/{dataset_version}.json" + + +def _compact(value: Any) -> bytes: + return json.dumps(value, ensure_ascii=False, separators=(",", ":")).encode() + + +def _decoded(sample_id: str, payload: str) -> bytes: + """``payload``'s bytes, when it is their canonical base64: only then does a + client's encoding of them rebuild the string the samples digest hashed.""" + with contextlib.suppress(ValueError): + data = base64.b64decode(payload, validate=True) + if base64.b64encode(data).decode("ascii") == payload: + return data + raise ValueError(f"sample {sample_id}: a media payload is not canonical base64, so it would not rebuild") + + +def _header(path: str, size: int) -> bytes: + """mgmt's ``format.rs`` ``header``: name, size and checksum, the rest fixed.""" + if not (path.isascii() and 1 <= len(path) <= 100) or size >= 8**11: + raise ValueError(f"{path!r} ({size} bytes) does not fit a ustar header") + h = bytearray(_BLOCK) + h[: len(path)] = path.encode("ascii") + h[100:108] = b"0000644\0" + h[108:116] = b"0000000\0" + h[116:124] = b"0000000\0" + h[124:136] = b"%011o\0" % size + h[136:148] = b"00000000000\0" + h[148:156] = b" " * 8 + h[156:157] = b"0" + h[257:263] = b"ustar\0" + h[263:265] = b"00" + h[148:156] = b"%06o\0 " % sum(h) + return bytes(h) + + +class Package: + """One profile's package, built as its samples arrive: their lines in + memory, the media they name as files under ``scratch`` (each written once, + shared by the dataset's profiles), and their samples digest.""" + + def __init__(self, eval_id: str, dataset_version: str, prompt_profile: str | None, scratch: pathlib.Path): + self.eval_id, self.dataset_version, self.prompt_profile = eval_id, dataset_version, prompt_profile + self.format = ASR_FORMAT if eval_id in ASR_EVAL_IDS else VLM_FORMAT + self._scratch = scratch + (scratch / "media").mkdir(exist_ok=True) + self._digest = hashlib.sha256() + self._lines: list[bytes] = [] + self._media: set[str] = set() + self._ids: set[str] = set() + + def __str__(self) -> str: + return f"{self.eval_id}/{self.dataset_version} profile={self.prompt_profile}" + + @property + def identity(self) -> str: + """The samples digest's hex, as a catalog pin's ``identity`` holds it.""" + return self._digest.hexdigest() + + def add(self, sample: EvalSample) -> None: + """The next sample as ``/samples`` serves it, hashed into the samples + digest and written as its line.""" + if sample.id in self._ids: + raise ValueError(f"{self}: sample id {sample.id} is served twice") + self._ids.add(sample.id) + self._digest.update(canonical_sample(sample)) + self._digest.update(b"\n") + served = _AS_SERVED.dump_python(sample, mode="json") + line = {"id": served["id"], "messages": [self._message(served["id"], m) for m in served["messages"]]} + if served.get("chat_template_kwargs") is not None: + # mgmt's map is ordered by key, at every depth. + line["chat_template_kwargs"] = json.loads(json.dumps(served["chat_template_kwargs"], sort_keys=True)) + self._lines.append(_compact(line) + b"\n") + + def _message(self, sample_id: str, message: dict[str, Any]) -> dict[str, Any]: + content = message["content"] + if not isinstance(content, str): + content = [self._part(sample_id, part) for part in content] + return {"role": message["role"], "content": content} + + def _part(self, sample_id: str, part: dict[str, Any]) -> dict[str, Any]: + """A served part as the package line holds it (mgmt's ``VlmEvalCorpusPart``): + text as it is, the format's own media part moved out to a file. A part + the line would not rebuild exactly is refused.""" + kind = part.get("type") + media, inner = ("input_audio", {"data", "format"}) if self.format == ASR_FORMAT else ("image_url", {"url"}) + if kind == "text" and part.keys() == {"type", "text"}: + return {"type": "text", "text": part["text"]} + if kind != media or part.keys() != {"type", media} or part[media].keys() != inner: + raise ValueError(f"sample {sample_id}: a {kind!r} part, keys {sorted(part)}, has no place in {self.format}") + if media == "input_audio": + if part[media]["format"] != "wav": + raise ValueError(f"sample {sample_id}: a {part[media]['format']!r} clip; {self.format} holds wav only") + path, size = self._put(sample_id, part[media]["data"], "wav") + return {"type": "audio", "path": path, "format": "wav", "bytes": size} + head, marker, payload = part[media]["url"].partition(";base64,") + subtype = head.removeprefix("data:image/") + if not (marker and head.startswith("data:image/") and _SUBTYPE.fullmatch(subtype)): + raise ValueError(f"sample {sample_id}: image URL {head[:40]!r}... is not data:image/;base64,") + path, size = self._put(sample_id, payload, "jpg" if subtype == "jpeg" else subtype) + return {"type": "image", "path": path, "mime": f"image/{subtype}", "bytes": size} + + def _put(self, sample_id: str, payload: str, ext: str) -> tuple[str, int]: + data = _decoded(sample_id, payload) + path = f"media/{hashlib.sha256(data).hexdigest()}.{ext}" + file = self._scratch / path + if not file.exists(): + file.write_bytes(data) + self._media.add(path) + return path, len(data) + + def _rebuilt(self, line: bytes) -> EvalSample: + """The served sample back from its line and media, as a client rebuilds it.""" + sample = json.loads(line) + + def part(p: dict[str, Any]) -> dict[str, Any]: + if p["type"] == "text": + return p + payload = base64.b64encode((self._scratch / p["path"]).read_bytes()).decode("ascii") + if p["type"] == "image": + return {"type": "image_url", "image_url": {"url": f"data:{p['mime']};base64,{payload}"}} + return {"type": "input_audio", "input_audio": {"data": payload, "format": p["format"]}} + + for message in sample["messages"]: + if not isinstance(message["content"], str): + message["content"] = [part(p) for p in message["content"]] + return EvalSample.model_validate(sample) + + def tar(self) -> Iterator[bytes]: + """The package's bytes as mgmt's ``write_corpus`` writes them: entries + sorted by path, each padded to a block, then two zero blocks (no padding + to a 10240-byte record, so not `tarfile`).""" + upstream = { + "source": "evals_server", + "eval_id": self.eval_id, + "dataset_version": self.dataset_version, + **({} if self.prompt_profile is None else {"prompt_profile": self.prompt_profile}), + "identity": self.identity, + } + manifest = { + "format": self.format, + "adapter": ADAPTER, + "upstream": upstream, + "samples": len(self._lines), + "media": len(self._media), + } + entries: dict[str, bytes | pathlib.Path] = { + "manifest.json": _compact(manifest) + b"\n", + "samples.jsonl": b"".join(self._lines), + **{path: self._scratch / path for path in self._media}, + } + for path in sorted(entries): + body = entries[path] + size = len(body) if isinstance(body, bytes) else body.stat().st_size + yield _header(path, size) + if isinstance(body, bytes): + yield body + else: + with body.open("rb") as file: + yield from iter(functools.partial(file.read, 1 << 20), b"") + yield bytes(-size % _BLOCK) + yield bytes(2 * _BLOCK) + + def save(self, store: Store) -> dict[str, Any]: + """Require the samples rebuilt from their lines and media files to hash + to the samples digest (mgmt's ``rebuilt_digest``), store the tar unless + ``store`` holds it, and return the package's index entry.""" + if not self._lines: + raise ValueError(f"{self}: no samples") + rebuilt = hashlib.sha256() + for line in self._lines: + rebuilt.update(canonical_sample(self._rebuilt(line))) + rebuilt.update(b"\n") + if rebuilt.hexdigest() != self.identity: + raise ValueError(f"{self}: rebuilt from their lines and media, the samples hash to {rebuilt.hexdigest()}") + sha, size = hashlib.sha256(), 0 + for chunk in self.tar(): + sha.update(chunk) + size += len(chunk) + digest = sha.hexdigest() + written = store.create(f"tars/{digest}.tar", self.tar()) + logger.info( + "eval=%s dataset=%s profile=%s identity=%s digest=%s bytes=%d %s", + self.eval_id, + self.dataset_version, + self.prompt_profile, + self.identity, + digest, + size, + "uploaded" if written else "already present", + ) + return { + "prompt_profile": self.prompt_profile, + "samples_digest": f"sha256:{self.identity}", + "sample_count": len(self._lines), + "package": {"format": self.format, "digest": digest, "bytes": size, "media": len(self._media)}, + } + + +def build(eval_id: str, dataset_version: str, root_dir: pathlib.Path | str, store: Store, scores_version: str) -> None: + """Pack every profile of one dataset under ``root_dir`` (a + ``PIPETTE_SCORES_DATA_DIR``), store each tar ``store`` lacks, then the + dataset's index for ``scores_version``. A VLM set is loaded one profile at + a time; an ASR set is read in one pass over its audio for every profile.""" + with tempfile.TemporaryDirectory(prefix="packages-") as tmp: + scratch = pathlib.Path(tmp) + if eval_id in ASR_EVAL_IDS: + packages = {name: Package(eval_id, dataset_version, name, scratch) for name in profiles(eval_id)} + dataset_dir = dataset_catalog._dataset_dir(eval_id, dataset_version, root_dir) + for requests in _asr_common.iter_profile_requests(EvalId(eval_id), dataset_dir): + for name, request in requests.items(): + packages[name].add(request) + entries = [package.save(store) for package in packages.values()] + else: + entries = [] + for profile in profiles(eval_id): + package = Package(eval_id, dataset_version, profile, scratch) + for sample in dataset_catalog.load_prompt_samples.__wrapped__( + eval_id, dataset_version, root_dir, prompt_profile=profile + ): + package.add(sample) + entries.append(package.save(store)) + index = {"eval_id": eval_id, "dataset_version": dataset_version, "scores_version": scores_version} + key = index_key(scores_version, eval_id, dataset_version) + written = store.create(key, (json.dumps({**index, "profiles": entries}, indent=2) + "\n").encode()) + logger.info("eval=%s dataset=%s %s %s", eval_id, dataset_version, key, "uploaded" if written else "already present") + + +class S3Store: + """``s3:///``; a key is the object key under the prefix.""" + + def __init__(self, url: str): + import boto3 + + bucket, _, prefix = url.removeprefix("s3://").partition("/") + if not bucket or not prefix.strip("/"): + raise ValueError(f"{url} is not s3:///") + self._client = boto3.client("s3") + self._bucket, self._prefix = bucket, prefix.strip("/") + + def _key(self, key: str) -> str: + return f"{self._prefix}/{key}" + + def list(self, prefix: str) -> list[str]: + pages = self._client.get_paginator("list_objects_v2").paginate(Bucket=self._bucket, Prefix=self._key(prefix)) + return [o["Key"].removeprefix(self._key("")) for page in pages for o in page.get("Contents", [])] + + def read(self, key: str) -> bytes | None: + try: + return self._client.get_object(Bucket=self._bucket, Key=self._key(key))["Body"].read() + except self._client.exceptions.NoSuchKey: + return None + + def download(self, key: str, path: pathlib.Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + self._client.download_file(self._bucket, self._key(key), str(path)) + + def _exists(self, key: str) -> bool: + try: + self._client.head_object(Bucket=self._bucket, Key=key) + except self._client.exceptions.ClientError as e: + if e.response["Error"]["Code"] == "404": + return False + raise + return True + + def create(self, key: str, body: bytes | Iterable[bytes]) -> bool: + """Write ``key`` once (``body`` a whole object, or its chunks); False + when it is there already, a concurrent run's write included (412).""" + key = self._key(key) + if self._exists(key): + return False + try: + if isinstance(body, bytes): + self._client.put_object(Bucket=self._bucket, Key=key, Body=body, IfNoneMatch="*") + else: + self._upload(key, body) + except self._client.exceptions.ClientError as e: + if e.response["Error"]["Code"] == "PreconditionFailed": + return False + raise + return True + + def _upload(self, key: str, chunks: Iterable[bytes]) -> None: + """A multipart upload in parts of ``_PART_BYTES``, so a tar of gigabytes + is never held whole, completed only if ``key`` is still absent; given + up on any failure, so no parts are left behind.""" + upload = self._client.create_multipart_upload(Bucket=self._bucket, Key=key, ChecksumAlgorithm="CRC32") + ids = {"Bucket": self._bucket, "Key": key, "UploadId": upload["UploadId"]} + try: + parts = [] + for number, body in enumerate(_parts(chunks), start=1): + part = self._client.upload_part(**ids, PartNumber=number, Body=body, ChecksumAlgorithm="CRC32") + parts.append({"PartNumber": number, "ETag": part["ETag"], "ChecksumCRC32": part["ChecksumCRC32"]}) + self._client.complete_multipart_upload(**ids, MultipartUpload={"Parts": parts}, IfNoneMatch="*") + except BaseException: + self._client.abort_multipart_upload(**ids) + raise + + +def _parts(chunks: Iterable[bytes]) -> Iterator[bytes]: + """``chunks`` regrouped into parts of ``_PART_BYTES``, the last one shorter.""" + buffer = bytearray() + for chunk in chunks: + buffer += chunk + while len(buffer) >= _PART_BYTES: + yield bytes(buffer[:_PART_BYTES]) + del buffer[:_PART_BYTES] + if buffer: + yield bytes(buffer) + + +class LocalStore: + """A local directory in the bucket's place; a key is a path under it.""" + + def __init__(self, root: pathlib.Path): + self._root = root + + def read(self, key: str) -> bytes | None: + try: + return (self._root / key).read_bytes() + except FileNotFoundError: + return None + + def create(self, key: str, body: bytes | Iterable[bytes]) -> bool: + """Write ``key`` once: to a temporary file, then linked into place, + which fails rather than replace a file already there.""" + path = self._root / key + if path.exists(): + return False + path.parent.mkdir(parents=True, exist_ok=True) + fd, partial = tempfile.mkstemp(dir=path.parent, prefix=f".{path.name}.") + try: + with os.fdopen(fd, "wb") as out: + out.writelines([body] if isinstance(body, bytes) else body) + os.link(partial, path) + except FileExistsError: + return False + finally: + os.unlink(partial) + return True + + +Store = S3Store | LocalStore + + +def open_store(url: str) -> Store: + """The store at ``url``: ``s3:///``, or a local directory.""" + return S3Store(url) if url.startswith("s3://") else LocalStore(pathlib.Path(url)) diff --git a/packages/pipette-scores/pipette_scores/scoring/__init__.py b/packages/pipette-scores/pipette_scores/scoring/__init__.py index 877faae..4b6b036 100644 --- a/packages/pipette-scores/pipette_scores/scoring/__init__.py +++ b/packages/pipette-scores/pipette_scores/scoring/__init__.py @@ -3,14 +3,22 @@ import logging from typing import Any -from pipette_scores.types import EvalId, SampleScore +from pipette_scores.types import ASR_EVAL_IDS, EvalId, SampleScore from pipette_scores.memory import rss_mb from pipette_scores.scoring import ( gpqa_diamond, ifbench, ifstruct, math_500, + mmbench_en_dev, + mme, + mmmu_val, + mmstar, + ocrbench, + pope, + realworldqa, ) +from pipette_scores.scoring.asr import evaluate as asr logger = logging.getLogger(__name__) @@ -43,19 +51,50 @@ def __iter__(self): def score( - eval_id: EvalId, completions: list[Any], samples: list[Any], *, label: str | None = None + eval_id: EvalId, + completions: list[Any], + samples: list[Any], + *, + enable_thinking: bool | None = None, + label: str | None = None, + prompt_profile: str | None = None, ) -> tuple[list[SampleScore], dict[str, Any]]: + """``prompt_profile`` reaches the ASR scorer, which cleans each answer as that + profile says, and tells the VLM scorers whether it opens the model's reasoning + (prompt_profiles.REASONING_OPENERS), which sets their status, answer and strict + view; the text evals score the same either way.""" + if EvalId(eval_id) in ASR_EVAL_IDS: + scores, context = asr.score( + completions, samples, enable_thinking=enable_thinking, prompt_profile=prompt_profile + ) + logger.info( + "[%s] Done: WER %s over %d units", + f"{eval_id}/{label}" if label else eval_id, + "n/a" if context["result"] is None else f"{context['result']:.2f}%", + context["units_scored"], + ) + return scores, context + from pipette_scores.dataset_catalog.prompt_profiles import BENCHMARKS, REASONING_OPENERS + scorer = { EvalId.IFBENCH: ifbench.score, EvalId.IFSTRUCT: ifstruct.score, EvalId.GPQA_DIAMOND: gpqa_diamond.score, EvalId.MATH_500: math_500.score, + EvalId.MME: mme.score, + EvalId.MMSTAR: mmstar.score, + EvalId.OCRBENCH: ocrbench.score, + EvalId.MMMU_VAL: mmmu_val.score, + EvalId.REALWORLDQA: realworldqa.score, + EvalId.MMBENCH_EN_DEV: mmbench_en_dev.score, + EvalId.POPE: pope.score, }[EvalId(eval_id)] progress_label = f"{eval_id}/{label}" if label else eval_id total = len(completions) logger.info("[%s] Scoring %d completions...", progress_label, total) logged_completions = _ProgressList(completions, progress_label, log_every=500) - scores, context = scorer(logged_completions, samples) + vlm = {"opens_reasoning": prompt_profile in REASONING_OPENERS} if EvalId(eval_id) in BENCHMARKS else {} + scores, context = scorer(logged_completions, samples, enable_thinking=enable_thinking, **vlm) correct = sum(1 for v in scores if v.is_correct) logger.info( "[%s] Done: %d/%d correct (%.1f%%)", diff --git a/packages/pipette-scores/pipette_scores/scoring/asr/NOTICE.md b/packages/pipette-scores/pipette_scores/scoring/asr/NOTICE.md new file mode 100644 index 0000000..9d040de --- /dev/null +++ b/packages/pipette-scores/pipette_scores/scoring/asr/NOTICE.md @@ -0,0 +1,85 @@ +# ASR normalizer provenance + +ASR scoring follows the Open ASR Leaderboard's two public protocols, English +and multilingual (FLEURS), so its code is vendored unchanged from its public +source, huggingface/open_asr_leaderboard. + +## `open_asr_normalizer/normalizer.py`, `english_abbreviations.py` (English) + +Byte-identical to `huggingface/open_asr_leaderboard` at immutable revision +`67e8bd6acea240819ad67080f6f31e15d4a90da5`, and unchanged at +`62a457c5c32d0de4df77e9470308aae7d45b055f` (the revision the rest of the +leaderboard method is cited at): + +- https://github.com/huggingface/open_asr_leaderboard/blob/67e8bd6acea240819ad67080f6f31e15d4a90da5/normalizer/normalizer.py +- https://github.com/huggingface/open_asr_leaderboard/blob/67e8bd6acea240819ad67080f6f31e15d4a90da5/normalizer/english_abbreviations.py + +`normalizer.py` also holds the `BasicMultilingualTextNormalizer` the +multilingual normalizer below builds on. Licensed under the Apache License 2.0 +(see the source headers). + +## `open_asr_normalizer/multilingual.py` (multilingual, FLEURS) + +Excerpts of `huggingface/open_asr_leaderboard` at +`62a457c5c32d0de4df77e9470308aae7d45b055f`, unchanged: + +- `normalizer/data_utils.py` lines 28-94 (`FILLER_WORDS`, `MultilingualNormalizer`; + the file's sha256 at that revision is + `1d1a8320567d431bb4bb0066cb40c2d94361148d5eb745f30553b8e7f974f513`): + https://github.com/huggingface/open_asr_leaderboard/blob/62a457c5c32d0de4df77e9470308aae7d45b055f/normalizer/data_utils.py#L28-L94 +- `normalizer/eval_utils.py` lines 68-99 (`normalize_compound_pairs`; sha256 + `362756990a8087cf45acc0cde211e52401e2dca9d13a17b1bf3e1aadf270f6fb`): + https://github.com/huggingface/open_asr_leaderboard/blob/62a457c5c32d0de4df77e9470308aae7d45b055f/normalizer/eval_utils.py#L68-L99 + +`data_utils.py` is the leaderboard's data loader as well (it imports `datasets`, +`huggingface_hub` and its own `normalizer` package), so only these parts are +vendored. pipette's are the module header, the imports, and a stand-in for the +leaderboard's `ArmenianTextNormalizer` (`normalizer/armenian_normalizer.py`, +not vendored), which `MultilingualNormalizer` constructs for `hy` and which +refuses to normalize: pipette evaluates no Armenian. `scoring/asr/normalize.py` +selects the normalizer as the leaderboard's `eval_utils._normalize` does (its +English normalizer for `en`, else `data_utils.ml_normalizer`, a +`MultilingualNormalizer(remove_diacritics=False)`, with the language), and +`scoring/asr/align.py` aligns as its `_align_pair` does with +`multilingual=True`. Licensed under the Apache License 2.0 (upstream's +`LICENSE`). + +## Checks + +`tests/test_asr_scoring.py` pins all three vendored files by sha256. +`tests/test_asr_public_reference.py` compares both protocols with the +leaderboard's own scoring code at the pinned revision, and checks that the +excerpts in `multilingual.py` are that revision's source text (see +`docs/asr-evals.md`, "Checking against the public references"). + +## Pinned dependencies + +`packages/pipette-scores/pyproject.toml`: + +- `kaldialign==0.12.0`: the version the leaderboard's scoring environment pins + (`requirements/requirements_jobs.txt`). +- `num2words==0.5.14`: the multilingual normalizer spells numbers out with it. + The leaderboard's requirements name it unpinned + (`requirements/requirements_jobs.txt`), which installs its latest release, + 0.5.14 (2024-12-17); it is pinned so that the FLEURS numbers are + reproducible. +- `regex==2026.4.4`: the normalizers' `\p{…}` classes come from its Unicode + tables. +- `soundfile==0.14.0` (libsndfile 1.2.2) and `soxr==1.1.0`: the ASR audio is + decoded with them, and a clip not at 16 kHz mono resampled with them when its + dataset is built (`dataset_catalog/_asr_audio.py`), as + `datasets.Audio(sampling_rate=16000)` does through librosa's default + `soxr_hq`. + +## Licenses + +All sources are listed in the repository's `NOTICE`, with their license texts in +`THIRD_PARTY_LICENSES.md` (both ship in the wheel): Apache-2.0 for +open_asr_leaderboard, whose English normalizer is, per its header, mostly +copied from openai/whisper (MIT). + +## Updating + +An update pins new immutable revisions, re-vendors unchanged, re-runs the +public-reference parity tests, and updates this record. Numbers scored before +and after a normalizer change are not comparable. diff --git a/packages/pipette-scores/pipette_scores/scoring/asr/__init__.py b/packages/pipette-scores/pipette_scores/scoring/asr/__init__.py new file mode 100644 index 0000000..651ea1c --- /dev/null +++ b/packages/pipette-scores/pipette_scores/scoring/asr/__init__.py @@ -0,0 +1 @@ +"""ASR (speech recognition) scoring: Open ASR Leaderboard and FLEURS word error rates.""" diff --git a/packages/pipette-scores/pipette_scores/scoring/asr/align.py b/packages/pipette-scores/pipette_scores/scoring/asr/align.py new file mode 100644 index 0000000..d2b3b4f --- /dev/null +++ b/packages/pipette-scores/pipette_scores/scoring/asr/align.py @@ -0,0 +1,76 @@ +"""Edit-distance alignment for ASR scoring, with the Open ASR Leaderboard's aligners. + +Both are the leaderboard's ``normalizer/eval_utils._align_pair`` +(huggingface/open_asr_leaderboard @ 62a457c5c32d0de4df77e9470308aae7d45b055f), +over the normalized texts: + +- ``KALDIALIGN_COMPOUND``: its English scoring, + ``kaldialign.edit_distance(ref.split(), pred.split(), merge_compounds=True)`` + (kaldialign 0.12.0, the version its ``requirements/requirements_jobs.txt`` + pins), so that a split compound ("white paper" against "whitepaper") is no + error in either direction. +- ``KALDIALIGN_COMPOUND_PAIRS``: its multilingual scoring (``multilingual=True``), + which first aligns compound word boundaries between the two texts + (``eval_utils.normalize_compound_pairs``: a mismatched region that is the same + characters without spaces takes the joined form on both sides), then aligns + as ``KALDIALIGN_COMPOUND``. +""" + +from __future__ import annotations + +import enum +from dataclasses import dataclass +from typing import assert_never + + +class Aligner(enum.StrEnum): + # Open ASR Leaderboard: kaldialign over whitespace words, compounds merged (eval_utils._align_pair). + KALDIALIGN_COMPOUND = "kaldialign_compound" + # Its multilingual scoring: normalize_compound_pairs, then the same kaldialign alignment. + KALDIALIGN_COMPOUND_PAIRS = "kaldialign_compound_pairs" + + +@dataclass(frozen=True) +class Counts: + """One sample's alignment. `reference_units` is the reference's words.""" + + hits: int + substitutions: int + deletions: int + insertions: int + reference_units: int + + @property + def errors(self) -> int: + return self.substitutions + self.deletions + self.insertions + + +def _kaldialign_compound(reference: str, hypothesis: str) -> Counts: + from kaldialign import edit_distance + + result = edit_distance(reference.split(), hypothesis.split(), merge_compounds=True) + substitutions = int(result["sub"]) + deletions = int(result["del"]) + reference_units = int(result["ref_len"]) + return Counts( + hits=reference_units - substitutions - deletions, + substitutions=substitutions, + deletions=deletions, + insertions=int(result["ins"]), + reference_units=reference_units, + ) + + +def align(aligner: Aligner, reference: str, hypothesis: str) -> Counts: + """Align normalized `hypothesis` against normalized `reference`.""" + match aligner: + case Aligner.KALDIALIGN_COMPOUND: + return _kaldialign_compound(reference, hypothesis) + case Aligner.KALDIALIGN_COMPOUND_PAIRS: + from pipette_scores.scoring.asr.open_asr_normalizer.multilingual import normalize_compound_pairs + + # _align_pair: (ref,), (pred,) = normalize_compound_pairs([ref], [pred]). + (reference,), (hypothesis,) = normalize_compound_pairs([reference], [hypothesis]) + return _kaldialign_compound(reference, hypothesis) + case _: + assert_never(aligner) diff --git a/packages/pipette-scores/pipette_scores/scoring/asr/evaluate.py b/packages/pipette-scores/pipette_scores/scoring/asr/evaluate.py new file mode 100644 index 0000000..da26253 --- /dev/null +++ b/packages/pipette-scores/pipette_scores/scoring/asr/evaluate.py @@ -0,0 +1,150 @@ +"""The ASR evals' scorer: the standard `score(completions, samples)` contract over `wer.py`. + +A completion is cleaned as its prompt profile says (``prompt_profile``; none +leaves it as sent; dataset_catalog/asr_prompt_profiles.py), then the +dataset's recipe (its metadata.json ``scoring``, checked against the catalog at +load) normalizes and aligns it. A completion whose ``stop_reason`` is +``"failure"`` is a failed generation, not an answer. + +A result is an answer to every sample, or no result: a run that leaves a +sample unanswered, or carries a failed generation, is refused (``ValueError``; +``/score`` refuses it with a 400 first), never scored with the gap as an empty +transcript. The result (``context``) is the dataset's corpus rate; see +docs/asr-evals.md §3 for every field. + +Per-sample ``measures`` are sum-safe: a unit's counts and normalized texts are on +one sample (an Earnings22 call's on its first chunk; the others carry only the +unit id), so summing them over the samples gives the totals. +""" + +from __future__ import annotations + +from collections import Counter +from collections.abc import Callable, Sequence +from typing import Any + +from pipette_scores.scoring.asr.align import Aligner +from pipette_scores.scoring.asr.normalize import NormalizerRecipe +from pipette_scores.scoring.asr.wer import AsrReference, AsrScoringSpec, score_asr +from pipette_scores.types import AsrSample, SampleCompletion, SampleScore + +FAILED_STOP_REASON = "failure" + + +def spec_of(samples: Sequence[AsrSample]) -> AsrScoringSpec: + recipes = {tuple(sorted(s.scoring.items())) for s in samples} + if len(recipes) != 1: + raise ValueError(f"an ASR dataset has one scoring recipe, these samples carry {len(recipes)}") + recipe = samples[0].scoring + return AsrScoringSpec( + normalizer=NormalizerRecipe(recipe["normalizer"]), + aligner=Aligner(recipe["aligner"]), + open_asr_exclusions=bool(recipe["open_asr_exclusions"]), + ) + + +def _cleaner(prompt_profile: str | None) -> Callable[[str], str]: + if prompt_profile is None: + return lambda text: text + from pipette_scores.dataset_catalog.asr_prompt_profiles import profile + + return profile(prompt_profile).postprocess + + +def score( + completions: Sequence[SampleCompletion], + samples: Sequence[AsrSample], + *, + enable_thinking: bool | None = None, + prompt_profile: str | None = None, +) -> tuple[list[SampleScore], dict[str, Any]]: + """Per-sample scores for the given completions, and the dataset's result. + + ``enable_thinking=True`` is refused: every ASR profile runs with thinking + off, so such a run is not comparable. + """ + if enable_thinking: + raise ValueError("ASR evals run with thinking off; enable_thinking=true is not comparable") + if not samples: + raise ValueError("no samples to score") + known = {s.id for s in samples} + seen = Counter(c.id for c in completions) + if duplicates := sorted(i for i, n in seen.items() if n > 1): + raise ValueError(f"duplicate completion ids: {', '.join(duplicates[:10])}") + if unknown := sorted(set(seen) - known): + raise ValueError(f"completions reference ids not in the dataset: {', '.join(unknown[:10])}") + + unanswered = sorted(known - set(seen)) + failed = sorted(c.id for c in completions if c.stop_reason == FAILED_STOP_REASON) + if unanswered or failed: + gaps = [ + f"{label}: {', '.join(ids[:10])}" for label, ids in (("unanswered", unanswered), ("failed", failed)) if ids + ] + raise ValueError(f"an ASR eval is scored only with an answer to every sample; {'; '.join(gaps)}") + + spec = spec_of(samples) + clean = _cleaner(prompt_profile) + hypotheses = {c.id: clean(c.completion) for c in completions} + references = [ + AsrReference( + id=s.id, + reference=s.reference, + language=s.language, + session=s.session, + chunk_index=s.chunk_index, + session_size=s.raw_doc.get("parent_chunk_count") if s.session is not None else None, + ) + for s in samples + ] + result = score_asr(spec, references, hypotheses) + + measures_of: dict[str, dict[str, Any]] = {} + correct: set[str] = set() + for unit in result.units: + counts = unit.counts + owner, *members = unit.sample_ids + measures_of[owner] = { + "unit": unit.unit_id, + "hits": counts.hits, + "substitutions": counts.substitutions, + "deletions": counts.deletions, + "insertions": counts.insertions, + "reference_units": counts.reference_units, + "reference": unit.reference, + "hypothesis": unit.hypothesis, + } + for member in members: + measures_of[member] = {"unit": unit.unit_id} + if counts.errors == 0: + correct.update(unit.sample_ids) + for sample_id in result.excluded_sample_ids: + measures_of[sample_id] = {"excluded": True} + + scores = [SampleScore(id=c.id, is_correct=c.id in correct, measures=measures_of[c.id]) for c in completions] + totals = result.totals + rate = result.corpus_error_rate + context = { + "error_unit": result.error_unit, + # The dataset's corpus error rate (summed edits / summed reference units) + # as a ratio, and in percent (`result`, the leaderboard's unit). Both + # unrounded (the leaderboard shows round(result, 2)); null only when the + # dataset has no reference unit. + "error_rate": rate, + "result": None if rate is None else rate * 100, + "hits": totals.hits, + "substitutions": totals.substitutions, + "deletions": totals.deletions, + "insertions": totals.insertions, + "reference_units": totals.reference_units, + "units_scored": len(result.units), + "samples_total": len(samples), + "samples_excluded": len(result.excluded_sample_ids), + "stop_reasons": dict(sorted(Counter(c.stop_reason or "unreported" for c in completions).items())), + "prompt_profile": prompt_profile, + "scoring": { + "normalizer": spec.normalizer.value, + "aligner": spec.aligner.value, + "open_asr_exclusions": spec.open_asr_exclusions, + }, + } + return scores, context diff --git a/packages/pipette-scores/pipette_scores/scoring/asr/normalize.py b/packages/pipette-scores/pipette_scores/scoring/asr/normalize.py new file mode 100644 index 0000000..1e593c2 --- /dev/null +++ b/packages/pipette-scores/pipette_scores/scoring/asr/normalize.py @@ -0,0 +1,88 @@ +"""Text normalizers for ASR scoring, as the Open ASR Leaderboard applies them. + +Each recipe maps raw text (and the sample's ISO 639-3 language) to the form the +aligner compares, on both the reference and the hypothesis. The vendored code is +unchanged (see NOTICE.md); this module only selects it, as the leaderboard's +``normalizer/eval_utils._normalize`` does (huggingface/open_asr_leaderboard +@ 62a457c5c32d0de4df77e9470308aae7d45b055f): + +- ``OPEN_ASR_ENGLISH``: its ``EnglishTextNormalizer()`` (``normalizer/normalizer.py``), + for its English sets. +- ``OPEN_ASR_MULTILINGUAL``: what ``_normalize(text, language)`` applies for the + multilingual scoring (``score_results(multilingual=True, language=...)``), for + FLEURS: the English normalizer when the language is ``"en"``, otherwise + ``data_utils.ml_normalizer``, a ``MultilingualNormalizer(remove_diacritics=False)`` + called with ``lang=`` (lowercase, bracketed and parenthesized spans + dropped, symbols and punctuation to spaces with diacritics and combining marks + kept, then every digit run spelled out by num2words in that language, so that + "20" and "twenty" compare equal). The language is the leaderboard's two-letter + code (its ``ML_LANG_DATASETS``), mapped from ISO 639-3 below. +""" + +from __future__ import annotations + +import enum +from collections.abc import Callable +from functools import cache +from typing import assert_never + + +class NormalizerRecipe(enum.StrEnum): + # Open ASR Leaderboard sets: its English normalizer. + OPEN_ASR_ENGLISH = "open_asr_english" + # FLEURS: the leaderboard's multilingual scoring normalizer for the sample's language. + OPEN_ASR_MULTILINGUAL = "open_asr_multilingual" + + +# ISO 639-3 to the code the leaderboard passes as `language` (its ML_LANG_DATASETS +# keys, and "en" for its English normalizer): the languages evaluated here. Any +# other language is refused rather than normalized without its number words. +OPEN_ASR_LANGUAGE_CODES: dict[str, str] = { + "eng": "en", + "spa": "es", + "por": "pt", + "deu": "de", + "fra": "fr", + "ita": "it", + "nld": "nl", +} + +type Normalizer = Callable[[str, str | None], str] + + +@cache +def _open_asr_english() -> Callable[[str], str]: + from pipette_scores.scoring.asr.open_asr_normalizer.normalizer import EnglishTextNormalizer + + return EnglishTextNormalizer() + + +@cache +def _open_asr_multilingual() -> Callable[..., str]: + from pipette_scores.scoring.asr.open_asr_normalizer.multilingual import MultilingualNormalizer + + # data_utils.ml_normalizer. + return MultilingualNormalizer(remove_diacritics=False) + + +def open_asr_language(language: str | None) -> str: + """The leaderboard's code for an ISO 639-3 `language`, refusing one not evaluated here.""" + code = OPEN_ASR_LANGUAGE_CODES.get((language or "").strip().lower()) + if code is None: + raise ValueError(f"the Open ASR Leaderboard's multilingual scoring is not set up for language {language!r}") + return code + + +def normalize(recipe: NormalizerRecipe, text: str, language: str | None) -> str: + """`text` as `recipe` normalizes it for a sample in `language` (ISO 639-3).""" + match recipe: + case NormalizerRecipe.OPEN_ASR_ENGLISH: + return _open_asr_english()(text) + case NormalizerRecipe.OPEN_ASR_MULTILINGUAL: + # eval_utils._normalize: the English normalizer for "en", else ml_normalizer(text, lang=language). + code = open_asr_language(language) + if code == "en": + return _open_asr_english()(text) + return _open_asr_multilingual()(text, lang=code) + case _: + assert_never(recipe) diff --git a/packages/pipette-scores/pipette_scores/scoring/asr/open_asr_normalizer/__init__.py b/packages/pipette-scores/pipette_scores/scoring/asr/open_asr_normalizer/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/packages/pipette-scores/pipette_scores/scoring/asr/open_asr_normalizer/english_abbreviations.py b/packages/pipette-scores/pipette_scores/scoring/asr/open_asr_normalizer/english_abbreviations.py new file mode 100644 index 0000000..1ef84b9 --- /dev/null +++ b/packages/pipette-scores/pipette_scores/scoring/asr/open_asr_normalizer/english_abbreviations.py @@ -0,0 +1,1934 @@ +english_spelling_normalizer = { + "ok": "okay", + "kay": "okay", + "etcetera": "etc", + "accessorise": "accessorize", + "accessorised": "accessorized", + "accessorises": "accessorizes", + "accessorising": "accessorizing", + "acclimatisation": "acclimatization", + "acclimatise": "acclimatize", + "acclimatised": "acclimatized", + "acclimatises": "acclimatizes", + "acclimatising": "acclimatizing", + "accoutrements": "accouterments", + "aeon": "eon", + "aeons": "eons", + "aerogramme": "aerogram", + "aerogrammes": "aerograms", + "aeroplane": "airplane", + "aeroplanes": "airplanes", + "aesthete": "esthete", + "aesthetes": "esthetes", + "aesthetic": "esthetic", + "aesthetically": "esthetically", + "aesthetics": "esthetics", + "aetiology": "etiology", + "ageing": "aging", + "aggrandisement": "aggrandizement", + "agonise": "agonize", + "agonised": "agonized", + "agonises": "agonizes", + "agonising": "agonizing", + "agonisingly": "agonizingly", + "almanack": "almanac", + "almanacks": "almanacs", + "aluminium": "aluminum", + "amortisable": "amortizable", + "amortisation": "amortization", + "amortisations": "amortizations", + "amortise": "amortize", + "amortised": "amortized", + "amortises": "amortizes", + "amortising": "amortizing", + "amphitheatre": "amphitheater", + "amphitheatres": "amphitheaters", + "anaemia": "anemia", + "anaemic": "anemic", + "anaesthesia": "anesthesia", + "anaesthetic": "anesthetic", + "anaesthetics": "anesthetics", + "anaesthetise": "anesthetize", + "anaesthetised": "anesthetized", + "anaesthetises": "anesthetizes", + "anaesthetising": "anesthetizing", + "anaesthetist": "anesthetist", + "anaesthetists": "anesthetists", + "anaesthetize": "anesthetize", + "anaesthetized": "anesthetized", + "anaesthetizes": "anesthetizes", + "anaesthetizing": "anesthetizing", + "analogue": "analog", + "analogues": "analogs", + "analyse": "analyze", + "analysed": "analyzed", + "analyses": "analyzes", + "analysing": "analyzing", + "anglicise": "anglicize", + "anglicised": "anglicized", + "anglicises": "anglicizes", + "anglicising": "anglicizing", + "annualised": "annualized", + "antagonise": "antagonize", + "antagonised": "antagonized", + "antagonises": "antagonizes", + "antagonising": "antagonizing", + "apologise": "apologize", + "apologised": "apologized", + "apologises": "apologizes", + "apologising": "apologizing", + "appal": "appall", + "appals": "appalls", + "appetiser": "appetizer", + "appetisers": "appetizers", + "appetising": "appetizing", + "appetisingly": "appetizingly", + "arbour": "arbor", + "arbours": "arbors", + "archaeologically": "archeologically", + "archaeologist": "archeologist", + "archaeologists": "archeologists", + "archaeology": "archeology", + "archaeological": "archeological", + "ardour": "ardor", + "armour": "armor", + "armoured": "armored", + "armourer": "armorer", + "armourers": "armorers", + "armouries": "armories", + "armoury": "armory", + "artefact": "artifact", + "artefacts": "artifacts", + "authorise": "authorize", + "authorised": "authorized", + "authorises": "authorizes", + "authorising": "authorizing", + "axe": "ax", + "backpedalled": "backpedaled", + "backpedalling": "backpedaling", + "bannister": "banister", + "bannisters": "banisters", + "baptise": "baptize", + "baptised": "baptized", + "baptises": "baptizes", + "baptising": "baptizing", + "bastardise": "bastardize", + "bastardised": "bastardized", + "bastardises": "bastardizes", + "bastardising": "bastardizing", + "battleax": "battleaxe", + "baulk": "balk", + "baulked": "balked", + "baulking": "balking", + "baulks": "balks", + "bedevilled": "bedeviled", + "bedevilling": "bedeviling", + "behaviour": "behavior", + "behavioural": "behavioral", + "behaviourism": "behaviorism", + "behaviourist": "behaviorist", + "behaviourists": "behaviorists", + "behaviours": "behaviors", + "behove": "behoove", + "behoved": "behooved", + "behoves": "behooves", + "bejewelled": "bejeweled", + "belabour": "belabor", + "belaboured": "belabored", + "belabouring": "belaboring", + "belabours": "belabors", + "bevelled": "beveled", + "bevvies": "bevies", + "bevvy": "bevy", + "biassed": "biased", + "biassing": "biasing", + "bingeing": "binging", + "bougainvillaea": "bougainvillea", + "bougainvillaeas": "bougainvilleas", + "bowdlerise": "bowdlerize", + "bowdlerised": "bowdlerized", + "bowdlerises": "bowdlerizes", + "bowdlerising": "bowdlerizing", + "breathalyse": "breathalyze", + "breathalysed": "breathalyzed", + "breathalyser": "breathalyzer", + "breathalysers": "breathalyzers", + "breathalyses": "breathalyzes", + "breathalysing": "breathalyzing", + "brutalise": "brutalize", + "brutalised": "brutalized", + "brutalises": "brutalizes", + "brutalising": "brutalizing", + "busses": "buses", + "bussing": "busing", + "caesarean": "cesarean", + "caesareans": "cesareans", + "calibre": "caliber", + "calibres": "calibers", + "calliper": "caliper", + "callipers": "calipers", + "callisthenics": "calisthenics", + "canalise": "canalize", + "canalised": "canalized", + "canalises": "canalizes", + "canalising": "canalizing", + "cancellation": "cancelation", + "cancellations": "cancelations", + "cancelled": "canceled", + "cancelling": "canceling", + "candour": "candor", + "cannibalise": "cannibalize", + "cannibalised": "cannibalized", + "cannibalises": "cannibalizes", + "cannibalising": "cannibalizing", + "canonise": "canonize", + "canonised": "canonized", + "canonises": "canonizes", + "canonising": "canonizing", + "capitalise": "capitalize", + "capitalised": "capitalized", + "capitalises": "capitalizes", + "capitalising": "capitalizing", + "caramelise": "caramelize", + "caramelised": "caramelized", + "caramelises": "caramelizes", + "caramelising": "caramelizing", + "carbonise": "carbonize", + "carbonised": "carbonized", + "carbonises": "carbonizes", + "carbonising": "carbonizing", + "carolled": "caroled", + "carolling": "caroling", + "catalogue": "catalog", + "catalogued": "cataloged", + "catalogues": "catalogs", + "cataloguing": "cataloging", + "catalyse": "catalyze", + "catalysed": "catalyzed", + "catalyses": "catalyzes", + "catalysing": "catalyzing", + "categorise": "categorize", + "categorised": "categorized", + "categorises": "categorizes", + "categorising": "categorizing", + "cauterise": "cauterize", + "cauterised": "cauterized", + "cauterises": "cauterizes", + "cauterising": "cauterizing", + "cavilled": "caviled", + "cavilling": "caviling", + "centigramme": "centigram", + "centigrammes": "centigrams", + "centilitre": "centiliter", + "centilitres": "centiliters", + "centimetre": "centimeter", + "centimetres": "centimeters", + "centralise": "centralize", + "centralised": "centralized", + "centralises": "centralizes", + "centralising": "centralizing", + "centre": "center", + "centred": "centered", + "centrefold": "centerfold", + "centrefolds": "centerfolds", + "centrepiece": "centerpiece", + "centrepieces": "centerpieces", + "centres": "centers", + "channelled": "channeled", + "channelling": "channeling", + "characterise": "characterize", + "characterised": "characterized", + "characterises": "characterizes", + "characterising": "characterizing", + "cheque": "check", + "chequebook": "checkbook", + "chequebooks": "checkbooks", + "chequered": "checkered", + "cheques": "checks", + "chilli": "chili", + "chimaera": "chimera", + "chimaeras": "chimeras", + "chiselled": "chiseled", + "chiselling": "chiseling", + "circularise": "circularize", + "circularised": "circularized", + "circularises": "circularizes", + "circularising": "circularizing", + "civilise": "civilize", + "civilised": "civilized", + "civilises": "civilizes", + "civilising": "civilizing", + "clamour": "clamor", + "clamoured": "clamored", + "clamouring": "clamoring", + "clamours": "clamors", + "clangour": "clangor", + "clarinettist": "clarinetist", + "clarinettists": "clarinetists", + "collectivise": "collectivize", + "collectivised": "collectivized", + "collectivises": "collectivizes", + "collectivising": "collectivizing", + "colonisation": "colonization", + "colonise": "colonize", + "colonised": "colonized", + "coloniser": "colonizer", + "colonisers": "colonizers", + "colonises": "colonizes", + "colonising": "colonizing", + "colour": "color", + "colourant": "colorant", + "colourants": "colorants", + "coloured": "colored", + "coloureds": "coloreds", + "colourful": "colorful", + "colourfully": "colorfully", + "colouring": "coloring", + "colourize": "colorize", + "colourized": "colorized", + "colourizes": "colorizes", + "colourizing": "colorizing", + "colourless": "colorless", + "colours": "colors", + "commercialise": "commercialize", + "commercialised": "commercialized", + "commercialises": "commercializes", + "commercialising": "commercializing", + "compartmentalise": "compartmentalize", + "compartmentalised": "compartmentalized", + "compartmentalises": "compartmentalizes", + "compartmentalising": "compartmentalizing", + "computerise": "computerize", + "computerised": "computerized", + "computerises": "computerizes", + "computerising": "computerizing", + "conceptualise": "conceptualize", + "conceptualised": "conceptualized", + "conceptualises": "conceptualizes", + "conceptualising": "conceptualizing", + "connexion": "connection", + "connexions": "connections", + "contextualise": "contextualize", + "contextualised": "contextualized", + "contextualises": "contextualizes", + "contextualising": "contextualizing", + "cosier": "cozier", + "cosies": "cozies", + "cosiest": "coziest", + "cosily": "cozily", + "cosiness": "coziness", + "cosy": "cozy", + "councillor": "councilor", + "councillors": "councilors", + "counselled": "counseled", + "counselling": "counseling", + "counsellor": "counselor", + "counsellors": "counselors", + "crenelated": "crenellated", + "criminalise": "criminalize", + "criminalised": "criminalized", + "criminalises": "criminalizes", + "criminalising": "criminalizing", + "criticise": "criticize", + "criticised": "criticized", + "criticises": "criticizes", + "criticising": "criticizing", + "crueller": "crueler", + "cruellest": "cruelest", + "crystallisation": "crystallization", + "crystallise": "crystallize", + "crystallised": "crystallized", + "crystallises": "crystallizes", + "crystallising": "crystallizing", + "cudgelled": "cudgeled", + "cudgelling": "cudgeling", + "customise": "customize", + "customised": "customized", + "customises": "customizes", + "customising": "customizing", + "cypher": "cipher", + "cyphers": "ciphers", + "decentralisation": "decentralization", + "decentralise": "decentralize", + "decentralised": "decentralized", + "decentralises": "decentralizes", + "decentralising": "decentralizing", + "decriminalisation": "decriminalization", + "decriminalise": "decriminalize", + "decriminalised": "decriminalized", + "decriminalises": "decriminalizes", + "decriminalising": "decriminalizing", + "defence": "defense", + "defenceless": "defenseless", + "defences": "defenses", + "dehumanisation": "dehumanization", + "dehumanise": "dehumanize", + "dehumanised": "dehumanized", + "dehumanises": "dehumanizes", + "dehumanising": "dehumanizing", + "demeanour": "demeanor", + "demilitarisation": "demilitarization", + "demilitarise": "demilitarize", + "demilitarised": "demilitarized", + "demilitarises": "demilitarizes", + "demilitarising": "demilitarizing", + "demobilisation": "demobilization", + "demobilise": "demobilize", + "demobilised": "demobilized", + "demobilises": "demobilizes", + "demobilising": "demobilizing", + "democratisation": "democratization", + "democratise": "democratize", + "democratised": "democratized", + "democratises": "democratizes", + "democratising": "democratizing", + "demonise": "demonize", + "demonised": "demonized", + "demonises": "demonizes", + "demonising": "demonizing", + "demoralisation": "demoralization", + "demoralise": "demoralize", + "demoralised": "demoralized", + "demoralises": "demoralizes", + "demoralising": "demoralizing", + "denationalisation": "denationalization", + "denationalise": "denationalize", + "denationalised": "denationalized", + "denationalises": "denationalizes", + "denationalising": "denationalizing", + "deodorise": "deodorize", + "deodorised": "deodorized", + "deodorises": "deodorizes", + "deodorising": "deodorizing", + "depersonalise": "depersonalize", + "depersonalised": "depersonalized", + "depersonalises": "depersonalizes", + "depersonalising": "depersonalizing", + "deputise": "deputize", + "deputised": "deputized", + "deputises": "deputizes", + "deputising": "deputizing", + "desensitisation": "desensitization", + "desensitise": "desensitize", + "desensitised": "desensitized", + "desensitises": "desensitizes", + "desensitising": "desensitizing", + "destabilisation": "destabilization", + "destabilise": "destabilize", + "destabilised": "destabilized", + "destabilises": "destabilizes", + "destabilising": "destabilizing", + "dialled": "dialed", + "dialling": "dialing", + "dialogue": "dialog", + "dialogues": "dialogs", + "diarrhoea": "diarrhea", + "digitise": "digitize", + "digitised": "digitized", + "digitises": "digitizes", + "digitising": "digitizing", + "disc": "disk", + "discolour": "discolor", + "discoloured": "discolored", + "discolouring": "discoloring", + "discolours": "discolors", + "discs": "disks", + "disembowelled": "disemboweled", + "disembowelling": "disemboweling", + "disfavour": "disfavor", + "dishevelled": "disheveled", + "dishonour": "dishonor", + "dishonourable": "dishonorable", + "dishonourably": "dishonorably", + "dishonoured": "dishonored", + "dishonouring": "dishonoring", + "dishonours": "dishonors", + "disorganisation": "disorganization", + "disorganised": "disorganized", + "distil": "distill", + "distils": "distills", + "dramatisation": "dramatization", + "dramatisations": "dramatizations", + "dramatise": "dramatize", + "dramatised": "dramatized", + "dramatises": "dramatizes", + "dramatising": "dramatizing", + "draught": "draft", + "draughtboard": "draftboard", + "draughtboards": "draftboards", + "draughtier": "draftier", + "draughtiest": "draftiest", + "draughts": "drafts", + "draughtsman": "draftsman", + "draughtsmanship": "draftsmanship", + "draughtsmen": "draftsmen", + "draughtswoman": "draftswoman", + "draughtswomen": "draftswomen", + "draughty": "drafty", + "drivelled": "driveled", + "drivelling": "driveling", + "duelled": "dueled", + "duelling": "dueling", + "economise": "economize", + "economised": "economized", + "economises": "economizes", + "economising": "economizing", + "editorialise": "editorialize", + "editorialised": "editorialized", + "editorialises": "editorializes", + "editorialising": "editorializing", + "edoema": "edema", + "empathise": "empathize", + "empathised": "empathized", + "empathises": "empathizes", + "empathising": "empathizing", + "emphasise": "emphasize", + "emphasised": "emphasized", + "emphasises": "emphasizes", + "emphasising": "emphasizing", + "enamelled": "enameled", + "enamelling": "enameling", + "enamoured": "enamored", + "encyclopaedia": "encyclopedia", + "encyclopaedias": "encyclopedias", + "encyclopaedic": "encyclopedic", + "endeavour": "endeavor", + "endeavoured": "endeavored", + "endeavouring": "endeavoring", + "endeavours": "endeavors", + "energise": "energize", + "energised": "energized", + "energises": "energizes", + "energising": "energizing", + "enrol": "enroll", + "enrols": "enrolls", + "enthral": "enthrall", + "enthrals": "enthralls", + "epaulette": "epaulet", + "epaulettes": "epaulets", + "epicentre": "epicenter", + "epicentres": "epicenters", + "epilogue": "epilog", + "epilogues": "epilogs", + "epitomise": "epitomize", + "epitomised": "epitomized", + "epitomises": "epitomizes", + "epitomising": "epitomizing", + "equalisation": "equalization", + "equalise": "equalize", + "equalised": "equalized", + "equaliser": "equalizer", + "equalisers": "equalizers", + "equalises": "equalizes", + "equalising": "equalizing", + "eulogise": "eulogize", + "eulogised": "eulogized", + "eulogises": "eulogizes", + "eulogising": "eulogizing", + "evangelise": "evangelize", + "evangelised": "evangelized", + "evangelises": "evangelizes", + "evangelising": "evangelizing", + "exorcise": "exorcize", + "exorcised": "exorcized", + "exorcises": "exorcizes", + "exorcising": "exorcizing", + "extemporisation": "extemporization", + "extemporise": "extemporize", + "extemporised": "extemporized", + "extemporises": "extemporizes", + "extemporising": "extemporizing", + "externalisation": "externalization", + "externalisations": "externalizations", + "externalise": "externalize", + "externalised": "externalized", + "externalises": "externalizes", + "externalising": "externalizing", + "factorise": "factorize", + "factorised": "factorized", + "factorises": "factorizes", + "factorising": "factorizing", + "faecal": "fecal", + "faeces": "feces", + "familiarisation": "familiarization", + "familiarise": "familiarize", + "familiarised": "familiarized", + "familiarises": "familiarizes", + "familiarising": "familiarizing", + "fantasise": "fantasize", + "fantasised": "fantasized", + "fantasises": "fantasizes", + "fantasising": "fantasizing", + "favour": "favor", + "favourable": "favorable", + "favourably": "favorably", + "favoured": "favored", + "favouring": "favoring", + "favourite": "favorite", + "favourites": "favorites", + "favouritism": "favoritism", + "favours": "favors", + "feminise": "feminize", + "feminised": "feminized", + "feminises": "feminizes", + "feminising": "feminizing", + "fertilisation": "fertilization", + "fertilise": "fertilize", + "fertilised": "fertilized", + "fertiliser": "fertilizer", + "fertilisers": "fertilizers", + "fertilises": "fertilizes", + "fertilising": "fertilizing", + "fervour": "fervor", + "fibre": "fiber", + "fibreglass": "fiberglass", + "fibres": "fibers", + "fictionalisation": "fictionalization", + "fictionalisations": "fictionalizations", + "fictionalise": "fictionalize", + "fictionalised": "fictionalized", + "fictionalises": "fictionalizes", + "fictionalising": "fictionalizing", + "fillet": "filet", + "filleted": "fileted", + "filleting": "fileting", + "fillets": "filets", + "finalisation": "finalization", + "finalise": "finalize", + "finalised": "finalized", + "finalises": "finalizes", + "finalising": "finalizing", + "flautist": "flutist", + "flautists": "flutists", + "flavour": "flavor", + "flavoured": "flavored", + "flavouring": "flavoring", + "flavourings": "flavorings", + "flavourless": "flavorless", + "flavours": "flavors", + "flavoursome": "flavorsome", + "flyer": "flier", + "foetal": "fetal", + "foetid": "fetid", + "foetus": "fetus", + "foetuses": "fetuses", + "formalisation": "formalization", + "formalise": "formalize", + "formalised": "formalized", + "formalises": "formalizes", + "formalising": "formalizing", + "fossilisation": "fossilization", + "fossilise": "fossilize", + "fossilised": "fossilized", + "fossilises": "fossilizes", + "fossilising": "fossilizing", + "fraternisation": "fraternization", + "fraternise": "fraternize", + "fraternised": "fraternized", + "fraternises": "fraternizes", + "fraternising": "fraternizing", + "fulfil": "fulfill", + "fulfilment": "fulfillment", + "fulfils": "fulfills", + "funnelled": "funneled", + "funnelling": "funneling", + "gage": "gauge", + "gaged": "gauged", + "gages": "gauges", + "gaging": "gauging", + "galvanise": "galvanize", + "galvanised": "galvanized", + "galvanises": "galvanizes", + "galvanising": "galvanizing", + "gambolled": "gamboled", + "gambolling": "gamboling", + "gaol": "jail", + "gaolbird": "jailbird", + "gaolbirds": "jailbirds", + "gaolbreak": "jailbreak", + "gaolbreaks": "jailbreaks", + "gaoled": "jailed", + "gaoler": "jailer", + "gaolers": "jailers", + "gaoling": "jailing", + "gaols": "jails", + "gasses": "gases", + "generalisation": "generalization", + "generalisations": "generalizations", + "generalise": "generalize", + "generalised": "generalized", + "generalises": "generalizes", + "generalising": "generalizing", + "ghettoise": "ghettoize", + "ghettoised": "ghettoized", + "ghettoises": "ghettoizes", + "ghettoising": "ghettoizing", + "gipsies": "gypsies", + "glamor": "glamour", + "glamorise": "glamorize", + "glamorised": "glamorized", + "glamorises": "glamorizes", + "glamorising": "glamorizing", + "globalisation": "globalization", + "globalise": "globalize", + "globalised": "globalized", + "globalises": "globalizes", + "globalising": "globalizing", + "glueing": "gluing", + "goitre": "goiter", + "goitres": "goiters", + "gonorrhoea": "gonorrhea", + "gramme": "gram", + "grammes": "grams", + "gravelled": "graveled", + "grey": "gray", + "greyed": "grayed", + "greying": "graying", + "greyish": "grayish", + "greyness": "grayness", + "greys": "grays", + "grovelled": "groveled", + "grovelling": "groveling", + "groyne": "groin", + "groynes": "groins", + "gruelling": "grueling", + "gruellingly": "gruelingly", + "gryphon": "griffin", + "gryphons": "griffins", + "gynaecological": "gynecological", + "gynaecologist": "gynecologist", + "gynaecologists": "gynecologists", + "gynaecology": "gynecology", + "haematological": "hematological", + "haematologist": "hematologist", + "haematologists": "hematologists", + "haematology": "hematology", + "haemoglobin": "hemoglobin", + "haemophilia": "hemophilia", + "haemophiliac": "hemophiliac", + "haemophiliacs": "hemophiliacs", + "haemorrhage": "hemorrhage", + "haemorrhaged": "hemorrhaged", + "haemorrhages": "hemorrhages", + "haemorrhaging": "hemorrhaging", + "haemorrhoids": "hemorrhoids", + "harbour": "harbor", + "harboured": "harbored", + "harbouring": "harboring", + "harbours": "harbors", + "harmonisation": "harmonization", + "harmonise": "harmonize", + "harmonised": "harmonized", + "harmonises": "harmonizes", + "harmonising": "harmonizing", + "homoeopath": "homeopath", + "homoeopathic": "homeopathic", + "homoeopaths": "homeopaths", + "homoeopathy": "homeopathy", + "homogenise": "homogenize", + "homogenised": "homogenized", + "homogenises": "homogenizes", + "homogenising": "homogenizing", + "honour": "honor", + "honourable": "honorable", + "honourably": "honorably", + "honoured": "honored", + "honouring": "honoring", + "honours": "honors", + "hospitalisation": "hospitalization", + "hospitalise": "hospitalize", + "hospitalised": "hospitalized", + "hospitalises": "hospitalizes", + "hospitalising": "hospitalizing", + "humanise": "humanize", + "humanised": "humanized", + "humanises": "humanizes", + "humanising": "humanizing", + "humour": "humor", + "humoured": "humored", + "humouring": "humoring", + "humourless": "humorless", + "humours": "humors", + "hybridise": "hybridize", + "hybridised": "hybridized", + "hybridises": "hybridizes", + "hybridising": "hybridizing", + "hypnotise": "hypnotize", + "hypnotised": "hypnotized", + "hypnotises": "hypnotizes", + "hypnotising": "hypnotizing", + "hypothesise": "hypothesize", + "hypothesised": "hypothesized", + "hypothesises": "hypothesizes", + "hypothesising": "hypothesizing", + "idealisation": "idealization", + "idealise": "idealize", + "idealised": "idealized", + "idealises": "idealizes", + "idealising": "idealizing", + "idolise": "idolize", + "idolised": "idolized", + "idolises": "idolizes", + "idolising": "idolizing", + "immobilisation": "immobilization", + "immobilise": "immobilize", + "immobilised": "immobilized", + "immobiliser": "immobilizer", + "immobilisers": "immobilizers", + "immobilises": "immobilizes", + "immobilising": "immobilizing", + "immortalise": "immortalize", + "immortalised": "immortalized", + "immortalises": "immortalizes", + "immortalising": "immortalizing", + "immunisation": "immunization", + "immunise": "immunize", + "immunised": "immunized", + "immunises": "immunizes", + "immunising": "immunizing", + "impanelled": "impaneled", + "impanelling": "impaneling", + "imperilled": "imperiled", + "imperilling": "imperiling", + "individualise": "individualize", + "individualised": "individualized", + "individualises": "individualizes", + "individualising": "individualizing", + "industrialise": "industrialize", + "industrialised": "industrialized", + "industrialises": "industrializes", + "industrialising": "industrializing", + "inflexion": "inflection", + "inflexions": "inflections", + "initialise": "initialize", + "initialised": "initialized", + "initialises": "initializes", + "initialising": "initializing", + "initialled": "initialed", + "initialling": "initialing", + "instal": "install", + "instalment": "installment", + "instalments": "installments", + "instals": "installs", + "instil": "instill", + "instils": "instills", + "institutionalisation": "institutionalization", + "institutionalise": "institutionalize", + "institutionalised": "institutionalized", + "institutionalises": "institutionalizes", + "institutionalising": "institutionalizing", + "intellectualise": "intellectualize", + "intellectualised": "intellectualized", + "intellectualises": "intellectualizes", + "intellectualising": "intellectualizing", + "internalisation": "internalization", + "internalise": "internalize", + "internalised": "internalized", + "internalises": "internalizes", + "internalising": "internalizing", + "internationalisation": "internationalization", + "internationalise": "internationalize", + "internationalised": "internationalized", + "internationalises": "internationalizes", + "internationalising": "internationalizing", + "ionisation": "ionization", + "ionise": "ionize", + "ionised": "ionized", + "ioniser": "ionizer", + "ionisers": "ionizers", + "ionises": "ionizes", + "ionising": "ionizing", + "italicise": "italicize", + "italicised": "italicized", + "italicises": "italicizes", + "italicising": "italicizing", + "itemise": "itemize", + "itemised": "itemized", + "itemises": "itemizes", + "itemising": "itemizing", + "jeopardise": "jeopardize", + "jeopardised": "jeopardized", + "jeopardises": "jeopardizes", + "jeopardising": "jeopardizing", + "jewelled": "jeweled", + "jeweller": "jeweler", + "jewellers": "jewelers", + "jewellery": "jewelry", + "judgement": "judgment", + "kilogramme": "kilogram", + "kilogrammes": "kilograms", + "kilometre": "kilometer", + "kilometres": "kilometers", + "labelled": "labeled", + "labelling": "labeling", + "labour": "labor", + "laboured": "labored", + "labourer": "laborer", + "labourers": "laborers", + "labouring": "laboring", + "labours": "labors", + "lacklustre": "lackluster", + "legalisation": "legalization", + "legalise": "legalize", + "legalised": "legalized", + "legalises": "legalizes", + "legalising": "legalizing", + "legitimise": "legitimize", + "legitimised": "legitimized", + "legitimises": "legitimizes", + "legitimising": "legitimizing", + "leukaemia": "leukemia", + "levelled": "leveled", + "leveller": "leveler", + "levellers": "levelers", + "levelling": "leveling", + "libelled": "libeled", + "libelling": "libeling", + "libellous": "libelous", + "liberalisation": "liberalization", + "liberalise": "liberalize", + "liberalised": "liberalized", + "liberalises": "liberalizes", + "liberalising": "liberalizing", + "licence": "license", + "licenced": "licensed", + "licences": "licenses", + "licencing": "licensing", + "likeable": "likable", + "lionisation": "lionization", + "lionise": "lionize", + "lionised": "lionized", + "lionises": "lionizes", + "lionising": "lionizing", + "liquidise": "liquidize", + "liquidised": "liquidized", + "liquidiser": "liquidizer", + "liquidisers": "liquidizers", + "liquidises": "liquidizes", + "liquidising": "liquidizing", + "litre": "liter", + "litres": "liters", + "localise": "localize", + "localised": "localized", + "localises": "localizes", + "localising": "localizing", + "louvre": "louver", + "louvred": "louvered", + "louvres": "louvers", + "lustre": "luster", + "magnetise": "magnetize", + "magnetised": "magnetized", + "magnetises": "magnetizes", + "magnetising": "magnetizing", + "manoeuvrability": "maneuverability", + "manoeuvrable": "maneuverable", + "manoeuvre": "maneuver", + "manoeuvred": "maneuvered", + "manoeuvres": "maneuvers", + "manoeuvring": "maneuvering", + "manoeuvrings": "maneuverings", + "marginalisation": "marginalization", + "marginalise": "marginalize", + "marginalised": "marginalized", + "marginalises": "marginalizes", + "marginalising": "marginalizing", + "marshalled": "marshaled", + "marshalling": "marshaling", + "marvelled": "marveled", + "marvelling": "marveling", + "marvellous": "marvelous", + "marvellously": "marvelously", + "materialisation": "materialization", + "materialise": "materialize", + "materialised": "materialized", + "materialises": "materializes", + "materialising": "materializing", + "maximisation": "maximization", + "maximise": "maximize", + "maximised": "maximized", + "maximises": "maximizes", + "maximising": "maximizing", + "meagre": "meager", + "mechanisation": "mechanization", + "mechanise": "mechanize", + "mechanised": "mechanized", + "mechanises": "mechanizes", + "mechanising": "mechanizing", + "mediaeval": "medieval", + "memorialise": "memorialize", + "memorialised": "memorialized", + "memorialises": "memorializes", + "memorialising": "memorializing", + "memorise": "memorize", + "memorised": "memorized", + "memorises": "memorizes", + "memorising": "memorizing", + "mesmerise": "mesmerize", + "mesmerised": "mesmerized", + "mesmerises": "mesmerizes", + "mesmerising": "mesmerizing", + "metabolise": "metabolize", + "metabolised": "metabolized", + "metabolises": "metabolizes", + "metabolising": "metabolizing", + "metre": "meter", + "metres": "meters", + "mhm": "hmm", + "micrometre": "micrometer", + "micrometres": "micrometers", + "militarise": "militarize", + "militarised": "militarized", + "militarises": "militarizes", + "militarising": "militarizing", + "milligramme": "milligram", + "milligrammes": "milligrams", + "millilitre": "milliliter", + "millilitres": "milliliters", + "millimetre": "millimeter", + "millimetres": "millimeters", + "miniaturisation": "miniaturization", + "miniaturise": "miniaturize", + "miniaturised": "miniaturized", + "miniaturises": "miniaturizes", + "miniaturising": "miniaturizing", + "minibusses": "minibuses", + "minimise": "minimize", + "minimised": "minimized", + "minimises": "minimizes", + "minimising": "minimizing", + "misbehaviour": "misbehavior", + "misdemeanour": "misdemeanor", + "misdemeanours": "misdemeanors", + "misspelt": "misspelled", + "mitre": "miter", + "mitres": "miters", + "mm": "hmm", + "mmm": "hmm", + "mobilisation": "mobilization", + "mobilise": "mobilize", + "mobilised": "mobilized", + "mobilises": "mobilizes", + "mobilising": "mobilizing", + "modelled": "modeled", + "modeller": "modeler", + "modellers": "modelers", + "modelling": "modeling", + "modernise": "modernize", + "modernised": "modernized", + "modernises": "modernizes", + "modernising": "modernizing", + "moisturise": "moisturize", + "moisturised": "moisturized", + "moisturiser": "moisturizer", + "moisturisers": "moisturizers", + "moisturises": "moisturizes", + "moisturising": "moisturizing", + "monologue": "monolog", + "monologues": "monologs", + "monopolisation": "monopolization", + "monopolise": "monopolize", + "monopolised": "monopolized", + "monopolises": "monopolizes", + "monopolising": "monopolizing", + "moralise": "moralize", + "moralised": "moralized", + "moralises": "moralizes", + "moralising": "moralizing", + "motorised": "motorized", + "mould": "mold", + "moulded": "molded", + "moulder": "molder", + "mouldered": "moldered", + "mouldering": "moldering", + "moulders": "molders", + "mouldier": "moldier", + "mouldiest": "moldiest", + "moulding": "molding", + "mouldings": "moldings", + "moulds": "molds", + "mouldy": "moldy", + "moult": "molt", + "moulted": "molted", + "moulting": "molting", + "moults": "molts", + "moustache": "mustache", + "moustached": "mustached", + "moustaches": "mustaches", + "moustachioed": "mustachioed", + "multicoloured": "multicolored", + "nationalisation": "nationalization", + "nationalisations": "nationalizations", + "nationalise": "nationalize", + "nationalised": "nationalized", + "nationalises": "nationalizes", + "nationalising": "nationalizing", + "naturalisation": "naturalization", + "naturalise": "naturalize", + "naturalised": "naturalized", + "naturalises": "naturalizes", + "naturalising": "naturalizing", + "neighbour": "neighbor", + "neighbourhood": "neighborhood", + "neighbourhoods": "neighborhoods", + "neighbouring": "neighboring", + "neighbourliness": "neighborliness", + "neighbourly": "neighborly", + "neighbours": "neighbors", + "neutralisation": "neutralization", + "neutralise": "neutralize", + "neutralised": "neutralized", + "neutralises": "neutralizes", + "neutralising": "neutralizing", + "normalisation": "normalization", + "normalise": "normalize", + "normalised": "normalized", + "normalises": "normalizes", + "normalising": "normalizing", + "odour": "odor", + "odourless": "odorless", + "odours": "odors", + "oesophagus": "esophagus", + "oesophaguses": "esophaguses", + "oestrogen": "estrogen", + "offence": "offense", + "offences": "offenses", + "omelette": "omelet", + "omelettes": "omelets", + "optimise": "optimize", + "optimised": "optimized", + "optimises": "optimizes", + "optimising": "optimizing", + "organisation": "organization", + "organisational": "organizational", + "organisations": "organizations", + "organise": "organize", + "organised": "organized", + "organiser": "organizer", + "organisers": "organizers", + "organises": "organizes", + "organising": "organizing", + "orthopaedic": "orthopedic", + "orthopaedics": "orthopedics", + "ostracise": "ostracize", + "ostracised": "ostracized", + "ostracises": "ostracizes", + "ostracising": "ostracizing", + "outmanoeuvre": "outmaneuver", + "outmanoeuvred": "outmaneuvered", + "outmanoeuvres": "outmaneuvers", + "outmanoeuvring": "outmaneuvering", + "overemphasise": "overemphasize", + "overemphasised": "overemphasized", + "overemphasises": "overemphasizes", + "overemphasising": "overemphasizing", + "oxidisation": "oxidization", + "oxidise": "oxidize", + "oxidised": "oxidized", + "oxidises": "oxidizes", + "oxidising": "oxidizing", + "paederast": "pederast", + "paederasts": "pederasts", + "paediatric": "pediatric", + "paediatrician": "pediatrician", + "paediatricians": "pediatricians", + "paediatrics": "pediatrics", + "paedophile": "pedophile", + "paedophiles": "pedophiles", + "paedophilia": "pedophilia", + "palaeolithic": "paleolithic", + "palaeontologist": "paleontologist", + "palaeontologists": "paleontologists", + "palaeontology": "paleontology", + "panelled": "paneled", + "panelling": "paneling", + "panellist": "panelist", + "panellists": "panelists", + "paralyse": "paralyze", + "paralysed": "paralyzed", + "paralyses": "paralyzes", + "paralysing": "paralyzing", + "parcelled": "parceled", + "parcelling": "parceling", + "parlour": "parlor", + "parlours": "parlors", + "particularise": "particularize", + "particularised": "particularized", + "particularises": "particularizes", + "particularising": "particularizing", + "passivisation": "passivization", + "passivise": "passivize", + "passivised": "passivized", + "passivises": "passivizes", + "passivising": "passivizing", + "pasteurisation": "pasteurization", + "pasteurise": "pasteurize", + "pasteurised": "pasteurized", + "pasteurises": "pasteurizes", + "pasteurising": "pasteurizing", + "patronise": "patronize", + "patronised": "patronized", + "patronises": "patronizes", + "patronising": "patronizing", + "patronisingly": "patronizingly", + "pedalled": "pedaled", + "pedalling": "pedaling", + "pedestrianisation": "pedestrianization", + "pedestrianise": "pedestrianize", + "pedestrianised": "pedestrianized", + "pedestrianises": "pedestrianizes", + "pedestrianising": "pedestrianizing", + "penalise": "penalize", + "penalised": "penalized", + "penalises": "penalizes", + "penalising": "penalizing", + "pencilled": "penciled", + "pencilling": "penciling", + "personalise": "personalize", + "personalised": "personalized", + "personalises": "personalizes", + "personalising": "personalizing", + "pharmacopoeia": "pharmacopeia", + "pharmacopoeias": "pharmacopeias", + "philosophise": "philosophize", + "philosophised": "philosophized", + "philosophises": "philosophizes", + "philosophising": "philosophizing", + "philtre": "filter", + "philtres": "filters", + "phoney": "phony", + "plagiarise": "plagiarize", + "plagiarised": "plagiarized", + "plagiarises": "plagiarizes", + "plagiarising": "plagiarizing", + "plough": "plow", + "ploughed": "plowed", + "ploughing": "plowing", + "ploughman": "plowman", + "ploughmen": "plowmen", + "ploughs": "plows", + "ploughshare": "plowshare", + "ploughshares": "plowshares", + "polarisation": "polarization", + "polarise": "polarize", + "polarised": "polarized", + "polarises": "polarizes", + "polarising": "polarizing", + "politicisation": "politicization", + "politicise": "politicize", + "politicised": "politicized", + "politicises": "politicizes", + "politicising": "politicizing", + "popularisation": "popularization", + "popularise": "popularize", + "popularised": "popularized", + "popularises": "popularizes", + "popularising": "popularizing", + "pouffe": "pouf", + "pouffes": "poufs", + "practise": "practice", + "practised": "practiced", + "practises": "practices", + "practising": "practicing", + "praesidium": "presidium", + "praesidiums": "presidiums", + "pressurisation": "pressurization", + "pressurise": "pressurize", + "pressurised": "pressurized", + "pressurises": "pressurizes", + "pressurising": "pressurizing", + "pretence": "pretense", + "pretences": "pretenses", + "primaeval": "primeval", + "prioritisation": "prioritization", + "prioritise": "prioritize", + "prioritised": "prioritized", + "prioritises": "prioritizes", + "prioritising": "prioritizing", + "privatisation": "privatization", + "privatisations": "privatizations", + "privatise": "privatize", + "privatised": "privatized", + "privatises": "privatizes", + "privatising": "privatizing", + "professionalisation": "professionalization", + "professionalise": "professionalize", + "professionalised": "professionalized", + "professionalises": "professionalizes", + "professionalising": "professionalizing", + "programme": "program", + "programmes": "programs", + "prologue": "prolog", + "prologues": "prologs", + "propagandise": "propagandize", + "propagandised": "propagandized", + "propagandises": "propagandizes", + "propagandising": "propagandizing", + "proselytise": "proselytize", + "proselytised": "proselytized", + "proselytiser": "proselytizer", + "proselytisers": "proselytizers", + "proselytises": "proselytizes", + "proselytising": "proselytizing", + "psychoanalyse": "psychoanalyze", + "psychoanalysed": "psychoanalyzed", + "psychoanalyses": "psychoanalyzes", + "psychoanalysing": "psychoanalyzing", + "publicise": "publicize", + "publicised": "publicized", + "publicises": "publicizes", + "publicising": "publicizing", + "pulverisation": "pulverization", + "pulverise": "pulverize", + "pulverised": "pulverized", + "pulverises": "pulverizes", + "pulverising": "pulverizing", + "pummelled": "pummeled", + "pummelling": "pummeling", + "pyjama": "pajama", + "pyjamas": "pajamas", + "pzazz": "pizzazz", + "quarrelled": "quarreled", + "quarrelling": "quarreling", + "radicalise": "radicalize", + "radicalised": "radicalized", + "radicalises": "radicalizes", + "radicalising": "radicalizing", + "rancour": "rancor", + "randomise": "randomize", + "randomised": "randomized", + "randomises": "randomizes", + "randomising": "randomizing", + "rationalisation": "rationalization", + "rationalisations": "rationalizations", + "rationalise": "rationalize", + "rationalised": "rationalized", + "rationalises": "rationalizes", + "rationalising": "rationalizing", + "ravelled": "raveled", + "ravelling": "raveling", + "realisable": "realizable", + "realisation": "realization", + "realisations": "realizations", + "realise": "realize", + "realised": "realized", + "realises": "realizes", + "realising": "realizing", + "recognisable": "recognizable", + "recognisably": "recognizably", + "recognisance": "recognizance", + "recognise": "recognize", + "recognised": "recognized", + "recognises": "recognizes", + "recognising": "recognizing", + "reconnoitre": "reconnoiter", + "reconnoitred": "reconnoitered", + "reconnoitres": "reconnoiters", + "reconnoitring": "reconnoitering", + "refuelled": "refueled", + "refuelling": "refueling", + "regularisation": "regularization", + "regularise": "regularize", + "regularised": "regularized", + "regularises": "regularizes", + "regularising": "regularizing", + "remodelled": "remodeled", + "remodelling": "remodeling", + "remould": "remold", + "remoulded": "remolded", + "remoulding": "remolding", + "remoulds": "remolds", + "reorganisation": "reorganization", + "reorganisations": "reorganizations", + "reorganise": "reorganize", + "reorganised": "reorganized", + "reorganises": "reorganizes", + "reorganising": "reorganizing", + "revelled": "reveled", + "reveller": "reveler", + "revellers": "revelers", + "revelling": "reveling", + "revitalise": "revitalize", + "revitalised": "revitalized", + "revitalises": "revitalizes", + "revitalising": "revitalizing", + "revolutionise": "revolutionize", + "revolutionised": "revolutionized", + "revolutionises": "revolutionizes", + "revolutionising": "revolutionizing", + "rhapsodise": "rhapsodize", + "rhapsodised": "rhapsodized", + "rhapsodises": "rhapsodizes", + "rhapsodising": "rhapsodizing", + "rigour": "rigor", + "rigours": "rigors", + "ritualised": "ritualized", + "rivalled": "rivaled", + "rivalling": "rivaling", + "romanticise": "romanticize", + "romanticised": "romanticized", + "romanticises": "romanticizes", + "romanticising": "romanticizing", + "rumour": "rumor", + "rumoured": "rumored", + "rumours": "rumors", + "sabre": "saber", + "sabres": "sabers", + "saltpetre": "saltpeter", + "sanitise": "sanitize", + "sanitised": "sanitized", + "sanitises": "sanitizes", + "sanitising": "sanitizing", + "satirise": "satirize", + "satirised": "satirized", + "satirises": "satirizes", + "satirising": "satirizing", + "saviour": "savior", + "saviours": "saviors", + "savour": "savor", + "savoured": "savored", + "savouries": "savories", + "savouring": "savoring", + "savours": "savors", + "savoury": "savory", + "scandalise": "scandalize", + "scandalised": "scandalized", + "scandalises": "scandalizes", + "scandalising": "scandalizing", + "sceptic": "skeptic", + "sceptical": "skeptical", + "sceptically": "skeptically", + "scepticism": "skepticism", + "sceptics": "skeptics", + "sceptre": "scepter", + "sceptres": "scepters", + "scrutinise": "scrutinize", + "scrutinised": "scrutinized", + "scrutinises": "scrutinizes", + "scrutinising": "scrutinizing", + "secularisation": "secularization", + "secularise": "secularize", + "secularised": "secularized", + "secularises": "secularizes", + "secularising": "secularizing", + "sensationalise": "sensationalize", + "sensationalised": "sensationalized", + "sensationalises": "sensationalizes", + "sensationalising": "sensationalizing", + "sensitise": "sensitize", + "sensitised": "sensitized", + "sensitises": "sensitizes", + "sensitising": "sensitizing", + "sentimentalise": "sentimentalize", + "sentimentalised": "sentimentalized", + "sentimentalises": "sentimentalizes", + "sentimentalising": "sentimentalizing", + "sepulchre": "sepulcher", + "sepulchres": "sepulchers", + "serialisation": "serialization", + "serialisations": "serializations", + "serialise": "serialize", + "serialised": "serialized", + "serialises": "serializes", + "serialising": "serializing", + "sermonise": "sermonize", + "sermonised": "sermonized", + "sermonises": "sermonizes", + "sermonising": "sermonizing", + "sheikh": "sheik", + "shovelled": "shoveled", + "shovelling": "shoveling", + "shrivelled": "shriveled", + "shrivelling": "shriveling", + "signalise": "signalize", + "signalised": "signalized", + "signalises": "signalizes", + "signalising": "signalizing", + "signalled": "signaled", + "signalling": "signaling", + "smoulder": "smolder", + "smouldered": "smoldered", + "smouldering": "smoldering", + "smoulders": "smolders", + "snivelled": "sniveled", + "snivelling": "sniveling", + "snorkelled": "snorkeled", + "snorkelling": "snorkeling", + "snowplough": "snowplow", + "snowploughs": "snowplows", + "socialisation": "socialization", + "socialise": "socialize", + "socialised": "socialized", + "socialises": "socializes", + "socialising": "socializing", + "sodomise": "sodomize", + "sodomised": "sodomized", + "sodomises": "sodomizes", + "sodomising": "sodomizing", + "solemnise": "solemnize", + "solemnised": "solemnized", + "solemnises": "solemnizes", + "solemnising": "solemnizing", + "sombre": "somber", + "specialisation": "specialization", + "specialisations": "specializations", + "specialise": "specialize", + "specialised": "specialized", + "specialises": "specializes", + "specialising": "specializing", + "spectre": "specter", + "spectres": "specters", + "spiralled": "spiraled", + "spiralling": "spiraling", + "splendour": "splendor", + "splendours": "splendors", + "squirrelled": "squirreled", + "squirrelling": "squirreling", + "stabilisation": "stabilization", + "stabilise": "stabilize", + "stabilised": "stabilized", + "stabiliser": "stabilizer", + "stabilisers": "stabilizers", + "stabilises": "stabilizes", + "stabilising": "stabilizing", + "standardisation": "standardization", + "standardise": "standardize", + "standardised": "standardized", + "standardises": "standardizes", + "standardising": "standardizing", + "stencilled": "stenciled", + "stencilling": "stenciling", + "sterilisation": "sterilization", + "sterilisations": "sterilizations", + "sterilise": "sterilize", + "sterilised": "sterilized", + "steriliser": "sterilizer", + "sterilisers": "sterilizers", + "sterilises": "sterilizes", + "sterilising": "sterilizing", + "stigmatisation": "stigmatization", + "stigmatise": "stigmatize", + "stigmatised": "stigmatized", + "stigmatises": "stigmatizes", + "stigmatising": "stigmatizing", + "storey": "story", + "storeys": "stories", + "subsidisation": "subsidization", + "subsidise": "subsidize", + "subsidised": "subsidized", + "subsidiser": "subsidizer", + "subsidisers": "subsidizers", + "subsidises": "subsidizes", + "subsidising": "subsidizing", + "succour": "succor", + "succoured": "succored", + "succouring": "succoring", + "succours": "succors", + "sulphate": "sulfate", + "sulphates": "sulfates", + "sulphide": "sulfide", + "sulphides": "sulfides", + "sulphur": "sulfur", + "sulphurous": "sulfurous", + "summarise": "summarize", + "summarised": "summarized", + "summarises": "summarizes", + "summarising": "summarizing", + "swivelled": "swiveled", + "swivelling": "swiveling", + "symbolise": "symbolize", + "symbolised": "symbolized", + "symbolises": "symbolizes", + "symbolising": "symbolizing", + "sympathise": "sympathize", + "sympathised": "sympathized", + "sympathiser": "sympathizer", + "sympathisers": "sympathizers", + "sympathises": "sympathizes", + "sympathising": "sympathizing", + "synchronisation": "synchronization", + "synchronise": "synchronize", + "synchronised": "synchronized", + "synchronises": "synchronizes", + "synchronising": "synchronizing", + "synthesise": "synthesize", + "synthesised": "synthesized", + "synthesiser": "synthesizer", + "synthesisers": "synthesizers", + "synthesises": "synthesizes", + "synthesising": "synthesizing", + "syphon": "siphon", + "syphoned": "siphoned", + "syphoning": "siphoning", + "syphons": "siphons", + "systematisation": "systematization", + "systematise": "systematize", + "systematised": "systematized", + "systematises": "systematizes", + "systematising": "systematizing", + "tantalise": "tantalize", + "tantalised": "tantalized", + "tantalises": "tantalizes", + "tantalising": "tantalizing", + "tantalisingly": "tantalizingly", + "tasselled": "tasseled", + "technicolour": "technicolor", + "temporise": "temporize", + "temporised": "temporized", + "temporises": "temporizes", + "temporising": "temporizing", + "tenderise": "tenderize", + "tenderised": "tenderized", + "tenderises": "tenderizes", + "tenderising": "tenderizing", + "terrorise": "terrorize", + "terrorised": "terrorized", + "terrorises": "terrorizes", + "terrorising": "terrorizing", + "theatre": "theater", + "theatregoer": "theatergoer", + "theatregoers": "theatergoers", + "theatres": "theaters", + "theorise": "theorize", + "theorised": "theorized", + "theorises": "theorizes", + "theorising": "theorizing", + "tonne": "ton", + "tonnes": "tons", + "towelled": "toweled", + "towelling": "toweling", + "toxaemia": "toxemia", + "tranquillise": "tranquilize", + "tranquillised": "tranquilized", + "tranquilliser": "tranquilizer", + "tranquillisers": "tranquilizers", + "tranquillises": "tranquilizes", + "tranquillising": "tranquilizing", + "tranquillity": "tranquility", + "tranquillize": "tranquilize", + "tranquillized": "tranquilized", + "tranquillizer": "tranquilizer", + "tranquillizers": "tranquilizers", + "tranquillizes": "tranquilizes", + "tranquillizing": "tranquilizing", + "tranquilly": "tranquility", + "transistorised": "transistorized", + "traumatise": "traumatize", + "traumatised": "traumatized", + "traumatises": "traumatizes", + "traumatising": "traumatizing", + "travelled": "traveled", + "traveller": "traveler", + "travellers": "travelers", + "travelling": "traveling", + "travelogue": "travelog", + "travelogues": "travelogs", + "trialled": "trialed", + "trialling": "trialing", + "tricolour": "tricolor", + "tricolours": "tricolors", + "trivialise": "trivialize", + "trivialised": "trivialized", + "trivialises": "trivializes", + "trivialising": "trivializing", + "tumour": "tumor", + "tumours": "tumors", + "tunnelled": "tunneled", + "tunnelling": "tunneling", + "tyrannise": "tyrannize", + "tyrannised": "tyrannized", + "tyrannises": "tyrannizes", + "tyrannising": "tyrannizing", + "tyre": "tire", + "tyres": "tires", + "unauthorised": "unauthorized", + "uncivilised": "uncivilized", + "underutilised": "underutilized", + "unequalled": "unequaled", + "unfavourable": "unfavorable", + "unfavourably": "unfavorably", + "unionisation": "unionization", + "unionise": "unionize", + "unionised": "unionized", + "unionises": "unionizes", + "unionising": "unionizing", + "unorganised": "unorganized", + "unravelled": "unraveled", + "unravelling": "unraveling", + "unrecognisable": "unrecognizable", + "unrecognised": "unrecognized", + "unrivalled": "unrivaled", + "unsavoury": "unsavory", + "untrammelled": "untrammeled", + "urbanisation": "urbanization", + "urbanise": "urbanize", + "urbanised": "urbanized", + "urbanises": "urbanizes", + "urbanising": "urbanizing", + "utilisable": "utilizable", + "utilisation": "utilization", + "utilise": "utilize", + "utilised": "utilized", + "utilises": "utilizes", + "utilising": "utilizing", + "valour": "valor", + "vandalise": "vandalize", + "vandalised": "vandalized", + "vandalises": "vandalizes", + "vandalising": "vandalizing", + "vaporisation": "vaporization", + "vaporise": "vaporize", + "vaporised": "vaporized", + "vaporises": "vaporizes", + "vaporising": "vaporizing", + "vapour": "vapor", + "vapours": "vapors", + "verbalise": "verbalize", + "verbalised": "verbalized", + "verbalises": "verbalizes", + "verbalising": "verbalizing", + "victimisation": "victimization", + "victimise": "victimize", + "victimised": "victimized", + "victimises": "victimizes", + "victimising": "victimizing", + "videodisc": "videodisk", + "videodiscs": "videodisks", + "vigour": "vigor", + "visualisation": "visualization", + "visualisations": "visualizations", + "visualise": "visualize", + "visualised": "visualized", + "visualises": "visualizes", + "visualising": "visualizing", + "vocalisation": "vocalization", + "vocalisations": "vocalizations", + "vocalise": "vocalize", + "vocalised": "vocalized", + "vocalises": "vocalizes", + "vocalising": "vocalizing", + "vulcanised": "vulcanized", + "vulgarisation": "vulgarization", + "vulgarise": "vulgarize", + "vulgarised": "vulgarized", + "vulgarises": "vulgarizes", + "vulgarising": "vulgarizing", + "waggon": "wagon", + "waggons": "wagons", + "watercolour": "watercolor", + "watercolours": "watercolors", + "weaselled": "weaseled", + "weaselling": "weaseling", + "westernisation": "westernization", + "westernise": "westernize", + "westernised": "westernized", + "westernises": "westernizes", + "westernising": "westernizing", + "womanise": "womanize", + "womanised": "womanized", + "womaniser": "womanizer", + "womanisers": "womanizers", + "womanises": "womanizes", + "womanising": "womanizing", + "woollen": "woolen", + "woollens": "woolens", + "woollies": "woolies", + "woolly": "wooly", + "worshipped": "worshiped", + "worshipper": "worshiper", + "worshipping": "worshiping", + "yodelled": "yodeled", + "yodelling": "yodeling", + "yoghourt": "yogurt", + "yoghourts": "yogurts", + "yoghurt": "yogurt", + "yoghurts": "yogurts" +} + + +english_name_normalizer = { + # ── Double-letter variants ────────────────────────────────────────────── + "alan": "allen", + "allan": "allen", + "bridgette": "bridget", + "charly": "charlie", + "charley": "charlie", + "garry": "gary", + "gregg": "greg", + "jacky": "jackie", + "joann": "joanne", + "joane": "joanne", + "kellye": "kelly", + "kelli": "kelly", + "kelley": "kelly", + "lilly": "lily", + "micheal": "michael", + "michele": "michelle", + "mollie": "molly", + "phillip": "philip", + "sallie": "sally", + "stacey": "stacy", + "stacie": "stacy", + "tracey": "tracy", + "tracie": "tracy", + "bret": "brett", + "carrol": "carol", + "carole": "carol", + "carroll": "carol", + "allison": "alison", + "alyson": "alison", + "russel": "russell", + "douglass": "douglas", + "dominick": "dominic", + "robb": "rob", + # ── Chr/Kr variants ───────────────────────────────────────────────────── + "kris": "chris", + "kristopher": "christopher", + "cristopher": "christopher", + "kristina": "christina", + "kristen": "kristin", + # ── C/K variants ──────────────────────────────────────────────────────── + "karl": "carl", + "kathy": "cathy", + "katherine": "catherine", + "kathryn": "catherine", + "catharine": "catherine", + "erik": "eric", + "erick": "eric", + "caren": "karen", + "caryn": "karen", + "karin": "karen", + "katelyn": "caitlin", + "kaitlyn": "caitlin", + "kaitlin": "caitlin", + "nikole": "nicole", + "veronika": "veronica", + "viktor": "victor", + "viktoria": "victoria", + "kevan": "kevin", + "patrik": "patrick", + "frederik": "frederick", + "fredrick": "frederick", + "lukas": "lucas", + # ── Silent letters / alternate spellings ─────────────────────────────── + "ann": "anne", + "jon": "john", + "johnathan": "jonathan", + "jonathon": "jonathan", + "sara": "sarah", + "mathew": "matthew", + "nicolas": "nicholas", + "rachael": "rachel", + "rebekah": "rebecca", + "devorah": "deborah", + "theresa": "teresa", + "suzanne": "susanne", + "antony": "anthony", + "martyn": "martin", + "denis": "dennis", + "laurence": "lawrence", + "tomas": "thomas", + "tobey": "toby", + # ── Mac/Mc extensions ─────────────────────────────────────────────────── + "macarthur": "mcarthur", + "macartney": "mccartney", + "macarthy": "mccarthy", + "maccarthy": "mccarthy", + "macdonald": "mcdonald", + "mackay": "mckay", + "mackenzie": "mckenzie", + "macleod": "mcleod", + "maclean": "mclean", + "macmillan": "mcmillan", + "macintosh": "mcintosh", + "macintyre": "mcintyre", + "macnamara": "mcnamara", + "macgowan": "mcgowan", + # ── International ───────────────────────────────── + "mohamad": "mohammed", + "mohamed": "mohammed", + "mohammad": "mohammed", + "muhammad": "mohammed", + "muhamad": "mohammed", + "muhammed": "mohammed", + "mouhamed": "mohammed", + "mouhamad": "mohammed", + "mahomet": "mohammed", + "fatimah": "fatima", + "yusuf": "yousef", + "yusef": "yousef", + "myriam": "miriam", + "rajeev": "rajiv", + # ── Miscellaneous homophones ──────────────────────────────────────────── + "alphonso": "alfonso", + "bryan": "brian", + "geoffrey": "jeffrey", + "jeffery": "jeffrey", + "geoff": "jeff", + "neal": "neil", + "shaun": "sean", + "shawn": "sean", + "shayne": "shane", + "stephen": "steven", + "toni": "tony", + "leigh": "lee", + "lewis": "louis", + "marc": "mark", + "meghan": "megan", + "nathalie": "natalie", + "robyn": "robin", + "rodger": "roger", + "linsey": "lindsay", + "lindsey": "lindsay", + "zackary": "zachary", + "zachery": "zachary", + "zak": "zach", + "sheri": "sherry", + "cheri": "sherry", + "sherrie": "sherry", + "terri": "terry", + "lori": "laurie", + "jaime": "jamie", + "jayson": "jason", + "lesley": "leslie", + "lynda": "linda", + "lynne": "lynn", + "gayle": "gail", + "rhonda": "ronda", + "yvonne": "ivonne", + "stewart": "stuart", + "walther": "walter", + "symon": "simon", + "collin": "colin", + "dillon": "dylan", + "aron": "aaron", + "artur": "arthur", + "henri": "henry", + "josef": "joseph", + "pieter": "peter", +} + + +# Regex-based multi-word → single-token mappings, plus spellings that vary too +# freely to enumerate in english_spelling_normalizer (e.g. elongations). +# Applied after symbol removal, so hyphens/punctuation are already stripped. +# Keys are used with re.sub; values are the replacement strings. +english_compound_normalizer = { + r"\bet\s+cetera\b": "etc", + r"\bal\s+right\b": "alright", + r"\ball\s+right\b": "alright", + r"\bhow\s+ever\b": "however", + r"\bwi\s+fi\b": "wifi", + r"\bhi\s+fi\b": "hifi", + r"\blo\s+fi\b": "lofi", + r"\bsci\s+fi\b": "scifi", + r"\be\s+mail\b": "email", + r"\be\s+book\b": "ebook", + r"\be\s+commerce\b": "ecommerce", + r"\bx\s+ray\b": "xray", + r"\bt\s+shirt\b": "tshirt", + r"\ba\s+m\b": "am", + r"\bp\s+m\b": "pm", + r"\bo\s+k\b": "okay", + r"\bo+h+\b": "oh", + r"\booo+\b": "oh", +} diff --git a/packages/pipette-scores/pipette_scores/scoring/asr/open_asr_normalizer/multilingual.py b/packages/pipette-scores/pipette_scores/scoring/asr/open_asr_normalizer/multilingual.py new file mode 100644 index 0000000..5097752 --- /dev/null +++ b/packages/pipette-scores/pipette_scores/scoring/asr/open_asr_normalizer/multilingual.py @@ -0,0 +1,132 @@ +# From huggingface/open_asr_leaderboard, licensed under the Apache License, +# Version 2.0 (the repository's NOTICE and THIRD_PARTY_LICENSES.md). +# +# The Open ASR Leaderboard's multilingual normalization and compound alignment, +# excerpted unchanged from huggingface/open_asr_leaderboard at immutable revision +# 62a457c5c32d0de4df77e9470308aae7d45b055f (see ../NOTICE.md): +# +# - normalizer/data_utils.py lines 28-94: FILLER_WORDS and MultilingualNormalizer; +# - normalizer/eval_utils.py lines 68-99: normalize_compound_pairs. +# +# Only this header, the imports and the stand-in below are pipette's. data_utils.py +# is the leaderboard's data loader as well (it imports `datasets`, `huggingface_hub` +# and its own `normalizer` package), so it is not vendored whole. +# MultilingualNormalizer dispatches Armenian ("hy") to the leaderboard's +# ArmenianTextNormalizer (normalizer/armenian_normalizer.py), which is not +# vendored: pipette evaluates no Armenian, and the stand-in below refuses to +# normalize it rather than guess. + +import re +from difflib import SequenceMatcher + +import num2words + +from pipette_scores.scoring.asr.open_asr_normalizer.normalizer import BasicMultilingualTextNormalizer + + +class ArmenianTextNormalizer: + """Stand-in for the leaderboard's Armenian normalizer, which is not vendored.""" + + def __call__(self, s): + raise ValueError("the Open ASR Leaderboard's Armenian normalizer is not vendored") + + +# Language-specific filler words / hesitations, removed when the matching +# lang= is passed to the multilingual normalizer. Multi-word entries are +# supported (matched on whitespace boundaries). Currently empty +FILLER_WORDS = {} + + +class MultilingualNormalizer(BasicMultilingualTextNormalizer): + """BasicMultilingualTextNormalizer with optional number normalization. + + Call with just text for standard normalization (backward-compatible). + Pass lang= to also convert digits to words via num2words and remove + language-specific filler words (see FILLER_WORDS). + """ + + def __init__(self, remove_diacritics: bool = True): + super().__init__(remove_diacritics) + self._language_normalizers = {"hy": ArmenianTextNormalizer()} + # Pre-compile filler patterns. Each filler word is passed through the + # base normalization itself, so the pattern matches the normalized + # text exactly (base normalization may strip punctuation such as "…" + # or combining marks). Longest-first so that multi-word and longer + # variants match before their prefixes. Matched on whitespace + # boundaries ((? "10000") + text = re.sub(r"(\d)\s+(\d{3})\b", r"\1\2", text) + + # Convert remaining digit sequences to words + def _replace(m): + try: + return num2words.num2words(int(m.group()), lang=lang) + except Exception: + return m.group() + + return re.sub(r"\d+", _replace, text) + + def __call__(self, s, lang=None): + language_normalizer = self._language_normalizers.get(lang) + if language_normalizer is not None: + return language_normalizer(s) + + s = super().__call__(s) + if lang is not None: + s = self._remove_fillers(s, lang) + s = self._normalize_numbers(s, lang) + return s + + + +def normalize_compound_pairs(refs, preds): + """Align compound word boundaries between ref/pred pairs. + + When a mismatch region has identical characters ignoring whitespace, + normalize both sides to the joined form. + """ + new_refs, new_preds = [], [] + for ref_text, pred_text in zip(refs, preds): + ref_words = ref_text.split() + pred_words = pred_text.split() + + sm = SequenceMatcher(None, ref_words, pred_words) + new_rw, new_pw = [], [] + + for tag, i1, i2, j1, j2 in sm.get_opcodes(): + if tag == "equal": + new_rw.extend(ref_words[i1:i2]) + new_pw.extend(pred_words[j1:j2]) + else: + rc = "".join(ref_words[i1:i2]) + pc = "".join(pred_words[j1:j2]) + if rc == pc: + new_rw.append(rc) + new_pw.append(pc) + else: + new_rw.extend(ref_words[i1:i2]) + new_pw.extend(pred_words[j1:j2]) + + new_refs.append(" ".join(new_rw)) + new_preds.append(" ".join(new_pw)) + return new_refs, new_preds diff --git a/packages/pipette-scores/pipette_scores/scoring/asr/open_asr_normalizer/normalizer.py b/packages/pipette-scores/pipette_scores/scoring/asr/open_asr_normalizer/normalizer.py new file mode 100644 index 0000000..c6f4fa0 --- /dev/null +++ b/packages/pipette-scores/pipette_scores/scoring/asr/open_asr_normalizer/normalizer.py @@ -0,0 +1,753 @@ +# Copyright 2022 The OpenAI team and The HuggingFace Team. All rights reserved. +# Most of the code is copy pasted from the original whisper repository +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +import unicodedata +from fractions import Fraction +from typing import Iterator, List, Match, Optional, Union +from .english_abbreviations import english_name_normalizer, english_spelling_normalizer, english_compound_normalizer + +import regex + + +# non-ASCII letters that are not separated by "NFKD" normalization +ADDITIONAL_DIACRITICS = { + "œ": "oe", + "Œ": "OE", + "ø": "o", + "Ø": "O", + "æ": "ae", + "Æ": "AE", + "ß": "ss", + "ẞ": "SS", + "đ": "d", + "Đ": "D", + "ð": "d", + "Ð": "D", + "þ": "th", + "Þ": "th", + "ł": "l", + "Ł": "L", +} + + +def remove_symbols_and_diacritics(s: str, keep=""): + """ + Replace any other markers, symbols, and punctuations with a space, and drop any diacritics (category 'Mn' and some + manual mappings) + """ + + def replace_character(char): + if char in keep: + return char + elif char in ADDITIONAL_DIACRITICS: + return ADDITIONAL_DIACRITICS[char] + + elif unicodedata.category(char) == "Mn": + return "" + + elif unicodedata.category(char)[0] in "MSP": + return " " + + return char + + return "".join(replace_character(c) for c in unicodedata.normalize("NFKD", s)) + + +def remove_symbols(s: str): + """ + Replace any other markers, symbols, punctuations with a space, keeping diacritics + """ + return "".join(" " if unicodedata.category(c)[0] in "MSP" else c for c in unicodedata.normalize("NFKC", s)) + + +def remove_symbols_keep_marks(s: str): + """ + Replace symbols and punctuation with a space, keeping combining marks. + + Unlike `remove_symbols`, combining marks (category 'M') are preserved. This + is required for scripts like Devanagari, where vowel signs (matras) and the + virama are combining marks that are integral to words. + """ + return "".join(" " if unicodedata.category(c)[0] in "SP" else c for c in unicodedata.normalize("NFKC", s)) + + +class BasicTextNormalizer: + def __init__(self, remove_diacritics: bool = False, split_letters: bool = False): + self.clean = remove_symbols_and_diacritics if remove_diacritics else remove_symbols + self.split_letters = split_letters + + def __call__(self, s: str): + s = s.lower() + s = re.sub(r"[<\[][^>\]]*[>\]]", "", s) # remove words between brackets + s = re.sub(r"\(([^)]+?)\)", "", s) # remove words between parenthesis + s = self.clean(s).lower() + + if self.split_letters: + s = " ".join(regex.findall(r"\X", s, regex.U)) + + s = re.sub(r"\s+", " ", s) # replace any successive whitespace characters with a space + + return s + + +class BasicMultilingualTextNormalizer: + def __init__(self, remove_diacritics: bool = True): + # When keeping diacritics, also keep combining marks (category 'M'): + # scripts like Devanagari encode vowel signs and the virama as + # combining marks, so stripping them mangles words. + self.clean = remove_symbols_and_diacritics if remove_diacritics else remove_symbols_keep_marks + + def __call__(self, s: str): + s = s.lower() + s = re.sub(r"[<\[][^>\]]*[>\]]", "", s) # remove words between brackets + s = re.sub(r"\(([^)]+?)\)", "", s) # remove words between parenthesis + s = self.clean(s).lower() + + # Remove punctuation and extra spaces + s = regex.sub(r"[^\w\s]", "", s) + s = re.sub(r"\s+", " ", s).strip() + + return s + + +class EnglishNumberNormalizer: + """ + Convert any spelled-out numbers into arabic numbers, while handling: + + - remove any commas + - keep the suffixes such as: `1960s`, `274th`, `32nd`, etc. + - spell out currency symbols after the number. e.g. `$20 million` -> `20000000 dollars` + - spell out `one` and `ones` + - interpret successive single-digit numbers as nominal: `one oh one` -> `101` + """ + + def __init__(self): + super().__init__() + + self.zeros = {"o", "oh", "zero"} + # fmt: off + self.ones = { + name: i + for i, name in enumerate( + ["one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "ten", "eleven", "twelve", "thirteen", "fourteen", "fifteen", "sixteen", "seventeen", "eighteen", "nineteen"], + start=1, + ) + } + # fmt: on + self.ones_plural = { + "sixes" if name == "six" else name + "s": (value, "s") for name, value in self.ones.items() + } + self.ones_ordinal = { + "zeroth": (0, "th"), + "first": (1, "st"), + "second": (2, "nd"), + "third": (3, "rd"), + "fifth": (5, "th"), + "twelfth": (12, "th"), + **{ + name + ("h" if name.endswith("t") else "th"): (value, "th") + for name, value in self.ones.items() + if value > 3 and value != 5 and value != 12 + }, + } + self.ones_suffixed = {**self.ones_plural, **self.ones_ordinal} + + self.tens = { + "twenty": 20, + "thirty": 30, + "forty": 40, + "fifty": 50, + "sixty": 60, + "seventy": 70, + "eighty": 80, + "ninety": 90, + } + self.tens_plural = {name.replace("y", "ies"): (value, "s") for name, value in self.tens.items()} + self.tens_ordinal = {name.replace("y", "ieth"): (value, "th") for name, value in self.tens.items()} + self.tens_suffixed = {**self.tens_plural, **self.tens_ordinal} + + self.multipliers = { + "hundred": 100, + "thousand": 1_000, + "million": 1_000_000, + "billion": 1_000_000_000, + "trillion": 1_000_000_000_000, + "quadrillion": 1_000_000_000_000_000, + "quintillion": 1_000_000_000_000_000_000, + "sextillion": 1_000_000_000_000_000_000_000, + "septillion": 1_000_000_000_000_000_000_000_000, + "octillion": 1_000_000_000_000_000_000_000_000_000, + "nonillion": 1_000_000_000_000_000_000_000_000_000_000, + "decillion": 1_000_000_000_000_000_000_000_000_000_000_000, + } + self.multipliers_plural = {name + "s": (value, "s") for name, value in self.multipliers.items()} + self.multipliers_ordinal = {name + "th": (value, "th") for name, value in self.multipliers.items()} + self.multipliers_suffixed = {**self.multipliers_plural, **self.multipliers_ordinal} + self.decimals = {*self.ones, *self.tens, *self.zeros} + + self.preceding_prefixers = { + "minus": "-", + "negative": "-", + "plus": "+", + "positive": "+", + } + self.following_prefixers = { + "pound": "£", + "pounds": "£", + "euro": "€", + "euros": "€", + "dollar": "$", + "dollars": "$", + "cent": "¢", + "cents": "¢", + } + self.prefixes = set(list(self.preceding_prefixers.values()) + list(self.following_prefixers.values())) + self.suffixers = { + "per": {"cent": "%"}, + "percent": "%", + } + self.specials = {"and", "double", "triple", "point"} + + self.words = { + key + for mapping in [ + self.zeros, + self.ones, + self.ones_suffixed, + self.tens, + self.tens_suffixed, + self.multipliers, + self.multipliers_suffixed, + self.preceding_prefixers, + self.following_prefixers, + self.suffixers, + self.specials, + ] + for key in mapping + } + self.literal_words = {"one", "ones"} + + def process_words(self, words: List[str]) -> Iterator[str]: + prefix: Optional[str] = None + value: Optional[Union[str, int]] = None + skip = False + + def to_fraction(s: str): + try: + return Fraction(s) + except ValueError: + return None + + def is_digit_token(token: Optional[str]) -> bool: + """True for tokens that continue a digit sequence ("four", "oh", "20").""" + return token is not None and bool( + re.match(r"^\d+$", token) + or token in self.zeros + or token in self.ones + or token in self.tens + ) + + def output(result: Union[str, int]): + nonlocal prefix, value + result = str(result) + if prefix is not None: + result = prefix + result + value = None + prefix = None + return result + + if len(words) == 0: + return + + for i, current in enumerate(words): + prev = words[i - 1] if i != 0 else None + next = words[i + 1] if i != len(words) - 1 else None + if skip: + skip = False + continue + + next_is_numeric = next is not None and re.match(r"^\d+(\.\d+)?$", next) + has_prefix = current[0] in self.prefixes + current_without_prefix = current[1:] if has_prefix else current + if re.match(r"^\d+(\.\d+)?$", current_without_prefix): + # arabic numbers (potentially with signs and fractions) + f = to_fraction(current_without_prefix) + if f is None: + raise ValueError("Converting the fraction failed") + + if value is not None: + if isinstance(value, str) and value.endswith("."): + # concatenate decimals / ip address components + value = str(value) + str(current) + continue + else: + yield output(value) + + prefix = current[0] if has_prefix else prefix + if f.denominator == 1: + value = f.numerator # store integers as int + else: + value = current_without_prefix + elif current not in self.words: + # non-numeric words + if value is not None: + yield output(value) + yield output(current) + elif current in self.zeros: + # "oh" is far more often the interjection than a spoken zero, so + # read it as a digit only inside a digit sequence: the sequence + # has to continue on the right ("four oh one", "nineteen oh + # five"), and with nothing pending on the left it takes two more + # digit tokens, i.e. a serial/phone-style reading ("oh seven nine + # eight"). On its own — including the doubled "oh oh" — it stays + # a word. "o" and "zero" are unchanged. + next2 = words[i + 2] if i + 2 < len(words) else None + in_number = is_digit_token(next) and ( + value is not None or is_digit_token(next2) + ) + if current == "oh" and not in_number: + if value is not None: + yield output(value) # don't drop a pending number + yield output(current) + else: + value = str(value or "") + "0" + elif current in self.ones: + ones = self.ones[current] + + if value is None: + value = ones + elif isinstance(value, str) or prev in self.ones: + if prev in self.tens and ones < 10: # replace the last zero with the digit + value = value[:-1] + str(ones) + else: + value = str(value) + str(ones) + elif ones < 10: + if value % 10 == 0: + value += ones + else: + value = str(value) + str(ones) + else: # eleven to nineteen + if value % 100 == 0: + value += ones + else: + value = str(value) + str(ones) + elif current in self.ones_suffixed: + # ordinal or cardinal; yield the number right away + ones, suffix = self.ones_suffixed[current] + if value is None: + yield output(str(ones) + suffix) + elif isinstance(value, str) or prev in self.ones: + if prev in self.tens and ones < 10: + yield output(value[:-1] + str(ones) + suffix) + else: + yield output(str(value) + str(ones) + suffix) + elif ones < 10: + if value % 10 == 0: + yield output(str(value + ones) + suffix) + else: + yield output(str(value) + str(ones) + suffix) + else: # eleven to nineteen + if value % 100 == 0: + yield output(str(value + ones) + suffix) + else: + yield output(str(value) + str(ones) + suffix) + value = None + elif current in self.tens: + tens = self.tens[current] + if value is None: + value = tens + elif isinstance(value, str): + value = str(value) + str(tens) + else: + if value % 100 == 0: + value += tens + else: + value = str(value) + str(tens) + elif current in self.tens_suffixed: + # ordinal or cardinal; yield the number right away + tens, suffix = self.tens_suffixed[current] + if value is None: + yield output(str(tens) + suffix) + elif isinstance(value, str): + yield output(str(value) + str(tens) + suffix) + else: + if value % 100 == 0: + yield output(str(value + tens) + suffix) + else: + yield output(str(value) + str(tens) + suffix) + elif current in self.multipliers: + multiplier = self.multipliers[current] + if value is None: + value = multiplier + elif isinstance(value, str) or value == 0: + f = to_fraction(value) + p = f * multiplier if f is not None else None + if f is not None and p.denominator == 1: + value = p.numerator + else: + yield output(value) + value = multiplier + else: + before = value // 1000 * 1000 + residual = value % 1000 + value = before + residual * multiplier + elif current in self.multipliers_suffixed: + multiplier, suffix = self.multipliers_suffixed[current] + if value is None: + yield output(str(multiplier) + suffix) + elif isinstance(value, str): + f = to_fraction(value) + p = f * multiplier if f is not None else None + if f is not None and p.denominator == 1: + yield output(str(p.numerator) + suffix) + else: + yield output(value) + yield output(str(multiplier) + suffix) + else: # int + before = value // 1000 * 1000 + residual = value % 1000 + value = before + residual * multiplier + yield output(str(value) + suffix) + value = None + elif current in self.preceding_prefixers: + # apply prefix (positive, minus, etc.) if it precedes a number + if value is not None: + yield output(value) + + if next in self.words or next_is_numeric: + prefix = self.preceding_prefixers[current] + else: + yield output(current) + elif current in self.following_prefixers: + # apply prefix (dollars, cents, etc.) only after a number + if value is not None: + prefix = self.following_prefixers[current] + yield output(value) + else: + yield output(current) + elif current in self.suffixers: + # apply suffix symbols (percent -> '%') + if value is not None: + suffix = self.suffixers[current] + if isinstance(suffix, dict): + if next in suffix: + yield output(str(value) + suffix[next]) + skip = True + else: + yield output(value) + yield output(current) + else: + yield output(str(value) + suffix) + else: + yield output(current) + elif current in self.specials: + if next not in self.words and not next_is_numeric: + # apply special handling only if the next word can be numeric + if value is not None: + yield output(value) + yield output(current) + elif current == "and": + # ignore "and" after hundreds, thousands, etc. + if prev not in self.multipliers: + if value is not None: + yield output(value) + yield output(current) + elif current == "double" or current == "triple": + if next in self.ones or next in self.zeros: + repeats = 2 if current == "double" else 3 + ones = self.ones.get(next, 0) + value = str(value or "") + str(ones) * repeats + skip = True + else: + if value is not None: + yield output(value) + yield output(current) + elif current == "point": + if next in self.decimals or next_is_numeric: + value = str(value or "") + "." + else: + # should all have been covered at this point + raise ValueError(f"Unexpected token: {current}") + else: + # all should have been covered at this point + raise ValueError(f"Unexpected token: {current}") + + if value is not None: + yield output(value) + + def preprocess(self, s: str): + # replace " and a half" with " point five" + results = [] + + segments = re.split(r"\band\s+a\s+half\b", s) + for i, segment in enumerate(segments): + if len(segment.strip()) == 0: + continue + if i == len(segments) - 1: + results.append(segment) + else: + results.append(segment) + last_word = segment.rsplit(maxsplit=2)[-1] + if last_word in self.decimals or last_word in self.multipliers: + results.append("point five") + else: + results.append("and a half") + + s = " ".join(results) + + # put a space at number/letter boundary + s = re.sub(r"([a-z])([0-9])", r"\1 \2", s) + s = re.sub(r"([0-9])([a-z])", r"\1 \2", s) + + # but remove spaces which could be a suffix + s = re.sub(r"([0-9])\s+(st|nd|rd|th|s)\b", r"\1\2", s) + + return s + + def postprocess(self, s: str): + def combine_cents(m: Match): + try: + currency = m.group(1) + integer = m.group(2) + cents = int(m.group(3)) + return f"{currency}{integer}.{cents:02d}" + except ValueError: + return m.string + + def extract_cents(m: Match): + try: + return f"¢{int(m.group(1))}" + except ValueError: + return m.string + + # apply currency postprocessing; "$2 and ¢7" -> "$2.07" + s = re.sub(r"([€£$])([0-9]+) (?:and )?¢([0-9]{1,2})\b", combine_cents, s) + s = re.sub(r"[€£$]0.([0-9]{1,2})\b", extract_cents, s) + + # write "one(s)" instead of "1(s)", just for the readability + s = re.sub(r"\b1(s?)\b", r"one\1", s) + + return s + + def __call__(self, s: str): + s = self.preprocess(s) + s = " ".join(word for word in self.process_words(s.split()) if word is not None) + s = self.postprocess(s) + + return s + + +class EnglishSpellingNormalizer: + """ + Applies British-American spelling mappings as listed in [1]. + + [1] https://www.tysto.com/uk-us-spelling-list.html + """ + + def __init__(self, english_spelling_mapping): + self.mapping = english_spelling_mapping + + def __call__(self, s: str): + return " ".join(self.mapping.get(word, word) for word in s.split()) + + +class EnglishAcronymNormalizer: + """ + Collapse sequences of single-character tokens (letters or digits) into single words. + + This normalizes acronym spacing so that both spaced-out and joined forms match: + - "b b c" -> "bbc" + - "5 g" -> "5g" + + Lone single-character words surrounded by multi-character words are left untouched + (e.g. "a big cat" stays "a big cat"). + """ + + def __call__(self, s: str) -> str: + words = s.split() + result = [] + i = 0 + while i < len(words): + if len(words[i]) == 1 and words[i].isalnum(): + # Start of a potential acronym run + run = [words[i]] + j = i + 1 + while j < len(words) and len(words[j]) == 1 and words[j].isalnum(): + run.append(words[j]) + j += 1 + # Require 3+ tokens if the run contains common words "a" or "i", + # otherwise 2+ is enough (e.g. "5 g" -> "5g") + has_common_word = any(c in ("a", "i") for c in run) + min_run = 3 if has_common_word else 2 + if len(run) >= min_run: + result.append("".join(run)) + else: + result.extend(run) + i = j + else: + result.append(words[i]) + i += 1 + return " ".join(result) + + +class EnglishNameNormalizer: + """ + Collapse common name spelling variants to a single canonical form. + + This is intentionally conservative and token-based so it can be extended + with project-specific aliases when needed. + """ + + def __init__(self, english_name_mapping=english_name_normalizer): + self.mapping = english_name_mapping + + def __call__(self, s: str): + return " ".join(self.mapping.get(word, word) for word in s.split()) + + +class EnglishTextNormalizer: + def __init__(self, english_spelling_mapping=english_spelling_normalizer): + # Filler words / hesitations to remove. Written as regexes so that + # arbitrary elongation is covered without enumerating every spelling + # ("uh", "uhh", "uuuh", "uhhhh", ...). Each alternative is wrapped in + # \b...\b below, so a shorter alternative cannot match a prefix of a + # longer token and the order of the single-token patterns is irrelevant. + # Hyphens are word boundaries too, which is why most hyphenated forms + # need no entry ("um-hmm" is matched as "um" + "hmm") — but any whose + # halves are not both fillers must be listed *before* the patterns, + # otherwise only the first half is matched ("ah-ha" -> "ha"). + filler_words = [ + "ah-ha", # "ha" alone is not a filler, so match the pair first + r"a+h+m*", # ah, aah, ahh, ahhh, aaah, ahm, ahmm + r"a+h+a+", # aha, ahaa, ahaaa + r"e+h+m*", # eh, ehh, eeeh, ehhh, ehm, ehmm + r"e+m+", # em, emm + r"e+r+m*", # er, err, errr, erm + r"h+a+h+", # hah, hahh + r"h+e+h+", # heh, hehh + r"h+m+", # hm, hmm, hmmm, hhm + r"h+u+h+", # huh, huhh + r"m{2,}", # mm, mmm, mmmm + r"m+h+m*", # mh, mhm, mhmm, mmhm + r"t+s+k+", # tsk + r"u+g+h+", # ugh, uuugh + r"u+h+m*", # uh, uuh, uhh, uhhh, uuuh, uhm, uuuhm + r"u+h+u+[hm]*", # uhuh, uhum + r"u+m+h*", # um, umm, ummm, uuum, umh + # Irregular forms, not worth a pattern of their own. + "ahem", "eheh", "ehehe", "ehr", "hmmph", "hum", "hunh", "mhum", "mmkay", + ] + self.ignore_patterns = r"\b(" + "|".join(filler_words) + r")\b" + self.replacers = { + # Bare o'clock times: the ":00" is not spoken as words, so drop it + # ("2:00 AM" -> "2 am"). Applied here, while the colon is still + # present, so that a time is distinguishable from an unrelated + # digit sequence — by the time symbols are stripped "3:00" and + # "3 00" look alike. Without this the minutes are absorbed into the + # hour ("3:00" -> "30") or left as a stray token ("11:00" -> "11 0"). + r"\b(\d{1,2}):00\b": r"\1", + # common contractions + r"\bwon't\b": "will not", + r"\bcan't\b": "can not", + r"\blet's\b": "let us", + r"\bain't\b": "aint", + r"\by'all\b": "you all", + r"\bwanna\b": "want to", + r"\bgotta\b": "got to", + r"\bgonna\b": "going to", + r"\bi'ma\b": "i am going to", + r"\bimma\b": "i am going to", + r"\bwoulda\b": "would have", + r"\bcoulda\b": "could have", + r"\bshoulda\b": "should have", + r"\bma'am\b": "madam", + # contractions in titles/prefixes + r"\bmr\b": "mister ", + r"\bmrs\b": "missus ", + r"\bst\b": "saint ", + r"\bdr\b": "doctor ", + r"\bprof\b": "professor ", + r"\bcapt\b": "captain ", + r"\bgov\b": "governor ", + r"\bald\b": "alderman ", + r"\bgen\b": "general ", + r"\bsen\b": "senator ", + r"\brep\b": "representative ", + r"\bpres\b": "president ", + r"\brev\b": "reverend ", + r"\bhon\b": "honorable ", + r"\basst\b": "assistant ", + r"\bassoc\b": "associate ", + r"\blt\b": "lieutenant ", + r"\bcol\b": "colonel ", + r"\bjr\b": "junior ", + r"\bsr\b": "senior ", + r"\besq\b": "esquire ", + # prefect tenses, ideally it should be any past participles, but it's harder.. + r"'d been\b": " had been", + r"'s been\b": " has been", + r"'d gone\b": " had gone", + r"'s gone\b": " has gone", + r"'d done\b": " had done", # "'s done" is ambiguous + r"'s got\b": " has got", + # general contractions + r"n't\b": " not", + r"'re\b": " are", + r"\b(it|he|she|what|that|who|here|there|how|when|where|why|this)'s\b": r"\1 is", + r"'d\b": " would", + r"'ll\b": " will", + r"'t\b": " not", + r"'ve\b": " have", + r"'m\b": " am", + } + self.standardize_numbers = EnglishNumberNormalizer() + self.standardize_spellings = EnglishSpellingNormalizer(english_spelling_mapping) + self.standardize_names = EnglishNameNormalizer() + self.standardize_acronyms = EnglishAcronymNormalizer() + # Multi-word compound mappings — defined in english_abbreviations.py + self.compound_words = english_compound_normalizer + + def __call__(self, s: str): + s = s.lower() + + s = re.sub(r"[<\[][^>\]]*[>\]]", "", s) # remove words between brackets + s = re.sub(r"\(([^)]+?)\)", "", s) # remove words between parenthesis + s = re.sub(self.ignore_patterns, "", s) + s = re.sub(r"\s+'", "'", s) # standardize when there's a space before an apostrophe + + for pattern, replacement in self.replacers.items(): + s = re.sub(pattern, replacement, s) + + s = re.sub(r"(\d),(\d)", r"\1\2", s) # remove commas between digits + s = re.sub(r"\.([^0-9]|$)", r" \1", s) # remove periods not followed by numbers + s = remove_symbols_and_diacritics(s, keep=".%$¢€£") # keep some symbols for numerics + + # Normalize hardcoded compound words (e.g. "wi fi" -> "wifi" after hyphen removal) + for pattern, replacement in self.compound_words.items(): + s = re.sub(pattern, replacement, s) + + s = self.standardize_numbers(s) + s = self.standardize_spellings(s) + s = self.standardize_names(s) + s = self.standardize_acronyms(s) + + # now remove prefix/suffix symbols that are not preceded/followed by numbers + s = re.sub(r"[.$¢€£]([^0-9])", r" \1", s) + s = re.sub(r"([^0-9])%", r"\1 ", s) + + s = re.sub(r"\s+", " ", s) # replace any successive whitespace characters with a space + + return s diff --git a/packages/pipette-scores/pipette_scores/scoring/asr/suite.py b/packages/pipette-scores/pipette_scores/scoring/asr/suite.py new file mode 100644 index 0000000..388dc77 --- /dev/null +++ b/packages/pipette-scores/pipette_scores/scoring/asr/suite.py @@ -0,0 +1,101 @@ +"""The ASR suite headlines over per-dataset `/score` results: the reference definition. + +The scores server scores one dataset per request and keeps no run state, so a +suite headline is its consumer's (pipette-mgmt), as no VLM aggregate is computed +here either. These functions are the definition that consumer implements +(docs/asr-evals.md §3, "Suite headlines"), tested in tests/test_asr_suite.py: + +- **Open ASR Leaderboard, English**: the equal-weight mean of its 9 public sets' + corpus WERs (a macro mean: never the edits summed across datasets, never + weighted by size), as the leaderboard's ``score_results`` averages its + "public" columns (huggingface/open_asr_leaderboard @ 62a457c5 + ``normalizer/eval_utils.py``). ``result`` is that mean of the unrounded rates. + The leaderboard averages the per-set WERs it has already rounded to two + decimals (``round(100 * wer, 2)``): its public CSV ``avg`` is that mean, + unrounded (``score_results``: "Unrounded, to match english_short_latest.csv"), + and its console "Composite Results" line shows the same mean to two decimals + (``:0.2f``). ``leaderboard_result`` is the console figure: the mean of the + rounded per-set WERs, rounded to two decimals. It differs from ``result`` by + at most 0.01 points (up to 0.005 from the per-set rounding, up to 0.005 from + the final one). +- **FLEURS**: the equal-weight mean of its 7 locales' corpus WERs (en_us, + es_419, pt_br, de_de, fr_fr, it_it, nl_nl), the same macro mean. + +Each input is one dataset's ``context`` from `/score`. A suite headline needs +every one of its datasets exactly once, each with a corpus rate (a non-null +``error_rate``; `/score` gives none to a partial run) and scored in words; +anything else raises rather than averaging what is there. +""" + +from __future__ import annotations + +from collections import Counter +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass +from typing import Any + +from pipette_scores.dataset_catalog.asr_datasets import FLEURS_EVAL_IDS, OPEN_ASR_EVAL_IDS +from pipette_scores.types import EvalId + + +@dataclass(frozen=True) +class SuiteMean: + """An equal-weight mean of per-dataset corpus rates.""" + + error_unit: str # "word" + eval_ids: tuple[EvalId, ...] + # The mean of the datasets' `error_rate` ratios, and that mean x 100. + error_rate: float + result: float + # The datasets' percent rates each rounded to 2 decimals, averaged, the mean + # rounded to 2 decimals: the figure the leaderboard's console prints (its + # CSV `avg` is the same mean unrounded). At most 0.01 from `result`. + leaderboard_result: float + + +def _rates( + results: Iterable[tuple[str, Mapping[str, Any]]], expected: Sequence[EvalId], suite: str +) -> dict[EvalId, float]: + """Each expected dataset's corpus rate, refusing anything else.""" + pairs = [(EvalId(eval_id), context) for eval_id, context in results] + counts = Counter(eval_id for eval_id, _ in pairs) + if duplicates := sorted(e.value for e, n in counts.items() if n > 1): + raise ValueError(f"{suite}: datasets given more than once: {', '.join(duplicates)}") + if missing := [e.value for e in expected if e not in counts]: + raise ValueError(f"{suite}: datasets missing: {', '.join(missing)}") + if extra := sorted(e.value for e in counts if e not in expected): + raise ValueError(f"{suite}: datasets not in the suite: {', '.join(extra)}") + rates: dict[EvalId, float] = {} + for eval_id, context in pairs: + if context.get("error_rate") is None: + raise ValueError(f"{suite}: {eval_id.value} has no corpus rate") + if context.get("error_unit") != "word": + raise ValueError(f"{suite}: {eval_id.value} is scored in {context.get('error_unit')!r}, not 'word'") + rates[eval_id] = float(context["error_rate"]) + return rates + + +def _equal_weight(rates: Mapping[EvalId, float], eval_ids: Sequence[EvalId]) -> SuiteMean: + mean = sum(rates[e] for e in eval_ids) / len(eval_ids) + rounded = [round(100 * rates[e], 2) for e in eval_ids] + return SuiteMean( + error_unit="word", + eval_ids=tuple(eval_ids), + error_rate=mean, + result=mean * 100, + leaderboard_result=round(sum(rounded) / len(rounded), 2), + ) + + +def open_asr_mean(results: Iterable[tuple[str, Mapping[str, Any]]]) -> SuiteMean: + """Open ASR Leaderboard, English: the equal-weight mean WER of its 9 public + sets. ``results`` are (eval_id, `/score` context) pairs, exactly one per set.""" + rates = _rates(results, OPEN_ASR_EVAL_IDS, "Open ASR Leaderboard") + return _equal_weight(rates, OPEN_ASR_EVAL_IDS) + + +def fleurs_mean(results: Iterable[tuple[str, Mapping[str, Any]]]) -> SuiteMean: + """FLEURS: the equal-weight mean WER of its 7 locales. ``results`` are + (eval_id, `/score` context) pairs, exactly one per locale.""" + rates = _rates(results, FLEURS_EVAL_IDS, "FLEURS") + return _equal_weight(rates, FLEURS_EVAL_IDS) diff --git a/packages/pipette-scores/pipette_scores/scoring/asr/wer.py b/packages/pipette-scores/pipette_scores/scoring/asr/wer.py new file mode 100644 index 0000000..41be74b --- /dev/null +++ b/packages/pipette-scores/pipette_scores/scoring/asr/wer.py @@ -0,0 +1,179 @@ +"""Corpus word error rates, as the Open ASR Leaderboard computes them. + +Per scored unit: normalize the reference and the hypothesis with the dataset's +recipe and align them. Per dataset: sum the edits over every unit and divide by +the summed reference units (corpus-level, as the leaderboard's +``eval_utils.align_manifests`` does, for its English and its multilingual +scoring alike), never the mean of the per-unit rates. Values are kept +unrounded; the leaderboard reports ``round(100 * wer, 2)`` per set. + +Two rules of the Open ASR Leaderboard (huggingface/open_asr_leaderboard +@ 62a457c5c32d0de4df77e9470308aae7d45b055f) apply to its English sets: + +- a sample whose normalized reference is empty, or reads "ignore time segment + in scoring", is not scored (``data_utils.prepare_data`` filters it with + ``is_target_text_in_range`` before inference; here it is served and its + generation required, then left out of the score); +- chunked long-form audio (Earnings22) is scored per session: the chunks' + stripped outputs, the empty ones skipped, joined with one space in + ``chunk_index`` order, against the session's reference + (``eval_utils.merge_chunked_manifest``). + +pipette's own rule: a result is an answer to every sample, or no result. Every +sample (excluded ones included) must have a successful generation; a missing +one is never scored as an empty transcript, and ``score_asr`` refuses the run +instead (a failed generation is refused before it, in evaluate.py). (The +leaderboard scores what its manifest holds and only warns about a missing +chunk.) +""" + +from __future__ import annotations + +import functools +from collections import defaultdict +from collections.abc import Mapping, Sequence +from dataclasses import dataclass + +from pipette_scores.scoring.asr.align import Aligner, Counts, align +from pipette_scores.scoring.asr.normalize import NormalizerRecipe, normalize + +# The normalized references `is_target_text_in_range` rejects. +_EXCLUDED_REFERENCES = frozenset({"", "ignore time segment in scoring"}) + + +@dataclass(frozen=True) +class AsrScoringSpec: + """How a dataset is scored; read from its metadata.json.""" + + normalizer: NormalizerRecipe + aligner: Aligner + # Open ASR Leaderboard: drop samples whose normalized reference is empty or the ignore marker. + open_asr_exclusions: bool = False + + @property + def error_unit(self) -> str: + # Every aligner counts words; `/score` still names the unit, so that a + # consumer checks it rather than assumes it. + return "word" + + +@dataclass(frozen=True) +class AsrReference: + """One sample's ground truth. A chunk of a long-form session names it.""" + + id: str + reference: str + # ISO 639-3. + language: str | None + session: str | None = None + chunk_index: int | None = None + # The session's chunk count (Earnings22 `parent_chunk_count`), when known. + session_size: int | None = None + + +@dataclass(frozen=True) +class ScoredUnit: + """A sample, or a whole session for chunked audio, as scored.""" + + unit_id: str + sample_ids: tuple[str, ...] + reference: str + hypothesis: str + counts: Counts + + +@dataclass(frozen=True) +class AsrScore: + units: tuple[ScoredUnit, ...] + excluded_sample_ids: tuple[str, ...] + error_unit: str + + @property + def totals(self) -> Counts: + """Summed over the scored units.""" + return Counts( + hits=sum(u.counts.hits for u in self.units), + substitutions=sum(u.counts.substitutions for u in self.units), + deletions=sum(u.counts.deletions for u in self.units), + insertions=sum(u.counts.insertions for u in self.units), + reference_units=sum(u.counts.reference_units for u in self.units), + ) + + @property + def corpus_error_rate(self) -> float | None: + """Summed edits over summed reference units, as a ratio (0.05 = 5%). + None when there is no reference unit to divide by (the leaderboard + would report 0.0, or infinity if there are errors).""" + totals = self.totals + if totals.reference_units == 0: + return None + return totals.errors / totals.reference_units + + +def score_asr( + spec: AsrScoringSpec, + references: Sequence[AsrReference], + hypotheses: Mapping[str, str], +) -> AsrScore: + """Score `hypotheses` (successful generations, by sample id) against `references`. + + Every reference needs its hypothesis: a run without an answer to each + sample is no result, so it raises ``ValueError`` rather than score the gap. + """ + if unanswered := [r.id for r in references if r.id not in hypotheses]: + raise ValueError( + f"an ASR eval is scored only with an answer to every sample; unanswered: {', '.join(unanswered[:10])}" + ) + + # Cached per call: every Earnings22 chunk carries its whole call's transcript. + @functools.cache + def norm(text: str, language: str | None) -> str: + return normalize(spec.normalizer, text, language) + + def norm_hypothesis(text: str, language: str | None) -> str: + # pipette's own: a hypothesis the normalizer raises on scores as empty + # rather than failing the whole dataset. + try: + return norm(text, language) + except ValueError: + return "" + + excluded: list[str] = [] + units: list[ScoredUnit] = [] + sessions: dict[str, list[AsrReference]] = defaultdict(list) + for ref in references: + if spec.open_asr_exclusions and norm(ref.reference, ref.language).strip() in _EXCLUDED_REFERENCES: + excluded.append(ref.id) + continue + if ref.session is not None: + sessions[ref.session].append(ref) + continue + reference = norm(ref.reference, ref.language) + hypothesis = norm_hypothesis(hypotheses[ref.id], ref.language) + units.append(ScoredUnit(ref.id, (ref.id,), reference, hypothesis, align(spec.aligner, reference, hypothesis))) + + for session in sorted(sessions): + chunks = sorted(sessions[session], key=lambda r: _chunk_index(r)) + indexes = [_chunk_index(r) for r in chunks] + if indexes != list(range(len(chunks))): + raise ValueError(f"session {session!r} has chunk indexes {indexes}, not 0..{len(chunks) - 1}") + sizes = {r.session_size for r in chunks if r.session_size is not None} + if sizes and sizes != {len(chunks)}: + raise ValueError(f"session {session!r} declares {sorted(sizes)} chunks, the dataset holds {len(chunks)}") + if len({r.reference for r in chunks}) != 1: + raise ValueError(f"session {session!r}: its chunks carry different references") + sample_ids = tuple(r.id for r in chunks) + texts = (hypotheses[i].strip() for i in sample_ids) + joined = " ".join(t for t in texts if t) + language = chunks[0].language + reference = norm(chunks[0].reference, language) + hypothesis = norm_hypothesis(joined, language) + units.append(ScoredUnit(session, sample_ids, reference, hypothesis, align(spec.aligner, reference, hypothesis))) + + return AsrScore(tuple(units), tuple(excluded), spec.error_unit) + + +def _chunk_index(ref: AsrReference) -> int: + if ref.chunk_index is None: + raise ValueError(f"sample {ref.id!r} names session {ref.session!r} but no chunk_index") + return ref.chunk_index diff --git a/packages/pipette-scores/pipette_scores/scoring/completion_text.py b/packages/pipette-scores/pipette_scores/scoring/completion_text.py index 3d5450b..04ca775 100644 --- a/packages/pipette-scores/pipette_scores/scoring/completion_text.py +++ b/packages/pipette-scores/pipette_scores/scoring/completion_text.py @@ -1,6 +1,10 @@ """Completion-text preprocessing shared across scorers.""" import re +from collections import Counter +from dataclasses import dataclass + +from pipette_scores.types import SampleCompletion # Vendored from Liquid4All/liquid_evals (liquid_evals/parser.py). @@ -57,3 +61,363 @@ def remove_thinking_tags(text: str) -> str: text = re.sub(r"<\|(?:end|return)\|>\s*$", "", text) return text.strip() + + +# Stop reasons (plan-types' snake_case names) for a generation that ended +# before the model did: `truncated` is the output-token cap or a full slot +# context; `doom_loop` is a runaway-repetition stop (vLLM's +# repetition_detection, finish_reason "repetition", or a legacy client's +# detector). +_CUT_STOP_REASONS = frozenset({"truncated", "doom_loop"}) + +# What remove_thinking_tags treats as the end of the reasoning: a think close +# tag or the gpt-oss harmony transition into the final channel. +_REASONING_CLOSE = re.compile(r"(?i:|\[/THINK\])|assistant(?:<\|channel\|>)?final") + +# The closes remove_thinking_tags' orphan-close rule cuts through, in its order: +# the first , else the first [/THINK]. +_ORPHAN_CLOSES = (re.compile(r"", re.IGNORECASE), re.compile(r"\[/THINK\]", re.IGNORECASE)) + +# Wrappers around the answer that are chat-format markup, not answer text: +# ERNIE-4.5's (vLLM's ernie45 parser leaves it in the content) and +# Cohere North's <|START_TEXT|>. Each pair is (opener at a section start, +# closer at the end of the completion). +_ANSWER_WRAPPERS = tuple( + (re.compile(r"\s*(" + re.escape(opener) + ")"), re.compile(re.escape(closer) + r"\s*\Z")) + for opener, closer in (("", ""), ("<|START_TEXT|>", "<|END_TEXT|>")) +) + + +# ..., as Ministral-3-Reasoning writes it after its own [/THINK]. +_THINKING_TAG = re.compile(r"<(/?)thinking>", re.IGNORECASE) + + +def _gemma4_thought_to_think(text: str) -> str: + """Rewrite Gemma 4 thought channels as blocks. + + Reads the markup the way the Gemma 4 chat template's strip_thinking does: + each ``<|channel>`` up to the next ```` is reasoning. An unclosed + channel becomes an open , which remove_thinking_tags removes to the + end; a ```` with no opener before it is a stray marker and is + dropped. + """ + if "<|channel>" not in text and "" not in text: + return text + *closed, tail = text.split("") + out = [] + for part in closed: + head, opened, thought = part.partition("<|channel>") + out.append(head + "" + thought.replace("<|channel>", "") + "" if opened else part) + head, opened, thought = tail.partition("<|channel>") + out.append(head + "" + thought.replace("<|channel>", "") if opened else tail) + return "".join(out) + + +# A wrapper nests a handful deep at most. normalize_vlm_reasoning_markup bounds the +# peeling so a runaway repeating the opener, which would peel one copy per pass over +# the whole text, is read in linear time. +_MAX_PEELS = 16 + + +def _unwrap_answer(text: str, max_peels: int | None = None) -> str: + """Drop answer wrappers that open a section and close the completion. + + A section starts at the top of the completion or right where the + reasoning closes (where remove_thinking_tags' orphan-close rule cuts); the + same tags anywhere else are answer text and stay. Wrappers are peeled + until none is left, so one nested in another goes too, or for at most + ``max_peels`` passes. + """ + peeled = True + passes = 0 + while peeled and (max_peels is None or passes < max_peels): + passes += 1 + peeled = False + for opener, closer in _ANSWER_WRAPPERS: + # Find the section starts afresh: a peeled opener moves them. + close = next((m for c in _ORPHAN_CLOSES if (m := c.search(text))), None) + section_starts = (0, close.end()) if close else (0,) + spans = [m.span(1) for start in section_starts if (m := opener.match(text, start))] + if not spans: + continue + for begin, end in reversed(spans): + text = text[:begin] + text[end:] + text = closer.sub("", text) + peeled = True + return text + + +def normalize_reasoning_markup(text: str) -> str: + """Rewrite model-specific reasoning markup into the form remove_thinking_tags reads. + + - Gemma 4 thought channels (``<|channel>thought…``) become + blocks; stray channel markers are dropped. + - Cohere North's ``<|START_THINKING|>`` / ``<|END_THINKING|>`` become + / . + - An ERNIE ``…`` or North + ``<|START_TEXT|>…<|END_TEXT|>`` wrapper around the answer is dropped. + + Text without this markup is returned unchanged, and a second pass changes + nothing. The result + still carries its reasoning, so it goes through the usual stripping next: + remove_thinking_tags, or ifstruct's validator, which strips on its own. + """ + text = _gemma4_thought_to_think(text) + text = text.replace("<|START_THINKING|>", "").replace("<|END_THINKING|>", "") + return _unwrap_answer(text) + + +def normalize_vlm_reasoning_markup(text: str) -> str: + """normalize_reasoning_markup for the VLM evals (split_reasoning), which also rewrites: + + - ```` / ```` into / ; + - Kimi-VL's ``◁think▷`` / ``◁/think▷``, plain text in its vocabulary (the + server prints them), into / . A reply that never closes + its block loses everything from the opener on, as Kimi's own + ``extract_summary`` gives "" for it. + + The text evals keep normalize_reasoning_markup. Text without this markup is + returned unchanged, and a second pass changes nothing. + """ + text = _gemma4_thought_to_think(text) + text = text.replace("<|START_THINKING|>", "").replace("<|END_THINKING|>", "") + text = text.replace("◁think▷", "").replace("◁/think▷", "") + text = _THINKING_TAG.sub(lambda m: "" if m.group(1) else "", text) + return _unwrap_answer(text, max_peels=_MAX_PEELS) + + +# normalize_vlm_reasoning_markup has rewritten every family onto these. One scan finds them all, +# in time linear in the text however many a runaway completion repeats. +_TAG = re.compile(r"||\[THINK\]|\[/THINK\]", re.IGNORECASE) +_HARMONY_FINAL = re.compile(r"(?:<\|start\|>)?assistant(?:<\|channel\|>)?final(?:<\|message\|>)?") +_HARMONY_END = re.compile(r"<\|(?:end|return)\|>\s*$") +_ANSWER_TAG = re.compile(r"|", re.IGNORECASE) + +CLOSED = "closed" +ORPHAN_CLOSE = "orphan_close" +UNCLOSED_CUT = "unclosed_cut" +UNCLOSED_STOP = "unclosed_stop" +NO_REASONING = "no_reasoning" +STATUSES = (CLOSED, ORPHAN_CLOSE, UNCLOSED_CUT, UNCLOSED_STOP, NO_REASONING) + + +@dataclass(frozen=True, slots=True) +class Reasoning: + """One completion, split into its reasoning and its answer. + + ``answer`` is what a scorer grades. ``status`` is one of STATUSES. ``blanked`` + marks the samples the strict view grades as empty. ``answer_before_close`` marks + an answer read from before a close, see split_reasoning. + """ + + answer: str + status: str + blanked: bool + answer_before_close: bool = False + + +def _answer_wrapper_content(text: str) -> str | None: + """The content of the last complete ``...`` in the text.""" + opened: int | None = None + found: tuple[int, int] | None = None + for m in _ANSWER_TAG.finditer(text): + if m.group()[1] != "/": + opened = m.end() + elif opened is not None: + found = (opened, m.start()) + opened = None + return text[found[0] : found[1]] if found else None + + +def _peel_answer_tag(answer: str) -> str: + """Drop an ```` that opens the answer and its ```` that ends it.""" + head = answer.lstrip() + if head[:8].lower() != "": + return answer + body = head[8:].rstrip() + return body[:-9] if body[-9:].lower() == "" else body + + +def split_reasoning( + completion: SampleCompletion, + enable_thinking: bool | None = None, + *, + opens_reasoning: bool = False, +) -> Reasoning: + """Split a completion into its reasoning and the answer to grade, whatever + opened the reasoning and however it ended. + + Reasoning counts as **opened** when its opener is in the text (, + [THINK], and after normalize_vlm_reasoning_markup the ◁think▷, , North, + Gemma channel forms), when the prompt opened it for the model (``opens_reasoning``: + the run's prompt profile is registered in prompt_profiles.REASONING_OPENERS, + which covers an assistant prefill, a system prompt, a template and a model + that always thinks), when ``enable_thinking`` is true, or when a close + (, [/THINK], the gpt-oss ``assistantfinal`` transition) shows + it was opened out of sight, as with a hidden control token or a template's + ``\\n``. + + ``status``: + - ``closed``: a close was read, and the opener was known, or is visible with no + or [/THINK] before it. + - ``orphan_close``: a close was read, the opener was not known, and no opener is + visible or a or [/THINK] comes before the first one. + - ``unclosed_cut``: opened, no close, and generation was cut (``truncated`` or + ``doom_loop``). + - ``unclosed_stop``: opened, no close, and the model stopped on its own. + - ``no_reasoning``: no sign of reasoning. + + ``answer`` is graded as the thinking-tag rules of liquid_evals' parser + have always graded, so a completion with no tags is graded whole, whatever the + status: complete blocks are removed (a block is closed by the first close after + its opener; an opener inside a block is part of it); the text through the first + close that has no opener is reasoning, the first such , else the first such + [/THINK], as remove_thinking_tags' orphan-close rule cuts (a later stray close stays + as text); an + opener left at the end removes the rest; then the gpt-oss transition and trailing + harmony markers go. Beyond that, for the cases that parser grades empty or + wrongly: + - the two tag families pair with each other (``[THINK]...``), as ```` + ends a ``[THINK]`` block in Ministral-3-Reasoning's replies; + - **answer before close**: when the opener was not known, the reply opens with a + one-word answer and a close (``yes[/THINK]``), and everything after is more reasoning, + that word is the answer (``answer_before_close``); more words before the close are + reasoning, as Ministral writes in ``So the answer should be "yes".[/THINK]``, and grade empty; + - **unclosed_stop** with the opener from ``opens_reasoning`` or in the text (``enable_thinking`` + alone is not enough) grades the content of the last complete ``...`` if + there is one, else as the rules above; + - an ```` that opens the answer and its closing tag are dropped. + + ``blanked`` is the strict view: reasoning was opened by the opener in the text, the + prompt, or ``enable_thinking``, generation was cut, no close exists and there is + text. It does not depend on the run having recorded ``enable_thinking``: a reply that + opened its own thought and was cut counts even under ``enable_thinking`` false. + """ + text = normalize_vlm_reasoning_markup(completion.completion) + cut = completion.stop_reason in _CUT_STOP_REASONS + + blocks: list[tuple[int, int]] = [] + orphan_closes: list[tuple[int, int]] = [] + in_block, block_start = False, 0 + visible_open = any_close = False + for m in _TAG.finditer(text): + is_close = m.group()[1] == "/" or m.group()[1:3] == "[/" + if is_close: + any_close = True + else: + visible_open = True + if in_block: + if is_close: + blocks.append((block_start, m.end())) + in_block = False + elif is_close: + orphan_closes.append((m.start(), m.end())) + else: + in_block, block_start = True, m.start() + if in_block: + blocks.append((block_start, len(text))) + + def visible(start: int, end: int) -> str: + """The text of text[start:end] outside every block.""" + out, at = [], start + for b_start, b_end in blocks: + if b_end <= start or b_start >= end: + continue + out.append(text[at : max(b_start, at)]) + at = max(at, b_end) + out.append(text[at:end]) + return "".join(out) + + # remove_thinking_tags' orphan-close rule cuts through the first , else the first [/THINK]. + cut_at = next( + (end for start, end in orphan_closes if text[start] == "<"), orphan_closes[0][1] if orphan_closes else 0 + ) + answer = visible(cut_at, len(text)) + before = visible(0, orphan_closes[0][0]) if orphan_closes else "" + harmony = _HARMONY_FINAL.search(answer) + if harmony: + answer = answer[harmony.end() :] + any_close = any_close or _HARMONY_FINAL.search(text) is not None + + known = opens_reasoning or enable_thinking is True + opened = known or visible_open or any_close + if not opened: + status = NO_REASONING + elif any_close: + # Only a close with no opener before it is the sign of reasoning opened out of sight. + orphan_first = bool(orphan_closes) and (not blocks or orphan_closes[0][0] < blocks[0][0]) + status = ORPHAN_CLOSE if not known and (orphan_first or not visible_open) else CLOSED + else: + status = UNCLOSED_CUT if cut else UNCLOSED_STOP + + fallback = False + if ( + status == ORPHAN_CLOSE + and orphan_closes + and (not blocks or orphan_closes[0][0] < blocks[0][0]) + and not harmony + and not answer.strip() + and len(before.split()) == 1 + and any(b_start >= orphan_closes[0][1] for b_start, _ in blocks) + ): + answer, fallback = before, True + if status == UNCLOSED_STOP and (opens_reasoning or visible_open): + wrapped = _answer_wrapper_content(text) + if wrapped is not None: + answer = wrapped + answer = _HARMONY_END.sub("", answer) + if status != NO_REASONING: + answer = _peel_answer_tag(answer) + + return Reasoning( + answer=answer.strip(), + status=status, + blanked=(known or visible_open) and cut and not any_close and bool(text.strip()), + answer_before_close=fallback, + ) + + +class ReasoningTally: + """What a scorer reports of its completions' reasoning: how many came out in + each status, and how many answers were read from before a close.""" + + def __init__(self) -> None: + self.status: Counter[str] = Counter() + self.answer_before_close = 0 + + def add(self, reasoning: Reasoning) -> Reasoning: + self.status[reasoning.status] += 1 + self.answer_before_close += reasoning.answer_before_close + return reasoning + + def context(self) -> dict[str, object]: + return { + "reasoning_status": {status: self.status[status] for status in STATUSES}, + "reasoning_answer_before_close": self.answer_before_close, + } + + +def grading_text(completion: SampleCompletion, enable_thinking: bool | None) -> tuple[str, bool]: + """Return the text to grade for a completion, and whether the strict view blanks it. + + The text is the completion after normalize_reasoning_markup, whatever the + stop reason. Grading it with the thinking-tag rules is what liquid_evals + does with every response, cut or not: a completion whose reasoning the + prompt opened and the cap cut carries no tag and is graded whole, and one + that opened itself loses everything from the tag on. + + The flag marks the samples the strict view grades as empty instead: with + thinking on, the client cut generation (``truncated`` or ``doom_loop``) and + the text has no reasoning close. A completion that is already blank is not + flagged. Scorers count the flagged samples as ``_blanked_truncated`` + and report ``_pass_at_1_blanked``, their pass@1 with those samples + counted wrong. + """ + text = normalize_reasoning_markup(completion.completion) + blanked = ( + bool(enable_thinking) + and completion.stop_reason in _CUT_STOP_REASONS + and not _REASONING_CLOSE.search(text) + and bool(text.strip()) + ) + return text, blanked diff --git a/packages/pipette-scores/pipette_scores/scoring/gpqa_diamond/__init__.py b/packages/pipette-scores/pipette_scores/scoring/gpqa_diamond/__init__.py index 1c25369..37cbe96 100644 --- a/packages/pipette-scores/pipette_scores/scoring/gpqa_diamond/__init__.py +++ b/packages/pipette-scores/pipette_scores/scoring/gpqa_diamond/__init__.py @@ -5,7 +5,7 @@ from pipette_scores.scoring.gpqa_diamond.mcq import score_mcq from pipette_scores.repeats import logical_id -from pipette_scores.scoring.completion_text import remove_thinking_tags +from pipette_scores.scoring.completion_text import grading_text, remove_thinking_tags from pipette_scores.types import GPQADiamondSample, SampleCompletion, SampleScore # GPQA Diamond is a 4-option MCQ (Artificial Analysis methodology). @@ -13,15 +13,15 @@ def score( - completions: list[SampleCompletion], samples: list[GPQADiamondSample] + completions: list[SampleCompletion], samples: list[GPQADiamondSample], *, enable_thinking: bool | None = None ) -> tuple[list[SampleScore], dict[str, Any]]: """Score GPQA Diamond completions against the correct option letter. Each completion is run through the Artificial Analysis MCQ extractor (``score_mcq`` from this eval's ``mcq`` module, with ``valid_options="ABCD"``): the chosen letter is extracted from the response (after reasoning is - stripped via ``remove_thinking_tags``) and compared to the ground-truth - letter. + stripped via ``grading_text`` and ``remove_thinking_tags``) and compared to + the ground-truth letter. AA methodology runs each question with repeats (R=5) and reports pass@1 over the N·R rows. Repeats are realized as ``#k`` salted ids (see @@ -33,8 +33,11 @@ def score( Context keys: ``gpqa_diamond_pass_at_1`` (+ ``_stderr`` and ``_per_sample`` breakdown by logical id), ``gpqa_diamond_unparsed_rate`` (fraction of - received completions whose answer couldn't be extracted), and flat - ``gpqa_diamond_choice/{A,B,C,D,none}`` counts. + received completions whose answer couldn't be extracted), + ``gpqa_diamond_blanked_truncated`` (completions cut short inside their + reasoning, which the strict view grades as empty, see ``grading_text``), + ``gpqa_diamond_pass_at_1_blanked`` (pass@1 with those counted wrong), and + flat ``gpqa_diamond_choice/{A,B,C,D,none}`` counts. """ by_id = {s.id: s for s in samples} scores: list[SampleScore] = [] @@ -43,14 +46,19 @@ def score( # couldn't parse one) so the context can distinguish a wrong answer from an # extraction miss — a low score that's really a formatting/parse problem. choices: Counter[str] = Counter() + blanked = 0 + blanked_correct = 0 for c in completions: sample = by_id[c.id] - text = remove_thinking_tags(c.completion) + text, cut = grading_text(c, enable_thinking) + text = remove_thinking_tags(text) result = score_mcq(text, sample.answer, valid_options=_VALID_OPTIONS) scores.append(SampleScore(id=c.id, is_correct=result.correct)) if result.correct: correct += 1 + blanked += cut + blanked_correct += cut and result.correct choices[result.extracted or "none"] += 1 n_samples = len(samples) @@ -73,6 +81,8 @@ def score( # Of the completions we received, the fraction whose answer couldn't be # extracted (counts as wrong) — separates "wrong answer" from "unparsable". "gpqa_diamond_unparsed_rate": choices["none"] / n_completions if n_completions else 0.0, + "gpqa_diamond_blanked_truncated": blanked, + "gpqa_diamond_pass_at_1_blanked": (correct - blanked_correct) / n_samples if n_samples else 0.0, } # Flat, fixed-key choice distribution (counts), mirroring ifbench's # `/`-delimited breakdown keys. Always emit every key (A-D + none) so the diff --git a/packages/pipette-scores/pipette_scores/scoring/ifbench/__init__.py b/packages/pipette-scores/pipette_scores/scoring/ifbench/__init__.py index 1362517..8b0d2c1 100644 --- a/packages/pipette-scores/pipette_scores/scoring/ifbench/__init__.py +++ b/packages/pipette-scores/pipette_scores/scoring/ifbench/__init__.py @@ -3,7 +3,7 @@ from typing import Any from pipette_scores.repeats import logical_id -from pipette_scores.scoring.completion_text import remove_thinking_tags +from pipette_scores.scoring.completion_text import grading_text, remove_thinking_tags from pipette_scores.scoring.ifbench.upstream import ( InputExample, @@ -15,9 +15,14 @@ def score( - completions: list[SampleCompletion], samples: list[IFBenchSample] + completions: list[SampleCompletion], samples: list[IFBenchSample], *, enable_thinking: bool | None = None ) -> tuple[list[SampleScore], dict[str, Any]]: - """Score completions with upstream IFBench's strict and loose checkers. + """Score completions with IFBench's strict and loose checkers. + + The checkers are allenai/IFBench at the revision liquid_evals scores its + ``ifbench_aa`` task with (``vendor/ifbench-checkers``), so a completion gets + the same verdict here as there. The only exceptions are the few inputs on + which liquid_evals has no stable verdict; that directory's README lists them. Mirrors upstream ``run_eval.py``: every response is scored under both ``test_instruction_following_strict`` and ``test_instruction_following_loose``. @@ -33,9 +38,12 @@ def score( accuracy per full instruction id (``strict/:``) — each for both the strict and loose passes. - Reasoning chains are removed first via ``remove_thinking_tags``. Upstream - strips these during generation; because this scorer ingests raw completions, - it does the equivalent cleanup here before applying the unmodified checkers. + Reasoning chains are removed first via ``remove_thinking_tags``, the same + function liquid_evals applies to each response before its checkers run. + ``grading_text`` runs before it, rewriting model-specific reasoning markup. + The context counts completions cut short inside their reasoning as + ``ifbench_blanked_truncated``; ``ifbench_pass_at_1_blanked`` is + ``ifbench_pass_at_1`` with those counted wrong. When the dataset is run with repeats (``#k`` salted ids), the context also carries ``ifbench_pass_at_1`` — the loose prompt-level pass rate aggregated @@ -56,10 +64,13 @@ def score( strict_iid_passed: dict[str, int] = defaultdict(int) loose_iid_passed: dict[str, int] = defaultdict(int) per_iid_total: dict[str, int] = defaultdict(int) + blanked = 0 + blanked_correct = 0 for c in completions: sample = by_id[c.id] - text = remove_thinking_tags(c.completion) + text, cut = grading_text(c, enable_thinking) + text = remove_thinking_tags(text) inp = InputExample( key=sample.key, @@ -73,6 +84,8 @@ def score( loose_out = test_instruction_following_loose(inp, {sample.prompt: text}) scores.append(SampleScore(id=c.id, is_correct=loose_out.follow_all_instructions)) + blanked += cut + blanked_correct += cut and loose_out.follow_all_instructions if strict_out.follow_all_instructions: strict_prompt_correct += 1 @@ -121,6 +134,8 @@ def score( context["ifbench_pass_at_1_per_sample"] = { lid: per_sample_correct[lid] / per_sample_total[lid] for lid in sorted(per_sample_total) } + context["ifbench_blanked_truncated"] = blanked + context["ifbench_pass_at_1_blanked"] = (loose_prompt_correct - blanked_correct) / n_samples if n_samples else 0.0 # tier0 derives from tier1: each instruction id belongs to exactly one # category (the prefix before ":"), so category totals are just the sum # over their members. Keys have no ":", e.g. "strict/count". diff --git a/packages/pipette-scores/pipette_scores/scoring/ifbench/upstream.py b/packages/pipette-scores/pipette_scores/scoring/ifbench/upstream.py index b72e2c8..2c4f928 100644 --- a/packages/pipette-scores/pipette_scores/scoring/ifbench/upstream.py +++ b/packages/pipette-scores/pipette_scores/scoring/ifbench/upstream.py @@ -1,11 +1,13 @@ -"""Adapter that exposes the pristine ``vendor/ifbench`` submodule as a package. +"""Adapter that exposes the vendored IFBench checkers as a package. -Upstream IFBench ships four top-level modules (``evaluation_lib``, -``instructions``, ``instructions_registry``, ``instructions_util``) that -import each other by bare name. We don't want to patch upstream, so the -bootstrap below: +The checkers are allenai/IFBench at the revision Liquid4All/liquid_evals +scores its ``ifbench_aa`` task with (``vendor/ifbench-checkers``; see its +README for the commit and the local changes). They ship four top-level modules +(``evaluation_lib``, ``instructions``, ``instructions_registry``, +``instructions_util``) that import each other by bare name. We don't want to +rewrite those imports, so the bootstrap below: -1. Temporarily prepends the submodule dir to ``sys.path`` so the bare +1. Temporarily prepends the checker dir to ``sys.path`` so the bare imports resolve. 2. Imports the four modules in dependency order (primes ``sys.modules``). 3. Rehomes them under ``pipette_scores.scoring.ifbench._upstream.*`` so @@ -23,15 +25,15 @@ # `_upstream/` is NOT in the source tree — it is generated at wheel-build time by # the `[tool.hatch.build.targets.wheel.force-include]` block in -# packages/pipette-scores/pyproject.toml, which copies the `vendor/ifbench` -# submodule's scoring files (+ LICENSE) into the wheel. So: -# - installed wheel -> bundled copy at _upstream/ (_BUNDLED) -# - editable/source -> fall back to the vendor/ifbench submodule (_REPO) +# packages/pipette-scores/pyproject.toml, which copies the checker files in +# `vendor/ifbench-checkers` (+ LICENSE) into the wheel. So: +# - installed wheel -> bundled copy at _upstream/ (_BUNDLED) +# - editable/source -> fall back to vendor/ifbench-checkers (_REPO) # CI exercises the bundled path via the `wheel-smoke` job (see .github/workflows). _BUNDLED = pathlib.Path(__file__).resolve().parent / "_upstream" # parents climb from .../pipette_scores/scoring/ifbench/upstream.py: # ifbench[0] scoring[1] pipette_scores[2] pipette-scores[3] packages[4] [5]. -_REPO = pathlib.Path(__file__).resolve().parents[5] / "vendor" / "ifbench" +_REPO = pathlib.Path(__file__).resolve().parents[5] / "vendor" / "ifbench-checkers" if (_BUNDLED / "evaluation_lib.py").exists(): _SRC = _BUNDLED @@ -39,9 +41,9 @@ _SRC = _REPO else: raise RuntimeError( - "IFBench source not found. Expected either a bundled copy at " - f"{_BUNDLED} (installed wheel) or the submodule at {_REPO} " - "(editable install — run `git submodule update --init vendor/ifbench`)." + "IFBench checkers not found. Expected either a bundled copy at " + f"{_BUNDLED} (installed wheel) or the vendored copy at {_REPO} " + "(editable install)." ) _MODULES = ("instructions_util", "instructions", "instructions_registry", "evaluation_lib") diff --git a/packages/pipette-scores/pipette_scores/scoring/ifstruct.py b/packages/pipette-scores/pipette_scores/scoring/ifstruct.py index 177dae0..eb3ba25 100644 --- a/packages/pipette-scores/pipette_scores/scoring/ifstruct.py +++ b/packages/pipette-scores/pipette_scores/scoring/ifstruct.py @@ -3,19 +3,33 @@ from ifstruct.eval import EvalResult, build_summary from ifstruct.validator import validate_response +from pipette_scores.scoring.completion_text import grading_text from pipette_scores.types import IFStructSample, SampleCompletion, SampleScore def score( - completions: list[SampleCompletion], samples: list[IFStructSample] + completions: list[SampleCompletion], samples: list[IFStructSample], *, enable_thinking: bool | None = None ) -> tuple[list[SampleScore], dict[str, Any]]: + """Validate each completion against its sample's schema. + + The context is the vendored ``build_summary`` (``pass_rate`` over the + completions received, and its breakdowns) plus ``ifstruct_blanked_truncated`` + (completions cut short inside their reasoning, see ``grading_text``) and + ``ifstruct_pass_at_1_blanked``, ``pass_rate`` with those counted wrong. + ifstruct is served once per prompt, so its pass rate is its pass@1. + """ by_id = {s.id: s for s in samples} scores: list[SampleScore] = [] results: list[EvalResult] = [] + blanked = 0 + blanked_passed = 0 for c in completions: sample = by_id[c.id] + # validate_response strips thinking tags itself, so grading_text's + # output goes in as is; a second strip would cut at a later . + text, cut = grading_text(c, enable_thinking) v = validate_response( - response=c.completion, + response=text, json_schema=sample.json_schema, top_level_count=sample.top_level_count, require_no_commentary=sample.require_no_commentary, @@ -25,6 +39,8 @@ def score( require_code_block=sample.require_code_block, ) scores.append(SampleScore(id=c.id, is_correct=v.passed)) + blanked += cut + blanked_passed += cut and v.passed results.append( EvalResult( seed=sample.seed, @@ -41,4 +57,10 @@ def score( require_wrapper_key=sample.require_wrapper_key, ) ) - return scores, build_summary(results) + summary = build_summary(results) + total = summary["total"] + return scores, { + **summary, + "ifstruct_blanked_truncated": blanked, + "ifstruct_pass_at_1_blanked": (summary["passed"] - blanked_passed) / total if total else 0.0, + } diff --git a/packages/pipette-scores/pipette_scores/scoring/math_500/__init__.py b/packages/pipette-scores/pipette_scores/scoring/math_500/__init__.py index ded4cf2..e1b0705 100644 --- a/packages/pipette-scores/pipette_scores/scoring/math_500/__init__.py +++ b/packages/pipette-scores/pipette_scores/scoring/math_500/__init__.py @@ -1,28 +1,105 @@ import math from collections import Counter, defaultdict +from collections.abc import Callable +from dataclasses import dataclass from typing import Any +from pipette_scores.scoring.math_500 import liquid_evals_parser from pipette_scores.scoring.math_500.math_generic import score_math_generic from pipette_scores.repeats import logical_id -from pipette_scores.scoring.completion_text import remove_thinking_tags -from pipette_scores.types import Math500Sample, SampleCompletion, SampleScore +from pipette_scores.scoring.completion_text import grading_text, remove_thinking_tags +from pipette_scores.types import Math500Grader, Math500Sample, SampleCompletion, SampleScore -# Extraction/grading methods reported by score_math_generic. -_METHODS = ("prm800k", "robust_boxed", "no_match", "failed") + +def _grade_math_generic(text: str, gold: str) -> tuple[bool, str]: + result = score_math_generic(text, gold) + return result.correct, result.method + + +# extract_math_answer's patterns, in its priority order: the first that finds +# an answer is the one its prediction came from. +_LIQUID_EVALS_EXTRACTORS = ( + ("boxed", liquid_evals_parser._extract_boxed_pattern), + ("hash", liquid_evals_parser._extract_hash_pattern), + ("answer_colon", liquid_evals_parser._extract_answer_colon_pattern), + ("answer_is", liquid_evals_parser._extract_answer_is_pattern), + ("other", liquid_evals_parser._extract_other_patterns), + ("numeric", liquid_evals_parser._extract_numeric_fallback), +) + + +def liquid_evals_extraction(text: str) -> str: + """Name the extract_math_answer pattern that yields ``text``'s answer, or ``none``.""" + return next((name for name, extract in _LIQUID_EVALS_EXTRACTORS if extract(text)), "none") + + +def _grade_liquid_evals(text: str, gold: str) -> tuple[bool, str]: + # tasks/math500.py: predicted = extract_math_answer(model_response), then + # verify_math_answer(true_label, predicted, model_response). + predicted = liquid_evals_parser.extract_math_answer(text) + correct = liquid_evals_parser.verify_math_answer(gold, predicted, text) + return bool(correct), liquid_evals_extraction(text) + + +@dataclass(frozen=True) +class _Grader: + # (grading text, gold answer) -> (is correct, diagnostic bucket) + grade: Callable[[str, str], tuple[bool, str]] + # Context key prefix of the bucket counts, and every bucket, so the key set + # stays fixed whichever buckets were hit. + bucket_key: str + buckets: tuple[str, ...] + # The buckets in which the grader found no usable answer. + unparsed: frozenset[str] + + +_GRADERS = { + # score_math_generic's grading paths. + Math500Grader.MATH_GENERIC: _Grader( + grade=_grade_math_generic, + bucket_key="math_500_method", + buckets=("prm800k", "robust_boxed", "no_match", "failed"), + unparsed=frozenset({"no_match", "failed"}), + ), + # The extract_math_answer pattern the prediction came from. + Math500Grader.LIQUID_EVALS_MATH500: _Grader( + grade=_grade_liquid_evals, + bucket_key="math_500_extraction", + buckets=(*(name for name, _ in _LIQUID_EVALS_EXTRACTORS), "none"), + unparsed=frozenset({"none"}), + ), +} + + +def _dataset_grader(samples: list[Math500Sample]) -> Math500Grader: + graders = {s.grader for s in samples} + if len(graders) > 1: + raise ValueError(f"MATH-500 samples name more than one grader: {sorted(graders)}") + return graders.pop() if graders else Math500Grader.MATH_GENERIC def score( - completions: list[SampleCompletion], samples: list[Math500Sample] + completions: list[SampleCompletion], samples: list[Math500Sample], *, enable_thinking: bool | None = None ) -> tuple[list[SampleScore], dict[str, Any]]: """Score MATH-500 completions against the gold answer. - Each completion is graded by this eval's ``math_generic.score_math_generic`` - (boxed-answer extraction → PRM800K symbolic grader → robust-boxed fallback) - after reasoning is stripped via ``remove_thinking_tags``. The raw text is - passed through — the grader does its own ``\\boxed{}``/last-number extraction. + The dataset version picks the grader (``metadata.grader``, carried on each + sample): + + - ``math_generic`` (2026.06.1, liquid_evals' math500_std track): this + eval's ``math_generic.score_math_generic`` (boxed-answer extraction → + PRM800K symbolic grader → robust-boxed fallback). + - ``liquid_evals_math500`` (2026.10.1, liquid_evals' plain math500 task): + ``extract_math_answer`` then ``verify_math_answer``, vendored verbatim in + ``liquid_evals_parser`` — mathd/sympy equivalence on the extracted answer, + then on each of the response's last 150 lines. + + Either way the completion's reasoning is stripped first via + ``grading_text`` and ``remove_thinking_tags``; the grader does its own + answer extraction from what is left. - AA methodology runs each question with repeats (R) and reports pass@1 over the + Datasets may run each question with repeats (R) and report pass@1 over the N·R rows; repeats are realized as ``#k`` salted ids (see ``repeats.expand``). The pass@1 denominator is the SAMPLE count from ``samples`` (not received completions), so a missing attempt counts as a fail — matching the management @@ -30,24 +107,37 @@ def score( Context keys: ``math_500_pass_at_1`` (+ ``_stderr`` and ``_per_sample`` breakdown by logical id), ``math_500_unparsed_rate`` (fraction of received - completions the grader found no usable answer in), and flat - ``math_500_method/{prm800k,robust_boxed,no_match,failed}`` counts. + completions the grader found no usable answer in), + ``math_500_blanked_truncated`` (completions cut short inside their + reasoning, which the strict view grades as empty, see ``grading_text``), + ``math_500_pass_at_1_blanked`` (pass@1 with those counted wrong), and flat + per-grader counts: ``math_500_method/{prm800k,robust_boxed,no_match,failed}`` + for math_generic, and + ``math_500_extraction/{boxed,hash,answer_colon,answer_is,other,numeric,none}`` + (the extract_math_answer pattern that found the answer) for + liquid_evals_math500. """ + grader = _GRADERS[_dataset_grader(samples)] by_id = {s.id: s for s in samples} scores: list[SampleScore] = [] correct = 0 # Which grading path each completion took, so the context can separate a # wrong answer from one the grader couldn't extract an answer out of. - methods: Counter[str] = Counter() + buckets: Counter[str] = Counter() + blanked = 0 + blanked_correct = 0 for c in completions: sample = by_id[c.id] - text = remove_thinking_tags(c.completion) - result = score_math_generic(text, sample.answer) - scores.append(SampleScore(id=c.id, is_correct=result.correct)) - if result.correct: + text, cut = grading_text(c, enable_thinking) + text = remove_thinking_tags(text) + is_correct, bucket = grader.grade(text, sample.answer) + scores.append(SampleScore(id=c.id, is_correct=is_correct)) + if is_correct: correct += 1 - methods[result.method] += 1 + blanked += cut + blanked_correct += cut and is_correct + buckets[bucket] += 1 n_samples = len(samples) per_sample_total: dict[str, int] = defaultdict(int) @@ -60,9 +150,9 @@ def score( pass_at_1 = correct / n_samples if n_samples else 0.0 n_completions = len(completions) - # No usable answer extracted (no boxed/number match, or the grader errored) — - # separates "wrong answer" from "couldn't parse an answer". - unparsed = methods["no_match"] + methods["failed"] + # No usable answer extracted — separates "wrong answer" from "couldn't + # parse an answer". + unparsed = sum(buckets[b] for b in grader.unparsed) context: dict[str, Any] = { "math_500_pass_at_1": pass_at_1, "math_500_pass_at_1_stderr": (math.sqrt(pass_at_1 * (1.0 - pass_at_1) / n_samples) if n_samples else 0.0), @@ -70,9 +160,11 @@ def score( lid: per_sample_correct[lid] / per_sample_total[lid] for lid in sorted(per_sample_total) }, "math_500_unparsed_rate": unparsed / n_completions if n_completions else 0.0, + "math_500_blanked_truncated": blanked, + "math_500_pass_at_1_blanked": (correct - blanked_correct) / n_samples if n_samples else 0.0, } - # Flat, fixed-key grading-method distribution — always emit every key so the - # context schema stays stable regardless of which methods were hit. - for method in _METHODS: - context[f"math_500_method/{method}"] = methods[method] + # Flat, fixed-key grading-path distribution — always emit every key so the + # context schema stays stable regardless of which paths were hit. + for bucket in grader.buckets: + context[f"{grader.bucket_key}/{bucket}"] = buckets[bucket] return scores, context diff --git a/packages/pipette-scores/pipette_scores/scoring/math_500/liquid_evals_parser.py b/packages/pipette-scores/pipette_scores/scoring/math_500/liquid_evals_parser.py new file mode 100644 index 0000000..6f4f0b3 --- /dev/null +++ b/packages/pipette-scores/pipette_scores/scoring/math_500/liquid_evals_parser.py @@ -0,0 +1,717 @@ +"""liquid_evals' MATH-500 answer extraction and grading, vendored verbatim. + +Source: Liquid4All/liquid_evals, liquid_evals/parser.py at commit +d5534cfa17713b9fb2d1fe6626763d411380e9a8 (blob cd04c90839e366eb51d1071166d3727dcdcf0c99). +The plain `tasks/math500.py` task grades with these functions; the scorer here +calls them in the same order (see pipette_scores.scoring.math_500). + +What is copied, unchanged: +- parser.py:1-3 the stdlib imports the functions below use; +- parser.py:111-348 extract_math_answer and its _extract_* helpers, and + verify_math_answer (the last-150-lines fallback); +- parser.py:351-798 the mathd/sympy graders behind check_math_result, with their + imports and the pylatexenc log-level line. + +Not copied: template_prefills_thinking, compute_length_metadata, and +remove_thinking_tags (pipette_scores.scoring.completion_text already carries it). +The grader code descends from openai/prm800k's grader (MIT) and Hendrycks' +MATH answer normalization (MIT); see NOTICE and THIRD_PARTY_LICENSES.md. + +Do not edit the code below: tests/test_math_500_liquid_evals_parity.py checks it +against outputs of the original. To update it, copy the new upstream lines and +regenerate the parity fixture with scripts/gen_math_500_liquid_evals_parity.py. +""" + +import functools +import logging +import re + + +def extract_math_answer(text): + """ + Comprehensive answer extraction function that tries multiple patterns in order of priority. + + Priority order: + 1. \\boxed{} pattern + 2. ### or #### patterns + 3. "ANSWER:" pattern + 4. "answer is {}" pattern + 5. Other patterns (final answer, numeric fallback) + + Args: + text (str): Text to search for answer content + + Returns: + str: Extracted answer content or empty string if no pattern found + """ + if not isinstance(text, str): + return "" + + # 1. First priority: \\boxed{} pattern + boxed_answer = _extract_boxed_pattern(text) + if boxed_answer: + return boxed_answer + + # 2. Second priority: ### or #### patterns + hash_answer = _extract_hash_pattern(text) + if hash_answer: + return hash_answer + + # 3. Third priority: "ANSWER:" pattern + answer_colon = _extract_answer_colon_pattern(text) + if answer_colon: + return answer_colon + + # 4. Fourth priority: "answer is {}" pattern + answer_is = _extract_answer_is_pattern(text) + if answer_is: + return answer_is + + # 5. Fifth priority: Other patterns + other_answer = _extract_other_patterns(text) + if other_answer: + return other_answer + + # 6. Final fallback: numeric extraction from end of text + numeric_answer = _extract_numeric_fallback(text) + if numeric_answer: + return numeric_answer + + return "" + + +def _extract_boxed_pattern(text): + """Extract content from \\boxed{} pattern with proper brace handling.""" + matches = [] + i = 0 + while i < len(text): + # Look for \boxed{ (single backslash) + boxed_start = text.find("\\boxed{", i) + if boxed_start == -1: + break + + content_start = boxed_start + 7 # Length of "\\boxed{" + if content_start >= len(text): + break + + # Count braces to find the matching closing brace + brace_count = 1 + j = content_start + while j < len(text) and brace_count > 0: + if text[j] == "{": + brace_count += 1 + elif text[j] == "}": + brace_count -= 1 + j += 1 + + # If we found the matching closing brace + if brace_count == 0: + content = text[content_start : j - 1] + matches.append(content.strip()) + + # Continue searching from the end of this pattern + i = j + + if matches: + return matches[-1] # Return the last match + return "" + + +def _extract_hash_pattern(text): + """Extract content from ### or #### patterns.""" + # Try #### pattern first (more specific) + quad_hash_pattern = r"####\s*([^#]+?)\s*(?=\n|$)" + quad_matches = re.findall(quad_hash_pattern, text, re.IGNORECASE) + if quad_matches: + return quad_matches[-1].strip() + + # Try ### pattern + triple_hash_pattern = r"###\s*([^#]+?)\s*(?=\n|$)" + triple_matches = re.findall(triple_hash_pattern, text, re.IGNORECASE) + if triple_matches: + return triple_matches[-1].strip() + + return "" + + +def _extract_answer_colon_pattern(text): + """Extract content from 'ANSWER:' pattern.""" + answer_pattern = r"ANSWER\s*:\s*([^\n]+?)(?=\n|$)" + matches = re.findall(answer_pattern, text, re.IGNORECASE) + if matches: + return matches[-1].strip() + return "" + + +def _extract_answer_is_pattern(text): + """Extract content from 'answer is {}' patterns.""" + # Pattern for "the answer is [content]" or "answer is [content]" + answer_is_patterns = [ + r"(?:the\s+)?answer\s+is\s*\[([^\]]+)\]", # answer is [content] + r"(?:the\s+)?answer\s+is\s*\{([^}]+)\}", # answer is {content} + r"(?:the\s+)?answer\s+is\s*\(([^)]+)\)", # answer is (content) + r"(?:the\s+)?answer\s+is\s+([^\n.!?]+)", # answer is content + ] + + for pattern in answer_is_patterns: + matches = re.findall(pattern, text, re.IGNORECASE) + if matches: + return matches[-1].strip() + + return "" + + +def _extract_other_patterns(text): + """Extract content from other common answer patterns.""" + # Try "The final answer is [answer]" pattern + final_answer_pattern = r"The final answer is\s*\[?([^\]]+?)\]?\s*(?=\n|$)" + final_answer_matches = re.findall(final_answer_pattern, text, re.IGNORECASE) + if final_answer_matches: + return final_answer_matches[-1].strip() + + # Try "Final answer:" pattern + final_colon_pattern = r"Final answer\s*:\s*([^\n]+?)(?=\n|$)" + final_colon_matches = re.findall(final_colon_pattern, text, re.IGNORECASE) + if final_colon_matches: + return final_colon_matches[-1].strip() + + # Try simple "Answer:" at start of line + simple_answer_pattern = r"^Answer\s*:\s*([^\n]+?)(?=\n|$)" + simple_matches = re.findall( + simple_answer_pattern, text, re.IGNORECASE | re.MULTILINE + ) + if simple_matches: + return simple_matches[-1].strip() + + return "" + + +def _extract_numeric_fallback(text): + """Extract numeric answer as fallback from the end of text.""" + lines = text.strip().split("\n") + for line in reversed(lines): + line = line.strip() + if line: + # Look for numbers in the last non-empty line + numbers = re.findall(r"-?\d+(?:\.\d+)?", line) + if numbers: + return numbers[-1] + return "" + + +def verify_math_answer(true_label, predicted_label, *fallback_sources): + """Return True when the predicted answer matches the true answer. + + Falls back to the sympy/mathd based checker when direct string equality + fails. ``fallback_sources`` may contain additional strings (e.g. the raw + model response) that the math parser can inspect. + """ + + def _is_non_empty_string(value): + return isinstance(value, str) and value.strip() != "" + + truth = true_label.strip() if isinstance(true_label, str) else "" + + # Special case: if ground truth is "None" and predicted is empty, it's correct + if truth.lower() == "None" and not _is_non_empty_string(predicted_label): + return True + + if not truth: + return False + + if _is_non_empty_string(predicted_label) and predicted_label.strip() == truth: + return True + + candidates = [] + if _is_non_empty_string(predicted_label): + candidates.append(predicted_label.strip()) + + for source in fallback_sources: + if not _is_non_empty_string(source): + continue + stripped = source.strip() + cleaned = re.sub( + r"^(?:final answer:|the final answer is|answer:|###)\s*", + "", + stripped, + flags=re.IGNORECASE, + ).strip() + if cleaned and cleaned != stripped: + candidates.append(cleaned) + if "\n" in stripped: + # Limit the depth of line-based fallback parsing to speed up the process + MAX_FALLBACK_LINES = 150 + for n_lines, line in enumerate(reversed(stripped.splitlines()), start=1): + line = line.strip() + if _is_non_empty_string(line): + candidates.append(line) + cleaned_line = re.sub( + r"^(?:final answer:|the final answer is|answer:|###)\s*", + "", + line, + flags=re.IGNORECASE, + ).strip() + if cleaned_line and cleaned_line != line: + candidates.append(cleaned_line) + if n_lines >= MAX_FALLBACK_LINES: + break + + seen = set() + for candidate in candidates: + if candidate in seen: + continue + seen.add(candidate) + if check_math_result(candidate, truth): + return True + + return False + + +from typing import Optional + +import sympy +from pylatexenc import latex2text +from sympy.parsing import sympy_parser + +# Suppress noisy pylatexenc parse warnings that flood output on malformed LaTeX +logging.getLogger("pylatexenc").setLevel(logging.ERROR) + + +# Dan Hendrycks' code +def mathd_normalize_answer(answer: Optional[str]) -> Optional[str]: + if answer is None: + return None + answer = answer.strip() + try: + # Remove enclosing `\text{}`. + m = re.search(r"^\\text\{(?P.+?)\}$", answer) + if m is not None: + answer = m.group("text").strip() + return _strip_string(answer) + except: + return answer + + +def _strip_string(string): + def _fix_fracs(string): + substrs = string.split("\\frac") + new_str = substrs[0] + if len(substrs) > 1: + substrs = substrs[1:] + for substr in substrs: + new_str += "\\frac" + if substr[0] == "{": + new_str += substr + else: + try: + assert len(substr) >= 2 + except: + return string + a = substr[0] + b = substr[1] + if b != "{": + if len(substr) > 2: + post_substr = substr[2:] + new_str += "{" + a + "}{" + b + "}" + post_substr + else: + new_str += "{" + a + "}{" + b + "}" + else: + if len(substr) > 2: + post_substr = substr[2:] + new_str += "{" + a + "}" + b + post_substr + else: + new_str += "{" + a + "}" + b + string = new_str + return string + + def _fix_a_slash_b(string): + if len(string.split("/")) != 2: + return string + a = string.split("/")[0] + b = string.split("/")[1] + try: + a = int(a) + b = int(b) + assert string == "{}/{}".format(a, b) + new_string = "\\frac{" + str(a) + "}{" + str(b) + "}" + return new_string + except: + return string + + def _remove_right_units(string): + # "\\text{ " only ever occurs (at least in the val set) when describing units + if "\\text{ " in string: + splits = string.split("\\text{ ") + assert len(splits) == 2 + return splits[0] + else: + return string + + def _fix_sqrt(string): + if "\\sqrt" not in string: + return string + splits = string.split("\\sqrt") + new_string = splits[0] + for split in splits[1:]: + if split[0] != "{": + a = split[0] + new_substr = "\\sqrt{" + a + "}" + split[1:] + else: + new_substr = "\\sqrt" + split + new_string += new_substr + return new_string + + # linebreaks + string = string.replace("\n", "") + # print(string) + + # remove inverse spaces + string = string.replace("\\!", "") + # print(string) + + # replace \\ with \ + string = string.replace("\\\\", "\\") + # print(string) + + # replace tfrac and dfrac with frac + string = string.replace("tfrac", "frac") + string = string.replace("dfrac", "frac") + # print(string) + + # remove \left and \right + string = string.replace("\\left", "") + string = string.replace("\\right", "") + # print(string) + + # Remove circ (degrees) + string = string.replace("^{\\circ}", "") + string = string.replace("^\\circ", "") + + # remove dollar signs + string = string.replace("\\$", "") + + # remove units (on the right) + string = _remove_right_units(string) + + # remove percentage + string = string.replace(r"\%", "") + string = string.replace("%", "") + + # " 0." equivalent to " ." and "{0." equivalent to "{." Alternatively, add "0" if "." is the start of the string + string = string.replace(" .", " 0.") + string = string.replace("{.", "{0.") + # if empty, return empty string + if len(string) == 0: + return string + if string[0] == ".": + string = "0" + string + + # to consider: get rid of e.g. "k = " or "q = " at beginning + if len(string.split("=")) == 2: + if len(string.split("=")[0]) <= 2: + string = string.split("=")[1] + + # fix sqrt3 --> sqrt{3} + string = _fix_sqrt(string) + + # remove spaces + string = string.replace(" ", "") + + # \frac1b or \frac12 --> \frac{1}{b} and \frac{1}{2}, etc. Even works with \frac1{72} (but not \frac{72}1). Also does a/b --> \\frac{a}{b} + string = _fix_fracs(string) + + # manually change 0.5 --> \frac{1}{2} + if string == "0.5": + string = "\\frac{1}{2}" + + # NOTE: X/Y changed to \frac{X}{Y} in dataset, but in simple cases fix in case the model output is X/Y + string = _fix_a_slash_b(string) + + return string + + +# sympy might hang -- we don't care about trying to be lenient in these cases +BAD_SUBSTRINGS = ["^{", "^("] +BAD_REGEXES = [r"\^[0-9]+\^", r"\^[0-9][0-9]+"] +TUPLE_CHARS = "()[]" + + +def _sympy_parse(expr: str): + """Parses an expression with sympy.""" + py_expr = expr.replace("^", "**") + return sympy_parser.parse_expr( + py_expr, + transformations=( + sympy_parser.standard_transformations + + (sympy_parser.implicit_multiplication_application,) + ), + ) + + +@functools.lru_cache(maxsize=4096) +def _parse_latex(expr: str) -> str: + """Attempts to parse latex to an expression sympy can read.""" + expr = expr.replace("\\tfrac", "\\frac") + expr = expr.replace("\\dfrac", "\\frac") + + # pylatexenc can blow up on deeply nested macros; + # anything beyond 100 LaTeX commands is hallucinated output anyway + if len(re.findall(r"\\[a-zA-Z]+", expr)) > 100: + raise ValueError("Too many LaTeX macros for pylatexenc") + + expr = expr.replace("\\frac", " \\frac") # Play nice with mixed numbers. + expr = latex2text.LatexNodes2Text().latex_to_text(expr) + + # Replace the specific characters that this parser uses. + expr = expr.replace("√", "sqrt") + expr = expr.replace("π", "pi") + expr = expr.replace("∞", "inf") + expr = expr.replace("∪", "U") + expr = expr.replace("·", "*") + expr = expr.replace("×", "*") + + return expr.strip() + + +def _is_float(num: str) -> bool: + try: + float(num) + return True + except ValueError: + return False + + +def _is_int(x: float) -> bool: + try: + return abs(x - int(round(x))) <= 1e-7 + except: + return False + + +def _is_frac(expr: str) -> bool: + return bool(re.search(r"^-?[0-9]+.?/0*[1-9][0-9]*.?$", expr)) + + +def _str_is_int(x: str) -> bool: + try: + x = _strip_properly_formatted_commas(x) + x = float(x) + return abs(x - int(round(x))) <= 1e-7 + except: + return False + + +def _str_to_int(x: str) -> bool: + x = x.replace(",", "") + x = float(x) + return int(x) + + +def _inject_implicit_mixed_number(step: str): + """ + Automatically make a mixed number evalable + e.g. 7 3/4 => 7+3/4 + """ + p1 = re.compile(r"([0-9]) +([0-9])") + step = p1.sub("\\1+\\2", step) ## implicit mults + return step + + +def _strip_properly_formatted_commas(expr: str): + # We want to be careful because we don't want to strip tuple commas + p1 = re.compile(r"(\d)(,)(\d\d\d)($|\D)") + while True: + next_expr = p1.sub("\\1\\3\\4", expr) + if next_expr == expr: + break + expr = next_expr + return next_expr + + +def _normalize(expr: str) -> str: + """Normalize answer expressions.""" + if expr is None: + return None + + # Remove enclosing `\text{}`. + m = re.search(r"^\\text\{(?P.+?)\}$", expr) + if m is not None: + expr = m.group("text") + + expr = expr.replace("\\%", "%") + expr = expr.replace("\\$", "$") + expr = expr.replace("$", "") + expr = expr.replace("%", "") + expr = expr.replace(" or ", " , ") + expr = expr.replace(" and ", " , ") + + expr = expr.replace("million", "*10^6") + expr = expr.replace("billion", "*10^9") + expr = expr.replace("trillion", "*10^12") + + for unit in [ + "degree", + "cm", + "centimeter", + "meter", + "mile", + "second", + "minute", + "hour", + "day", + "week", + "month", + "year", + "foot", + "feet", + "inch", + "yard", + ]: + expr = re.sub(rf"{unit}(es)?(s)? *(\^[0-9]+)?", "", expr) + expr = re.sub(r"\^ *\\circ", "", expr) + + if len(expr) > 0 and expr[0] == "{" and expr[-1] == "}": + expr = expr[1:-1] + + expr = re.sub(",\\\\! *", "", expr) + if _is_float(expr) and _is_int(float(expr)): + expr = str(int(round(float(expr)))) + if "\\" in expr: + try: + expr = _parse_latex(expr) + except: + pass + + # edge case with mixed numbers and negative signs + expr = re.sub("- *", "-", expr) + + expr = _inject_implicit_mixed_number(expr) + expr = expr.replace(" ", "") + + # if we somehow still have latex braces here, just drop them + expr = expr.replace("{", "") + expr = expr.replace("}", "") + + # don't be case sensitive for text answers + expr = expr.lower() + + if _str_is_int(expr): + expr = str(_str_to_int(expr)) + + return expr + + +def count_unknown_letters_in_expr(expr: str): + expr = expr.replace("sqrt", "") + expr = expr.replace("frac", "") + letters_in_expr = set([x for x in expr if x.isalpha()]) + return len(letters_in_expr) + + +def should_allow_eval(expr: str): + # we don't want to try parsing unknown text or functions of more than two variables + if count_unknown_letters_in_expr(expr) > 2: + return False + + for bad_string in BAD_SUBSTRINGS: + if bad_string in expr: + return False + + for bad_regex in BAD_REGEXES: + if re.search(bad_regex, expr) is not None: + return False + + return True + + +def are_equal_under_sympy(ground_truth_normalized: str, given_normalized: str): + are_equal = False + try: + expr = f"({ground_truth_normalized})-({given_normalized})" + # Very long expressions may freeze sympy simplification and trigger OOM. + MAX_SYMPY_EXPR_LEN = 3000 + if len(expr) > MAX_SYMPY_EXPR_LEN: + return False + if should_allow_eval(expr): + sympy_diff = _sympy_parse(expr) + simplified = sympy.simplify(sympy_diff) + if simplified == 0: + are_equal = True + except: + pass + return are_equal + + +def split_tuple(expr: str): + """ + Split the elements in a tuple/interval, while handling well-formatted commas in large numbers + """ + expr = _strip_properly_formatted_commas(expr) + if len(expr) == 0: + return [] + if ( + len(expr) > 2 + and expr[0] in TUPLE_CHARS + and expr[-1] in TUPLE_CHARS + and all([ch not in expr[1:-1] for ch in TUPLE_CHARS]) + ): + elems = [elem.strip() for elem in expr[1:-1].split(",")] + else: + elems = [expr] + return elems + + +def grade_answer_sympy(given_answer: str, ground_truth: str) -> bool: + ground_truth_normalized = _normalize(ground_truth) + given_normalized = _normalize(given_answer) + + if ground_truth_normalized is None: + return False + + if ground_truth_normalized == given_normalized: + return True + + if len(given_normalized) == 0: + return False + + ground_truth_elems = split_tuple(ground_truth_normalized) + given_elems = split_tuple(given_normalized) + + if len(ground_truth_elems) > 1 and ( + ground_truth_normalized[0] != given_normalized[0] + or ground_truth_normalized[-1] != given_normalized[-1] + ): + is_correct = False + elif len(ground_truth_elems) != len(given_elems): + is_correct = False + else: + for ground_truth_elem, given_elem in zip(ground_truth_elems, given_elems): + if _is_frac(ground_truth_elem) and _is_frac(given_elem): + # if fractions aren't reduced, then shouldn't be marked as correct + # so, we don't want to allow sympy.simplify in this case + is_correct = ground_truth_elem == given_elem + elif _str_is_int(ground_truth_elem) != _str_is_int(given_elem): + # if the ground truth answer is an integer, we require the given answer to be a strict match (no sympy.simplify) + is_correct = False + else: + is_correct = are_equal_under_sympy(ground_truth_elem, given_elem) + if not is_correct: + break + + return is_correct + + +def grade_answer_mathd(given_answer: str, ground_truth: str) -> bool: + ground_truth_normalized_mathd = mathd_normalize_answer(ground_truth) + given_answer_normalized_mathd = mathd_normalize_answer(given_answer) + + # be at least as lenient as mathd + if ground_truth_normalized_mathd == given_answer_normalized_mathd: + return True + return False + + +def check_math_result(model_response, solution): + result_mathd = grade_answer_mathd(model_response, solution) + result_sympy = grade_answer_sympy(model_response, solution) + return result_mathd or result_sympy diff --git a/packages/pipette-scores/pipette_scores/scoring/mmbench_en_dev.py b/packages/pipette-scores/pipette_scores/scoring/mmbench_en_dev.py new file mode 100644 index 0000000..0a94ab7 --- /dev/null +++ b/packages/pipette-scores/pipette_scores/scoring/mmbench_en_dev.py @@ -0,0 +1,83 @@ +"""MMBench EN dev scorer — VLMEvalKit can_infer + circular (rotation) grouping. + +VLMEvalKit evaluates MMBench with `mcq_circular_eval`: the dataset contains +rotated copies of each base question (rotation rows share `index % 1e6`), and +a base question only counts as a hit when EVERY rotation is answered +correctly. Extraction is `can_infer`; unparseable = wrong (no judge). On a +dataset without rotation rows this degenerates to vanilla per-question +accuracy. + +Groups are enumerated from the dataset, so a partial submission cannot +inflate the headline: an unsubmitted rotation counts as wrong and makes its +base question a miss. +""" + +from typing import Any + +from pipette_scores.scoring.completion_text import ReasoningTally, split_reasoning +from pipette_scores.scoring.vlm_answer import option_hit +from pipette_scores.scoring.vlm_upstream.multiple_choice import build_choices +from pipette_scores.types import SampleCompletion, SampleScore, VlmSample + + +def score( + completions: list[SampleCompletion], + samples: list[VlmSample], + *, + enable_thinking: bool | None = None, + opens_reasoning: bool = False, +) -> tuple[list[SampleScore], dict[str, Any]]: + by_id = {s.id: s for s in samples} + scores: list[SampleScore] = [] + blanked = 0 + tally = ReasoningTally() + fallbacks = 0 + hits: dict[str, bool] = {} + for c in completions: + sample = by_id[c.id] + graded = tally.add(split_reasoning(c, enable_thinking, opens_reasoning=opens_reasoning)) + blanked += graded.blanked + text = graded.answer + choices = build_choices(sample.raw_doc) + hit, fallback = option_hit(text, choices, str(sample.raw_doc.get("answer", ""))) + fallbacks += fallback + scores.append(SampleScore(id=c.id, is_correct=hit)) + hits[c.id] = hit + + # Circular groups are built from the DATASET rows (all base questions), + # not just the submitted ones: a rotation with no submitted completion + # counts as wrong, so a group with any missing rotation is a miss (no + # partial-submission inflation). + per_base: dict[int, list[bool]] = {} + per_base_category: dict[int, str] = {} + for s in samples: + base = int(s.raw_doc.get("index", 0)) % 1_000_000 + per_base.setdefault(base, []).append(hits.get(s.id, False)) + per_base_category.setdefault(base, str(s.raw_doc.get("category", "") or "unknown")) + + total = len(scores) + correct = sum(1 for s in scores if s.is_correct) + num_questions = len(per_base) + base_hits = {base: all(rots) for base, rots in per_base.items()} + num_hits = sum(1 for h in base_hits.values() if h) + hit_rate = num_hits / num_questions if num_questions else 0.0 + + cat_buckets: dict[str, list[bool]] = {} + for base, h in base_hits.items(): + cat_buckets.setdefault(per_base_category[base], []).append(h) + + context = { + # Circular accuracy over base questions (VLMEvalKit MMBench headline). + "accuracy": round(hit_rate, 6), + "leaderboard_score": round(hit_rate * 100, 4), + "num_questions": num_questions, + "num_hits": num_hits, + "samples_missing": len(samples) - len(hits), + "blanked_truncated": blanked, + **tally.context(), + "fallback_answers": fallbacks, + # Diagnostic: hit rate over the SUBMITTED rotations only. + "per_rotation_accuracy": round(correct / total, 4) if total else 0.0, + "per_category_accuracy": {cat: round(sum(v) / len(v), 6) for cat, v in sorted(cat_buckets.items()) if v}, + } + return scores, context diff --git a/packages/pipette-scores/pipette_scores/scoring/mme.py b/packages/pipette-scores/pipette_scores/scoring/mme.py new file mode 100644 index 0000000..dc61f65 --- /dev/null +++ b/packages/pipette-scores/pipette_scores/scoring/mme.py @@ -0,0 +1,89 @@ +"""MME scorer — VLMEvalKit Y/N extraction + MME rating (perception/cognition).""" + +from typing import Any + +from pipette_scores.scoring.completion_text import ReasoningTally, split_reasoning +from pipette_scores.scoring.vlm_answer import yes_no +from pipette_scores.scoring.vlm_upstream.yorn import mme_rating +from pipette_scores.types import SampleCompletion, SampleScore, VlmSample + +# Pairing key: VLMEvalKit's MME_rating groups the two yes/no questions of each +# image by `image_path`; the lmms-eval MME export carries the same image +# identity in `question_id` (the image's relative path, shared by both +# questions). Prefer `image_path`, fall back to `question_id`. +_PAIR_KEYS = ("image_path", "question_id") + + +def _image_key(raw_doc: dict) -> str: + for k in _PAIR_KEYS: + v = raw_doc.get(k) + if v is not None and str(v) != "": + return str(v) + # Last resort: no pairing key -> every question its own "image", which + # collapses acc_plus to acc; dataset generation must provide a pair key. + return str(raw_doc.get("index", id(raw_doc))) + + +def score( + completions: list[SampleCompletion], + samples: list[VlmSample], + *, + enable_thinking: bool | None = None, + opens_reasoning: bool = False, +) -> tuple[list[SampleScore], dict[str, Any]]: + by_id = {s.id: s for s in samples} + scores: list[SampleScore] = [] + blanked = 0 + tally = ReasoningTally() + fallbacks = 0 + hits: dict[str, bool] = {} + for c in completions: + sample = by_id[c.id] + graded = tally.add(split_reasoning(c, enable_thinking, opens_reasoning=opens_reasoning)) + blanked += graded.blanked + text = graded.answer + extracted, fallback = yes_no(text) + fallbacks += fallback + gt = str(sample.raw_doc.get("answer", "")).strip().capitalize() # 'Yes' / 'No' + hit = extracted == gt and gt in ("Yes", "No") + scores.append(SampleScore(id=c.id, is_correct=hit)) + hits[c.id] = hit + + # Rating records cover ALL dataset samples so a partial submission cannot + # inflate the headline: a sample with no submitted completion scores 0.0, + # contributing 0 to both acc (it stays in the denominator) and acc_plus + # (its image pair can no longer be fully correct) — exactly like a wrong + # answer. + records = [ + { + "category": str(s.raw_doc.get("category", "")), + "image_key": _image_key(s.raw_doc), + "score": 1.0 if hits.get(s.id, False) else 0.0, + } + for s in samples + ] + + rating = mme_rating(records) + perception = rating.pop("perception") + cognition = rating.pop("cognition") + mme_total = perception + cognition + + total = len(scores) + correct = sum(1 for s in scores if s.is_correct) + context = { + # NOTE: for MME, context["accuracy"] is the MME total score / 2800 + # (the conventional 0-2800 scale normalized to 0-1), not raw hit rate. + "accuracy": round(mme_total / 2800, 6), + "leaderboard_score": round(mme_total, 2), + "mme_perception_score": round(perception, 2), + "mme_cognition_score": round(cognition, 2), + "mme_total_score": round(mme_total, 2), + "samples_missing": sum(1 for s in samples if s.id not in hits), + "blanked_truncated": blanked, + **tally.context(), + "fallback_answers": fallbacks, + "per_category_score": {k: round(v, 2) for k, v in sorted(rating.items())}, + # Diagnostic: hit rate over the SUBMITTED completions only. + "per_sample_accuracy": round(correct / total, 4) if total else 0.0, + } + return scores, context diff --git a/packages/pipette-scores/pipette_scores/scoring/mmmu_val.py b/packages/pipette-scores/pipette_scores/scoring/mmmu_val.py new file mode 100644 index 0000000..001ddc4 --- /dev/null +++ b/packages/pipette-scores/pipette_scores/scoring/mmmu_val.py @@ -0,0 +1,68 @@ +"""MMMU (val) scorer — VLMEvalKit MCQ path with MMMU_preproc. + +Open questions are re-formulated as 2-choice MCQ (A = gold, B = 'Other +Answers') per VLMEvalKit's `MMMU_preproc`; the official-MMMU +`parse_open_response` path is deliberately NOT used. Unparseable = wrong +(no judge, no random fallback). Headline = micro accuracy. +""" + +from typing import Any + +from pipette_scores.scoring.completion_text import ReasoningTally, split_reasoning +from pipette_scores.scoring.vlm_answer import option_hit +from pipette_scores.scoring.vlm_upstream.multiple_choice import ( + mmmu_preproc_choices, + mmmu_preproc_gt, +) +from pipette_scores.types import SampleCompletion, SampleScore, VlmSample + + +def score( + completions: list[SampleCompletion], + samples: list[VlmSample], + *, + enable_thinking: bool | None = None, + opens_reasoning: bool = False, +) -> tuple[list[SampleScore], dict[str, Any]]: + # Headline accuracy is over the FULL dataset: a sample with no submitted + # completion counts as wrong (no partial-submission inflation). + by_id = {s.id: s for s in samples} + scores: list[SampleScore] = [] + blanked = 0 + tally = ReasoningTally() + fallbacks = 0 + hits: dict[str, bool] = {} + n_open = 0 + for c in completions: + sample = by_id[c.id] + graded = tally.add(split_reasoning(c, enable_thinking, opens_reasoning=opens_reasoning)) + blanked += graded.blanked + text = graded.answer + choices = mmmu_preproc_choices(sample.raw_doc) + if choices.get("B") == "Other Answers" and "C" not in choices: + n_open += 1 + gt = mmmu_preproc_gt(sample.raw_doc.get("answer", "")) + hit, fallback = option_hit(text, choices, gt) + fallbacks += fallback + scores.append(SampleScore(id=c.id, is_correct=hit)) + hits[c.id] = hit + + cat_buckets: dict[str, list[bool]] = {} + for s in samples: + cat = str(s.raw_doc.get("category", "") or "unknown") + cat_buckets.setdefault(cat, []).append(hits.get(s.id, False)) + + total = len(samples) + correct = sum(1 for h in hits.values() if h) + accuracy = correct / total if total else 0.0 + context = { + "accuracy": round(accuracy, 6), + "leaderboard_score": round(accuracy * 100, 4), + "samples_missing": total - len(hits), + "blanked_truncated": blanked, + **tally.context(), + "fallback_answers": fallbacks, + "num_open_questions_reformulated": n_open, + "per_category_accuracy": {cat: round(sum(v) / len(v), 6) for cat, v in sorted(cat_buckets.items()) if v}, + } + return scores, context diff --git a/packages/pipette-scores/pipette_scores/scoring/mmstar.py b/packages/pipette-scores/pipette_scores/scoring/mmstar.py new file mode 100644 index 0000000..c91e84d --- /dev/null +++ b/packages/pipette-scores/pipette_scores/scoring/mmstar.py @@ -0,0 +1,57 @@ +"""MMStar scorer — VLMEvalKit can_infer MCQ matching, micro accuracy headline.""" + +from typing import Any + +from pipette_scores.scoring.completion_text import ReasoningTally, split_reasoning +from pipette_scores.scoring.vlm_answer import option_hit +from pipette_scores.scoring.vlm_upstream.multiple_choice import build_choices +from pipette_scores.types import SampleCompletion, SampleScore, VlmSample + + +def score( + completions: list[SampleCompletion], + samples: list[VlmSample], + *, + enable_thinking: bool | None = None, + opens_reasoning: bool = False, +) -> tuple[list[SampleScore], dict[str, Any]]: + # Headline = micro mean(hit) over all DATASET samples (VLMEvalKit + # `report_acc` 'Overall'), NOT the macro mean of per-L2-category + # accuracies. On the full, category-balanced MMStar the two coincide; + # micro is the exact VLMEvalKit definition. A dataset sample with no + # submitted completion counts as wrong (no partial-submission inflation). + by_id = {s.id: s for s in samples} + scores: list[SampleScore] = [] + blanked = 0 + tally = ReasoningTally() + fallbacks = 0 + hits: dict[str, bool] = {} + for c in completions: + sample = by_id[c.id] + graded = tally.add(split_reasoning(c, enable_thinking, opens_reasoning=opens_reasoning)) + blanked += graded.blanked + text = graded.answer + choices = build_choices(sample.raw_doc) + hit, fallback = option_hit(text, choices, str(sample.raw_doc.get("answer", ""))) + fallbacks += fallback + scores.append(SampleScore(id=c.id, is_correct=hit)) + hits[c.id] = hit + + l2_buckets: dict[str, list[bool]] = {} + for s in samples: + l2 = str(s.raw_doc.get("l2_category") or s.raw_doc.get("l2-category") or "unknown") + l2_buckets.setdefault(l2, []).append(hits.get(s.id, False)) + + total = len(samples) + correct = sum(1 for h in hits.values() if h) + accuracy = correct / total if total else 0.0 + context = { + "accuracy": round(accuracy, 6), + "leaderboard_score": round(accuracy * 100, 4), + "samples_missing": total - len(hits), + "blanked_truncated": blanked, + **tally.context(), + "fallback_answers": fallbacks, + "per_l2_category_accuracy": {cat: round(sum(v) / len(v), 6) for cat, v in sorted(l2_buckets.items()) if v}, + } + return scores, context diff --git a/packages/pipette-scores/pipette_scores/scoring/ocrbench.py b/packages/pipette-scores/pipette_scores/scoring/ocrbench.py new file mode 100644 index 0000000..c214545 --- /dev/null +++ b/packages/pipette-scores/pipette_scores/scoring/ocrbench.py @@ -0,0 +1,73 @@ +"""OCRBench scorer — VLMEvalKit category-keyed matching; headline = Final Score / 1000.""" + +from typing import Any + +from pipette_scores.scoring.completion_text import ReasoningTally, split_reasoning +from pipette_scores.scoring.vlm_upstream.ocrbench import ( + ocrbench_final_score, + ocrbench_match, + parse_answers, +) +from pipette_scores.types import SampleCompletion, SampleScore, VlmSample + +# OCRBench has exactly 1000 questions; the conventional 'Final Score' is the +# number correct (denominator fixed at 1000 regardless of subset size). +_OCRBENCH_TOTAL = 1000 + + +def _category(raw_doc: dict) -> str: + # VLMEvalKit column is `category`; the lmms-eval export calls the same + # values `question_type`. + return str(raw_doc.get("category") or raw_doc.get("question_type") or "") + + +def score( + completions: list[SampleCompletion], + samples: list[VlmSample], + *, + enable_thinking: bool | None = None, + opens_reasoning: bool = False, +) -> tuple[list[SampleScore], dict[str, Any]]: + by_id = {s.id: s for s in samples} + submitted_ids = {c.id for c in completions} + scores: list[SampleScore] = [] + blanked = 0 + tally = ReasoningTally() + per_category_correct: dict[str, int] = {} + for c in completions: + sample = by_id[c.id] + graded = tally.add(split_reasoning(c, enable_thinking, opens_reasoning=opens_reasoning)) + blanked += graded.blanked + text = graded.answer + category = _category(sample.raw_doc) + answers = parse_answers(sample.raw_doc.get("answer", "")) + hit = ocrbench_match(text, answers, category) + scores.append(SampleScore(id=c.id, is_correct=hit)) + if hit: + per_category_correct[category] = per_category_correct.get(category, 0) + 1 + + final = ocrbench_final_score(per_category_correct) + + total = len(scores) + correct = sum(1 for s in scores if s.is_correct) + context = { + # NOTE: for OCRBench, context["accuracy"] is Final Score / 1000 (the + # conventional fixed-denominator scale); on a subset this is NOT the + # per-sample hit rate (see per_sample_accuracy). + "accuracy": round(final["Final Score"] / _OCRBENCH_TOTAL, 6), + "leaderboard_score": round(final["Final Score"], 2), + "final_score": round(final["Final Score"], 2), + "final_score_norm": round(final["Final Score Norm"], 2), # 0-100 scale + # A dataset sample with no submitted completion simply contributes no + # correct count; the /1000 denominator is fixed, so a partial + # submission cannot inflate the headline. + "samples_missing": sum(1 for s in samples if s.id not in submitted_ids), + "blanked_truncated": blanked, + **tally.context(), + "per_category_score": { + k: round(v, 2) for k, v in final.items() if k not in ("Final Score", "Final Score Norm") + }, + # Diagnostic: hit rate over the SUBMITTED completions only. + "per_sample_accuracy": round(correct / total, 4) if total else 0.0, + } + return scores, context diff --git a/packages/pipette-scores/pipette_scores/scoring/pope.py b/packages/pipette-scores/pipette_scores/scoring/pope.py new file mode 100644 index 0000000..0ae9f8e --- /dev/null +++ b/packages/pipette-scores/pipette_scores/scoring/pope.py @@ -0,0 +1,84 @@ +"""POPE scorer — VLMEvalKit Y/N extraction; headline metric is F1, not accuracy.""" + +from typing import Any + +from pipette_scores.scoring.completion_text import ReasoningTally, split_reasoning +from pipette_scores.scoring.vlm_answer import yes_no +from pipette_scores.scoring.vlm_upstream.yorn import pope_rating +from pipette_scores.types import SampleCompletion, SampleScore, VlmSample + + +def _gt(sample: VlmSample) -> str: + gt = sample.raw_doc.get("answer") + if not isinstance(gt, str): + gt = str(sample.answer or "") + return gt.strip().capitalize() # 'Yes' / 'No' + + +def score( + completions: list[SampleCompletion], + samples: list[VlmSample], + *, + enable_thinking: bool | None = None, + opens_reasoning: bool = False, +) -> tuple[list[SampleScore], dict[str, Any]]: + by_id = {s.id: s for s in samples} + completions_by_id = {c.id: c for c in completions} + scores: list[SampleScore] = [] + blanked = 0 + tally = ReasoningTally() + fallbacks = 0 + extracted_by_id: dict[str, str] = {} + for c in completions: + sample = by_id[c.id] + graded = tally.add(split_reasoning(c, enable_thinking, opens_reasoning=opens_reasoning)) + blanked += graded.blanked + text = graded.answer + gt = _gt(sample) + extracted, fallback = yes_no(text) + fallbacks += fallback + extracted_by_id[c.id] = extracted + scores.append(SampleScore(id=c.id, is_correct=extracted == gt and gt in ("Yes", "No"))) + + # The confusion matrix is built over ALL dataset samples so a partial + # submission cannot inflate the headline F1: a sample with no submitted + # completion counts as the wrong answer to it, a false negative or a + # false positive. 'Unknown' (a 'No' prediction, as upstream reads an + # unparseable answer) would make every missing negative a free true + # negative, which F1 does not see. + records: list[dict] = [] + for sample in samples: + gt = _gt(sample) + records.append( + { + "answer": gt, + "extracted": extracted_by_id.get(sample.id, "No" if gt == "Yes" else "Yes"), + "category": str(sample.raw_doc.get("category", "") or ""), + } + ) + + rating = pope_rating(records) + overall = rating["Overall"] + + total = len(scores) + correct = sum(1 for s in scores if s.is_correct) + context = { + # NOTE: for POPE, context["accuracy"] is the F1 SCORE (VLMEvalKit's + # 'Overall' headline for POPE), on a 0-1 scale — not raw accuracy. + "accuracy": round(overall["f1"], 6), + "leaderboard_score": round(overall["f1"] * 100, 4), + "f1": round(overall["f1"], 6), + "precision": round(overall["precision"], 6), + "recall": round(overall["recall"], 6), + "yes_no_accuracy": round(overall["accuracy"], 6), + "samples_missing": sum(1 for s in samples if s.id not in completions_by_id), + "blanked_truncated": blanked, + **tally.context(), + "fallback_answers": fallbacks, + "per_category": { + k: {m: round(v, 6) for m, v in stats.items()} for k, stats in rating.items() if k != "Overall" + }, + # Diagnostic: hit rate over the SUBMITTED completions only. + "per_sample_accuracy": round(correct / total, 4) if total else 0.0, + } + return scores, context diff --git a/packages/pipette-scores/pipette_scores/scoring/profile_answers.py b/packages/pipette-scores/pipette_scores/scoring/profile_answers.py new file mode 100644 index 0000000..c29fcc1 --- /dev/null +++ b/packages/pipette-scores/pipette_scores/scoring/profile_answers.py @@ -0,0 +1,118 @@ +"""The answer a model family's VLMEvalKit wrapper cuts out of a reply before the +benchmark's own scoring sees it, for the prompt profiles whose wrapper does. + +VLMEvalKit's MiniCPM-V wrappers prompt step by step to an ``Answer:`` line (the +``minicpm_v_cot`` profile) and then reduce the reply to that answer: MiniCPM-V +4.6's ``extract_answer`` (vlmeval/vlm/minicpm_v.py:1504-1529 at 8eca8e5), +ported here line for line. MiniCPM-V 4.5's (:971-999) reads the last +``Answer:`` letter of the whole reply, which this rule also reads, since it +cuts at a closing ```` only when there is one. A reply that ends +inside an unclosed reasoning block therefore still gives its stated answer, as +it does under VLMEvalKit; without this the reasoning rules would drop the whole +reply, answer included. + +Kimi-VL's wrapper (``KimiVL.generate_inner``, vlmeval/vlm/kimi_vl.py:140-143 +at 8eca8e5) keeps the text after the first ``◁/think▷`` (nothing at all when +``◁think▷`` opened and never closed) and, on MMMU and MMStar, the content of the +last ``\\boxed{}`` in it. The first part is what the reasoning rules do for every +family; the boxed part is ported below. + +Every other profile's wrapper leaves the reply as it is, and so does this +module. OCRBench is left whole on every profile, as upstream leaves it. The +reasoning of the other families is cut by the scorers' thinking-tag rules +(scoring.completion_text), not here. +""" + +import re + +from pipette_scores.types import EvalId, SampleCompletion + +# DATASET_TYPE upstream: the multiple-choice benchmarks, and the yes/no ones. +_MCQ = frozenset({EvalId.MMSTAR, EvalId.REALWORLDQA, EvalId.MMBENCH_EN_DEV, EvalId.MMMU_VAL}) +_YES_NO = frozenset({EvalId.MME, EvalId.POPE}) + +_ANSWER_TAG = re.compile(r"(.*?)", re.DOTALL) +_MCQ_ANSWER = re.compile(r"(?:Answer(?:\s+is|:)|答案(?:是|为)?[::]?)\s*([A-Ja-j])(?![A-Za-z])", re.DOTALL) +_TAIL_ANSWER = re.compile(r"Answer:\s*(.*)\s*$", re.DOTALL) + + +def minicpm_v_cot(benchmark: EvalId, reply: str) -> str: + """MiniCPM-V-4.6's ``extract_answer`` for a benchmark it prompts with CoT.""" + if "" in reply: + reply = reply.rsplit("", 1)[-1].strip() + if tag := _ANSWER_TAG.search(reply): + return tag.group(1).strip() + if benchmark in _MCQ: + if matches := _MCQ_ANSWER.findall(reply): + return matches[-1].strip() + elif benchmark in _YES_NO: + if match := _TAIL_ANSWER.search(reply): + return match.group(1) + return reply + + +def _last_boxed(answer: str) -> str: + """Kimi's ``extract_boxed_content``: the content of the last ``\\boxed{``, + braces balanced; the text itself when it has none or its braces do not + balance.""" + start = answer.rfind("\\boxed{") + if start == -1: + return answer + start += len("\\boxed{") + depth = 1 + end = start + while end < len(answer): + if answer[end] == "{": + depth += 1 + elif answer[end] == "}": + depth -= 1 + if depth == 0: + break + end += 1 + return answer[start:end] if depth == 0 else answer + + +def kimi_vl(benchmark: EvalId, reply: str) -> str: + """Kimi-VL-A3B-Thinking-2506's reply with, on MMMU and MMStar, the summary + after its thought cut down to the boxed answer in it, as its VLMEvalKit wrapper + reduces it (``extract_summary`` takes the text after the first ``◁/think▷``, + then ``extract_boxed_content`` the last ``\\boxed{}``). + + The thought stays in the reply, closed, so the reasoning rules still see it + and read the rest as they do for every family; text before its opener is + dropped, since the summary is only what follows the close. A thought that + never closes has no summary, and the reply is left for the rules to cut.""" + from pipette_scores.dataset_catalog.prompt_profiles import KIMI_VL_BENCHMARKS + + if "◁/think▷" not in reply: + if benchmark not in KIMI_VL_BENCHMARKS: + return reply + return reply if "◁think▷" in reply else _last_boxed(reply) + head, _, summary = reply.partition("◁/think▷") + if "◁think▷" in head: + head = head[head.index("◁think▷") :] + if benchmark in KIMI_VL_BENCHMARKS: + summary = _last_boxed(summary.strip()) + return f"{head}◁/think▷{summary}" + + +# Profile -> the wrapper's extraction. A profile not here leaves replies whole. +EXTRACTIONS = { + "minicpm_v_cot": minicpm_v_cot, + "kimi_vl": kimi_vl, +} + + +def apply(prompt_profile: str | None, benchmark: str, completions: list[SampleCompletion]) -> list[SampleCompletion]: + """The completions as the profile's wrapper hands them to the benchmark's + scoring: each reply reduced to its answer where the wrapper does that.""" + extract = EXTRACTIONS.get(prompt_profile or "") + if extract is None: + return completions + try: + eval_id = EvalId(benchmark) + except ValueError: + return completions + if eval_id == EvalId.OCRBENCH: + return completions + return [c.model_copy(update={"completion": extract(eval_id, c.completion)}) for c in completions] diff --git a/packages/pipette-scores/pipette_scores/scoring/realworldqa.py b/packages/pipette-scores/pipette_scores/scoring/realworldqa.py new file mode 100644 index 0000000..9c1151f --- /dev/null +++ b/packages/pipette-scores/pipette_scores/scoring/realworldqa.py @@ -0,0 +1,48 @@ +"""RealWorldQA scorer — VLMEvalKit ImageMCQ path (can_infer over A-D choices).""" + +from typing import Any + +from pipette_scores.scoring.completion_text import ReasoningTally, split_reasoning +from pipette_scores.scoring.vlm_answer import option_hit +from pipette_scores.scoring.vlm_upstream.multiple_choice import build_choices +from pipette_scores.types import SampleCompletion, SampleScore, VlmSample + + +def score( + completions: list[SampleCompletion], + samples: list[VlmSample], + *, + enable_thinking: bool | None = None, + opens_reasoning: bool = False, +) -> tuple[list[SampleScore], dict[str, Any]]: + # Headline accuracy is over the FULL dataset: a sample with no submitted + # completion counts as wrong (no partial-submission inflation). + by_id = {s.id: s for s in samples} + scores: list[SampleScore] = [] + blanked = 0 + tally = ReasoningTally() + fallbacks = 0 + hits: dict[str, bool] = {} + for c in completions: + sample = by_id[c.id] + graded = tally.add(split_reasoning(c, enable_thinking, opens_reasoning=opens_reasoning)) + blanked += graded.blanked + text = graded.answer + choices = build_choices(sample.raw_doc) + hit, fallback = option_hit(text, choices, str(sample.raw_doc.get("answer", ""))) + fallbacks += fallback + scores.append(SampleScore(id=c.id, is_correct=hit)) + hits[c.id] = hit + + total = len(samples) + correct = sum(1 for h in hits.values() if h) + accuracy = correct / total if total else 0.0 + context = { + "accuracy": round(accuracy, 6), + "leaderboard_score": round(accuracy * 100, 4), + "samples_missing": total - len(hits), + "blanked_truncated": blanked, + **tally.context(), + "fallback_answers": fallbacks, + } + return scores, context diff --git a/packages/pipette-scores/pipette_scores/scoring/vlm_answer.py b/packages/pipette-scores/pipette_scores/scoring/vlm_answer.py new file mode 100644 index 0000000..f69edc6 --- /dev/null +++ b/packages/pipette-scores/pipette_scores/scoring/vlm_answer.py @@ -0,0 +1,181 @@ +"""Deterministic fallbacks for answers VLMEvalKit's exact matching cannot read. + +A pipette extension, not part of the VLMEvalKit port: upstream hands these +answers to an LLM judge, and pipette never uses one. The vendored rule runs +first (``can_infer`` for options, ``YOrN_Extraction`` for yes/no), so every +answer it reads is scored exactly as upstream scores it. Only when it finds +nothing do these rules look for an answer the text commits to, in order: + +1. the last explicit answer statement ("the correct answer is: **B**", + "Final Answer: No", a JSON field `"answer": "C"`), or the last line that + is that field alone, its key in backticks (`` `answer`: "C" ``), +2. the last ``\\boxed{...}``, +3. an answer given first and explained after: a first line that opens with + an option letter ("B. Not the same") or a yes/no, or a line that is exactly + an option ("C. painter"), unless the text walks through several options, +4. a quoted answer: the whole answer one quoted letter (`"B"`, the field's + value given without its key), or, after text that names no option letter, + a last line that is one quoted capital letter or one quoted option + (`"B. Not the same"`), +5. the vendored rule again, on the last non-empty line alone. + +Reasoning models are graded on the text after their reasoning is removed +(``grading_text`` and ``remove_thinking_tags``), so the same rules apply to +both. An answer none of them reads stays wrong. +""" + +import re + +from pipette_scores.scoring.vlm_upstream.matching_util import can_infer +from pipette_scores.scoring.vlm_upstream.yorn import YOrN_Extraction + +# "answer", "final answer", "correct option", ... then an optional "is" and +# colon, and any markdown before the answer itself. Quotes around the key and +# the value read a JSON field, `"answer": "C"`, the format Qwen3.5's model card +# asks for on multiple-choice questions. +_STATEMENT = ( + r"(?i:\b(?:final\s+answer|correct\s+answer|best\s+answer|correct\s+option|" + r"correct\s+choice|best\s+option|answer)\b" + r"(?:\s+(?:is|would\s+be|should\s+be))?)" + r"[\"']?\s*[::]?\s*(?:>\s*)?[*_`\"']*\s*[(\[]?\s*" +) +# An option letter is uppercase and not the start of a word ("All of ..."). +_OPTION_STATEMENT = re.compile(_STATEMENT + r"([A-Z])(?![A-Za-z0-9])") +# Not "answer yes or no": a reply that names both is saying it cannot choose. +_YES_NO_STATEMENT = re.compile(_STATEMENT + r"(?i:(yes|no))\b(?!\s*(?i:or|/)\s*(?i:yes|no)\b)") +_BOXED = re.compile(r"\\boxed\{\s*(?:\\text\{)?\s*([^{}]*?)\s*\}?\s*\}") +_MARKDOWN = re.compile(r"[*_`>#]+") + + +# A line that opens with an option letter: "B. ...", "(C) ...", "**D**". +_OPTION_LINE = re.compile(r"^[*_`>#\s]*\(?([A-Z])(?:\)|\.|:|\*\*|$)\s*(.*)$") + +# The shapes Qwen3.5 writes the `"answer": "C"` field its card asks for in, each +# read only as a whole line, so prose that mentions the format or refuses to +# choose is never one: +# - the field, its key quoted or in backticks: `` `answer`: "C" ``, `` `answer: "C"` ``; +_FIELD_LINE = r"^[\s{]*`?[\"']?answer[\"']?`?\s*[::]?\s*[`\"'](VALUE)[`\"']\s*`?\s*[,}]?\s*$" +_ANSWER_FIELD_LINE = re.compile(_FIELD_LINE.replace("VALUE", "[A-Za-z]"), re.I | re.M) +_YES_NO_FIELD_LINE = re.compile(_FIELD_LINE.replace("VALUE", "yes|no"), re.I | re.M) +# - the field's value alone, the whole answer: `"B"`, `"a"`, an unclosed `"B`; +_QUOTED_LETTER = re.compile(r"^[\"'“‘]([A-Za-z])[\"'”’]?\.?$") +# - one option, quoted, as the last line: `"B. Not the same"`. +_QUOTED_OPTION_LINE = re.compile(r"^[\"'“‘]\s*([A-Z])[.)]\s*(.+?)\s*[\"'”’]\.?$") + + +def _last_commitment(text: str, patterns: tuple[re.Pattern[str], ...], valid) -> str | None: + """The value of the last statement or answer field in the text, whichever + comes last: a reply that revises its answer is read at its final word.""" + found = [(m.start(1), m.group(1)) for p in patterns for m in p.finditer(text) if valid(m.group(1))] + return max(found)[1] if found else None + + +def _lines(text: str) -> list[str]: + return [ln.strip() for ln in text.strip().splitlines() if ln.strip()] + + +def _option_lines(text: str, choices: dict[str, str]) -> list[tuple[str, str]]: + """Each line that opens with an option letter, with the rest of the line.""" + out = [] + for line in _lines(text): + match = _OPTION_LINE.match(line) + if match and match.group(1) in choices: + out.append((match.group(1), _MARKDOWN.sub(" ", match.group(2)).strip())) + return out + + +def _leading_option(text: str, choices: dict[str, str]) -> str | None: + """The option an answer opens with or states as a line of its own, when the + text does not walk through several options.""" + lines = _option_lines(text, choices) + if len({letter for letter, _ in lines}) != 1: + return None + first = _lines(text)[0] + match = _OPTION_LINE.match(first) + if match and match.group(1) in choices: + return match.group(1) + letter, rest = lines[0] + return letter if rest and _words(rest) == _words(choices[letter]) else None + + +def _words(value: str) -> str: + return re.sub(r"[^a-z0-9]+", " ", value.lower()).strip() + + +def _last_line(text: str) -> str: + lines = [ln.strip() for ln in text.strip().splitlines() if ln.strip()] + return _MARKDOWN.sub(" ", lines[-1]).strip() if lines else "" + + +def _option_in(value: str, choices: dict[str, str]) -> str | None: + match = re.match(r"\(?\s*([A-Z])(?![A-Za-z0-9])", value.strip()) + return match.group(1) if match and match.group(1) in choices else None + + +def option(text: str, choices: dict[str, str]) -> tuple[str | None, bool]: + """The option letter the answer gives, and whether a fallback found it.""" + upstream = can_infer(text, choices) + if upstream: + return upstream, False + stated = _last_commitment(text, (_OPTION_STATEMENT, _ANSWER_FIELD_LINE), lambda v: v.upper() in choices) + if stated: + return stated.upper(), True + boxed = [o for b in _BOXED.findall(text) if (o := _option_in(b, choices))] + if boxed: + return boxed[-1], True + leading = _leading_option(text, choices) + if leading: + return leading, True + quoted = _quoted_answer(text, choices) + if quoted: + return quoted, True + last = can_infer(_last_line(text), choices) + return (last, True) if last and last in choices else (None, False) + + +def _quoted_answer(text: str, choices: dict[str, str]) -> str | None: + """The option a quoted answer gives: the whole answer one quoted letter + (any case), or a last line that is one quoted capital letter, or one quoted + option, after text that names no option letter.""" + lines = _lines(text) + if not lines: + return None + whole = _QUOTED_LETTER.match(text.strip()) + if whole: + return whole.group(1).upper() if whole.group(1).upper() in choices else None + if any(re.search(rf"\b{letter}\b", line) for line in lines[:-1] for letter in choices): + return None + letter = _QUOTED_LETTER.match(lines[-1]) + if letter and letter.group(1) in choices: + return letter.group(1) + option_line = _QUOTED_OPTION_LINE.match(lines[-1]) + if option_line and option_line.group(1) in choices: + letter, rest = option_line.groups() + return letter if _words(rest) == _words(choices[letter]) else None + return None + + +def option_hit(text: str, choices: dict[str, str], gt: str) -> tuple[bool, bool]: + """Whether the answer gives the ground-truth letter, and whether a fallback + read it. Unreadable is wrong, as under upstream's exact matching.""" + letter, fallback = option(text, choices) + return letter is not None and letter == str(gt).strip(), fallback + + +def yes_no(text: str) -> tuple[str, bool]: + """'Yes', 'No' or 'Unknown', and whether a fallback read it.""" + upstream = YOrN_Extraction(text) + if upstream != "Unknown": + return upstream, False + stated = _last_commitment(text, (_YES_NO_STATEMENT, _YES_NO_FIELD_LINE), lambda _: True) + if stated: + return stated.capitalize(), True + boxed = [b.strip().lower() for b in _BOXED.findall(text)] + boxed = [b for b in boxed if b in ("yes", "no")] + if boxed: + return boxed[-1].capitalize(), True + first = re.match(r"[*_`>#\s]*(yes|no)[*_`]*\s*(?:[,.!:;]|$)", text, re.I | re.M) + if first: + return first.group(1).capitalize(), True + last = YOrN_Extraction(_last_line(text)) + return (last, True) if last != "Unknown" else ("Unknown", False) diff --git a/packages/pipette-scores/pipette_scores/scoring/vlm_upstream/PORTING_NOTES.md b/packages/pipette-scores/pipette_scores/scoring/vlm_upstream/PORTING_NOTES.md new file mode 100644 index 0000000..3343f6d --- /dev/null +++ b/packages/pipette-scores/pipette_scores/scoring/vlm_upstream/PORTING_NOTES.md @@ -0,0 +1,198 @@ +# VLM scorer porting notes (VLMEvalKit @ 8eca8e5) + +Answer extraction and metrics for the 7 VLM evals are vendored from +**VLMEvalKit** (commit `8eca8e5`), deterministic-only: no LLM judge, no random +fallback — an answer `can_infer` cannot extract counts as **wrong** (the +deterministic equivalent of VLMEvalKit's `exact_matching` policy). + +Module provenance: + +| module | ported from | +|---|---| +| `matching_util.py` | `vlmeval/utils/matching_util.py` (`can_infer`, `can_infer_option`, `can_infer_text`) | +| `yorn.py` | `vlmeval/dataset/utils/yorn.py` (`YOrN_Extraction`, `MME_rating`, `POPE_rating`) + `vlmeval/smp/misc.py` (`process_punctuation`) | +| `multiple_choice.py` | `vlmeval/dataset/utils/multiple_choice.py` (`build_choices`, `MMMU_preproc`, vanilla `eval_vanilla`/`report_acc` semantics) | +| `ocrbench.py` | `vlmeval/dataset/utils/ocrbench.py` (`OCRBench_eval`) | + +## Metric contract (pipette-mgmt) + +Every scorer's `context` includes: + +- `"accuracy"` — 0-1 float, the primary comparable metric read by + pipette-mgmt. MCQ evals → hit rate; **POPE → F1** (0-1); **MME → total + score / 2800**; **OCRBench → Final Score / 1000**. +- `"leaderboard_score"` — the conventional human-facing scale: MCQ acc x100, + POPE F1 x100, MME total (0-2800), OCRBench Final Score (0-1000). + +## Per-benchmark raw_doc + prompt requirements + +The scorer reads the parquet `prompt` column (what the model saw) and the +JSON `raw_doc` column. Dataset regeneration must produce VLMEvalKit-style +prompts (`ImageMCQDataset.build_prompt`) and these `raw_doc` fields: + +### MCQ prompt format (mmbench_en_dev, mmstar, mmmu_val, realworldqa) + +``` +[Hint: {hint}\n] # only if the row has a hint (MMBench) +Question: {question}\n +Options:\nA. {A}\nB. {B}\n... # only option letters present in the row +Please select the correct answer from the options above. \n +``` + +(For MMMU, `` placeholders in the question are interleaved with the +images — `MMMUDataset.split_MMMU`; text content is unchanged.) + +### mmbench_en_dev +- raw_doc: `question`, option columns `A`-`D` (some rows have fewer), `answer` + (letter), `index` (int; circular rotations of one base question share + `index % 1e6` — the dataset MUST contain the rotation rows from + `MMBench_DEV_EN_V11.tsv` for circular scoring to be meaningful), `category`, + optional `hint`, `l2-category`. +- Scoring: `can_infer` per rotation; base-question hit = all rotations + correct; headline = base-question hit rate. + +### mmstar +- raw_doc: `question`, `A`-`D`, `answer` (letter), `category`, `l2_category` + (or `l2-category`), `index`. +- Scoring: `can_infer`; headline = micro mean(hit) (VLMEvalKit `report_acc` + Overall — not the macro per-L2-category mean). + +### mmmu_val +- raw_doc: `question`, option columns `A`-`I` (absent/None for open + questions), `answer` (letter for MCQ, free text for open), `category` + (subject), `index`. +- Scoring: `MMMU_preproc` — open questions (no `A` option) become 2-choice + MCQ with `A = gold answer`, `B = "Other Answers"`, GT = `A`; then + `can_infer`. Headline = micro accuracy. + +### realworldqa +- raw_doc: `question`, `A`-`D`, `answer` (letter), `index`. +- Scoring: `can_infer` (VLMEvalKit ImageMCQ path). Headline = micro accuracy. + +### mme +- prompt: the raw MME question (ends with "Please answer yes or no."). +- raw_doc: `question`, `answer` ("Yes"/"No"), `category` (one of the 14 MME + categories, exact strings incl. `OCR` capitalization), and a **pairing + key** identifying the image: `image_path` (VLMEvalKit) or `question_id` + (lmms-eval export, where it is the image's relative path). The two yes/no + questions of one image MUST share the pairing key, else `acc_plus` + collapses and the score is deflated. +- Scoring: `YOrN_Extraction`; per category `acc + acc_plus`; perception + (10 cats, max 2000) + cognition (4 cats, max 800) = total (max 2800). + +### pope +- prompt: the raw POPE question ("Is there a ... in the image?"). +- raw_doc: `question`, `answer` ("Yes"/"No"), `category` (subset name: + adversarial / popular / random; may be comma-joined — it is exploded). +- Scoring: `YOrN_Extraction`; headline = **F1** over yes/no predictions + ('Unknown' counts as a "No" prediction); precision / recall / accuracy in + context. + +### ocrbench +- prompt: the raw OCRBench question. +- raw_doc: `question`, `answer` (list of acceptable strings, or a stringified + list, or a single string), `category` (VLMEvalKit name; the lmms-eval field + `question_type` holds the same values and is accepted) — one of the 10 + OCRBench categories, **exact case**. +- Scoring: substring match of any gold answer in the prediction; lowercase + for all categories EXCEPT `Handwritten Mathematical Expression Recognition`, + which is case-sensitive with all whitespace removed. Headline = Final Score + (sum correct, /1000). + +## Coverage and denominator policy + +The `/score` API scores a VLM eval only when the request answers every +dataset sample once, none with `stop_reason: failure`; anything else is a +400 (pipette-mgmt refuses such a run before it calls, and pipette's runner +retries a failed sample and never submits a run without an answer to each). +The scorers below hold the full-dataset line regardless, for any direct +caller of the scoring functions. Every headline metric (`context["accuracy"]` / `context["leaderboard_score"]`) +is computed over the **full dataset**, never over the submitted subset: + +- mmstar / realworldqa / mmmu_val: micro accuracy with `len(samples)` as the + denominator — an unsubmitted sample counts as wrong. +- mmbench_en_dev: circular base groups are enumerated from the dataset; a + group with any unsubmitted rotation is a miss. +- pope: the confusion matrix covers all dataset samples; a missing completion + counts as the wrong answer (a false negative or a false positive), not as + `Unknown`: that reads as "No", which would make a missing negative a free + true negative that F1 never sees. +- mme: a missing completion contributes 0 to both `acc` and `acc_plus` + (denominators are per-category dataset counts; total is /2800). +- ocrbench: Final Score /1000 regardless of coverage. + +Each scorer also reports `context["samples_missing"]` — the number of dataset +samples with no submitted completion (0 for a complete run). Diagnostics named +`per_sample_accuracy` / `per_rotation_accuracy` remain submitted-only and are +labeled as such in code. + +Rationale: a subset must never be able to inflate a leaderboard number, at +any layer. This matches the text `math_500` / `gpqa_diamond` convention of +dividing by `len(samples)`. + +## Scoring parity (verified against VLMEvalKit's own code, 2026-10-19) + +The port was checked answer by answer against VLMEvalKit `8eca8e5` itself, +unmodified, on 49,626 stored answers: LFM2.5-VL-3B (no thinking), Qwen3.5-4B +and Gemma-4-E2B (thinking), each on all seven benchmarks. VLMEvalKit built its +datasets from the seven pinned TSVs (MD5 checked) and scored with its exact +matching (`ImageMCQDataset.evaluate`, MMBench circular, `MMMUDataset`, +`ImageYORNDataset` for MME and POPE, `OCRBench.evaluate`), no LLM judge. Both +scorers graded the same text: the completion with its reasoning removed. + +- With the fallbacks below switched off, pipette gives VLMEvalKit's verdict on + every answer and its headline on all 21 runs. The extracted letter or yes/no + agrees on all 44,256 multiple-choice and yes/no rows, and the verdict on all + 3,000 OCRBench rows. No porting difference was found. +- Every disagreement with the fallbacks on is a fallback reading an answer + exact matching could not (1,557 of 49,626): the last answer statement, + a line that opens with an option, the last line. VLMEvalKit hands those to + its LLM judge, which pipette never uses. LFM2.5-VL-3B, which answers with a + letter, has none; a reasoning model that explains its answer has many. +- The check found one fallback misread, now fixed: "I cannot answer yes or no" + read as Yes (4 Gemma-4-E2B MME answers, 2.6 of 2,800 points). + +- A later audit of the final runs found the format Qwen3.5's card prompt asks + for, `"answer": "C"`, written three more ways the fallbacks missed: the key + in backticks (`` `answer`: "C" ``), the value alone (`"B"`, `"a"`, an + unclosed `"B`), and a quoted option line (`"B. Not the same"`). Each is read + only in an anchored shape: the field as a whole line; the value alone as + the whole answer, or as a last line (capital letter or exact option) after + text that names no option letter. Prose that describes the format, refuses + to choose, or quotes two options is not read. Qwen3.5-4B RealWorldQA goes + from 42.0 to 79.3 (its card: 79.5); Qwen3.5-2B MMBench from 58.0 to 77.7. + No other family's score moves. + +## Prompt parity (verified against VLMEvalKit source) + +The served prompts match VLMEvalKit's dataset-default `build_prompt` verbatim, +with NO extra instruction beyond what upstream emits: + +- **MMStar / MMBench_DEV_EN_V11 / RealWorldQA** use the base `ImageMCQDataset.build_prompt`: + `Question:` + `Options:\nA. …` + the soft directive + `"Please select the correct answer from the options above. \n"`. There is NO + stricter `"Answer with the option's letter … directly"` suffix — that belongs + to *other* datasets (MMMU-Pro, GMAI, LEGO, VisualPuzzles), not these. Adding + it would diverge from upstream and shift scores. +- **MMMU_DEV_VAL** = the same base prompt + `split_MMMU` image interleaving. +- **POPE / MME** use the raw question (bare prompt). VLMEvalKit only appends + `"\nPlease answer yes or no."` for AMBER (not run here); MME's own question + text already ends with that sentence. +- **OCRBench** uses the raw question. + +Verbose models that describe the image instead of emitting the option letter +are handled by `can_infer` extraction over the full generation (up to +`max_new_tokens`), exactly as upstream does — not by a stricter prompt. + +## Generation length + +`parameter_max_tokens` is set to **1024** for all 7, matching our VLMEvalKit +fork (`vlmeval/vlm/liquid.py`, `max_new_tokens=1024`) +so on-device pipette numbers are comparable to internal LFM VLMEvalKit runs. +Generation length in VLMEvalKit is a per-*model* serving default, not a +per-dataset constant (Qwen2-VL/LLaVA=2048, InternVL=4096/512/8196), so there +is no single upstream value to copy — the LFM fork's 1024 is the one that +makes our results comparable. It is a ceiling, not a target: answer-first +models emit EOS in a handful of tokens (median 5 on LFM2-VL-450M mmstar); the +budget only binds for reasoning/CoT models that state their answer last, which +the previous tight caps (mme/pope 16, MCQ 64) would have silently clipped. diff --git a/packages/pipette-scores/pipette_scores/scoring/vlm_upstream/__init__.py b/packages/pipette-scores/pipette_scores/scoring/vlm_upstream/__init__.py new file mode 100644 index 0000000..99a006e --- /dev/null +++ b/packages/pipette-scores/pipette_scores/scoring/vlm_upstream/__init__.py @@ -0,0 +1,12 @@ +"""Deterministic answer extraction + metrics vendored from VLMEvalKit @ 8eca8e5. + +Each module cites the exact upstream file it was ported from. All logic is +pure string/number manipulation — no pandas, no file IO, and NO LLM judge +(unparseable answers deterministically count as wrong). + +See PORTING_NOTES.md for the per-benchmark raw_doc / prompt contract. +""" + +from . import matching_util, multiple_choice, ocrbench, yorn + +__all__ = ["matching_util", "multiple_choice", "ocrbench", "yorn"] diff --git a/packages/pipette-scores/pipette_scores/scoring/vlm_upstream/matching_util.py b/packages/pipette-scores/pipette_scores/scoring/vlm_upstream/matching_util.py new file mode 100644 index 0000000..dadc584 --- /dev/null +++ b/packages/pipette-scores/pipette_scores/scoring/vlm_upstream/matching_util.py @@ -0,0 +1,91 @@ +"""Deterministic MCQ answer matching, ported from VLMEvalKit. + +Ported from VLMEvalKit @ 8eca8e5, ``vlmeval/utils/matching_util.py`` +(functions ``can_infer_option``, ``can_infer_text``, ``can_infer``). + +Deviations from upstream (behavior-preserving): +- Logging / ``VERBOSE`` env handling removed. Upstream's "A might be a + quantifier" branch only fires when ``VERBOSE`` is set (default off), so it + is dropped here to match upstream's default behavior. +- ``can_infer_text`` operates on a copy of ``choices`` instead of mutating + the caller's dict (upstream lowercases values in place). + +No LLM judge is used anywhere: when ``can_infer`` returns ``False`` the +caller must count the sample as WRONG (deterministic equivalent of +VLMEvalKit's ``exact_matching`` policy, which assigns option ``Z``). +""" + +import copy as cp +import re +import string + +# Matches verbose chain-of-thought answers such as: +# "The correct answer is **B**. Here's why: ..." +# "The answer is C." +_VERBOSE_ANSWER_RE = re.compile(r"(?i)(?:correct\s+)?answer\s+is\s+\**([ABCD])\**") + + +def can_infer_option(answer, choices): + # Choices is a dictionary + if "Failed to obtain answer via API" in answer: + return False + + reject_to_answer = [ + "Sorry, I can't help with images of people yet.", + "I can't process this file.", + "I'm sorry, but without the image provided", + "Cannot determine the answer", + ] + for err in reject_to_answer: + if err in answer: + return "Z" + + def count_choice(splits, choices, prefix="", suffix=""): + cnt = 0 + for c in choices: + if prefix + c + suffix in splits: + cnt += 1 + return cnt + + answer_mod = cp.copy(answer) + chars = ".()[],:;!*#{}" + for c in chars: + answer_mod = answer_mod.replace(c, " ") + + splits = [x.strip() for x in answer_mod.split()] + count = count_choice(splits, choices) + + if count == 1: + for ch in choices: + if ch in splits and splits.index(ch) > (len(splits) - 5): + return ch + elif count == 0 and count_choice(splits, {"Z", ""}) == 1: + return "Z" + + match = _VERBOSE_ANSWER_RE.search(answer or "") + if match and match.group(1).upper() in choices: + return match.group(1).upper() + + return False + + +def can_infer_text(answer, choices): + answer = answer.lower() + choices = {k: str(v).lower() for k, v in choices.items()} + if len(answer) > 2 * sum(len(v) for v in choices.values()): + return False + for k in choices: + assert k in string.ascii_uppercase + cands = [] + for k in choices: + if choices[k] in answer: + cands.append(k) + if len(cands) == 1: + return cands[0] + return False + + +def can_infer(answer, choices): + answer = str(answer) + copt = can_infer_option(answer, choices) + return copt if copt else can_infer_text(answer, choices) diff --git a/packages/pipette-scores/pipette_scores/scoring/vlm_upstream/multiple_choice.py b/packages/pipette-scores/pipette_scores/scoring/vlm_upstream/multiple_choice.py new file mode 100644 index 0000000..0294c1e --- /dev/null +++ b/packages/pipette-scores/pipette_scores/scoring/vlm_upstream/multiple_choice.py @@ -0,0 +1,63 @@ +"""MCQ scoring helpers, ported from VLMEvalKit. + +Ported from VLMEvalKit @ 8eca8e5, ``vlmeval/dataset/utils/multiple_choice.py``: +- ``build_choices`` — collects the non-empty option columns A..Z of a row. +- ``mmmu_preproc_*`` — port of ``MMMU_preproc`` + the ``answer_map`` + rewrite in ``mcq_vanilla_eval``: open questions are + re-formulated as 2-choice MCQ (A = gold answer, + B = 'Other Answers'), GT letter becomes 'A'. +- ``mcq_hit`` — port of the ``eval_vanilla`` exact-matching path: + ``can_infer`` the prediction; unparseable counts as + WRONG (upstream assigns 'Z' under ``exact_matching``). + +Headline metric ("Overall" in upstream ``report_acc``) is the MICRO mean of +per-sample hits — not a macro average over categories. +""" + +import string +from typing import Any + +from .matching_util import can_infer + +# 'Other Answers' literal from upstream MMMU_preproc. +MMMU_OTHER_ANSWERS = "Other Answers" + + +def _is_missing(v: Any) -> bool: + """Equivalent of pandas ``isna`` for raw_doc values (+ empty strings, + which a parquet/dict round-trip may produce instead of NaN).""" + if v is None: + return True + if isinstance(v, float) and v != v: # NaN + return True + s = str(v).strip() + return s == "" or s.lower() == "nan" + + +def build_choices(item: dict) -> dict[str, str]: + ret = {} + for ch in string.ascii_uppercase: + if ch in item and not _is_missing(item[ch]): + ret[ch] = str(item[ch]) + return ret + + +def mmmu_preproc_choices(item: dict) -> dict[str, str]: + """MMMU_preproc: open questions (no 'A' option) become a 2-choice MCQ.""" + choices = build_choices(item) + if "A" not in choices: + return {"A": str(item.get("answer", "")), "B": MMMU_OTHER_ANSWERS} + return choices + + +def mmmu_preproc_gt(answer: Any) -> str: + """GT letter after MMMU_preproc: non-letter (open) answers map to 'A'.""" + a = str(answer) + return a if a in list(string.ascii_uppercase) else "A" + + +def mcq_hit(prediction: str, choices: dict[str, str], gt: str) -> bool: + """Deterministic vanilla MCQ scoring: True iff ``can_infer`` extracts the + ground-truth letter. No judge, no random fallback — unparseable = wrong.""" + ret = can_infer(prediction, choices) + return bool(ret) and ret == str(gt).strip() diff --git a/packages/pipette-scores/pipette_scores/scoring/vlm_upstream/ocrbench.py b/packages/pipette-scores/pipette_scores/scoring/vlm_upstream/ocrbench.py new file mode 100644 index 0000000..ca686c7 --- /dev/null +++ b/packages/pipette-scores/pipette_scores/scoring/vlm_upstream/ocrbench.py @@ -0,0 +1,89 @@ +"""OCRBench matching + aggregation, ported from VLMEvalKit. + +Ported from VLMEvalKit @ 8eca8e5, ``vlmeval/dataset/utils/ocrbench.py`` +(``OCRBench_eval``), reshaped from a pandas loop into per-sample matching + +a pure aggregation. + +The branch keys off the VLMEvalKit **category** string. For +'Handwritten Mathematical Expression Recognition' the substring match is +CASE-SENSITIVE with all whitespace removed; every other category lowercases +both sides (whitespace kept, newlines flattened). + +The conventional headline is 'Final Score' out of 1000 (OCRBench has exactly +1000 questions, sum of correct); upstream also reports Final Score Norm = +score / 10 (a 0-100 scale). +""" + +import ast +from typing import Any + +OCRBENCH_CATEGORIES = [ + "Regular Text Recognition", + "Irregular Text Recognition", + "Artistic Text Recognition", + "Handwriting Recognition", + "Digit String Recognition", + "Non-Semantic Text Recognition", + "Scene Text-centric VQA", + "Doc-oriented VQA", + "Key Information Extraction", + "Handwritten Mathematical Expression Recognition", +] + +_TEXT_RECOGNITION_CATEGORIES = OCRBENCH_CATEGORIES[:6] + +HME_CATEGORY = "Handwritten Mathematical Expression Recognition" + + +def parse_answers(answer: Any) -> list[str]: + """Upstream stores the answer cell as a stringified list and ``eval``s it.""" + if isinstance(answer, (list, tuple)): + return [str(a) for a in answer] + s = str(answer) + if s.strip().startswith("["): + try: + parsed = ast.literal_eval(s) + if isinstance(parsed, (list, tuple)): + return [str(a) for a in parsed] + except (ValueError, SyntaxError): + pass + return [s] + + +def ocrbench_match(prediction: str, answers: list[str], category: str) -> bool: + """Per-line matching loop from ``OCRBench_eval``.""" + predict = str(prediction) + if category == HME_CATEGORY: + # CASE-SENSITIVE, whitespace stripped from both sides. + for ans in answers: + answer = ans.strip().replace("\n", " ").replace(" ", "") + pred = predict.strip().replace("\n", " ").replace(" ", "") + if answer in pred: + return True + else: + for ans in answers: + answer = ans.lower().strip().replace("\n", " ") + pred = predict.lower().strip().replace("\n", " ") + if answer in pred: + return True + return False + + +def ocrbench_final_score(per_category_correct: dict[str, int]) -> dict[str, float]: + """Aggregation from ``OCRBench_eval``: Final Score = sum correct (max 1000).""" + get = per_category_correct.get + ret: dict[str, float] = {} + ret["Text Recognition"] = float(sum(get(c, 0) for c in _TEXT_RECOGNITION_CATEGORIES)) + ret["Scene Text-centric VQA"] = float(get("Scene Text-centric VQA", 0)) + ret["Doc-oriented VQA"] = float(get("Doc-oriented VQA", 0)) + ret["Key Information Extraction"] = float(get("Key Information Extraction", 0)) + ret[HME_CATEGORY] = float(get(HME_CATEGORY, 0)) + ret["Final Score"] = ( + ret["Text Recognition"] + + ret["Scene Text-centric VQA"] + + ret["Doc-oriented VQA"] + + ret["Key Information Extraction"] + + ret[HME_CATEGORY] + ) + ret["Final Score Norm"] = ret["Final Score"] / 10 + return ret diff --git a/packages/pipette-scores/pipette_scores/scoring/vlm_upstream/yorn.py b/packages/pipette-scores/pipette_scores/scoring/vlm_upstream/yorn.py new file mode 100644 index 0000000..b365038 --- /dev/null +++ b/packages/pipette-scores/pipette_scores/scoring/vlm_upstream/yorn.py @@ -0,0 +1,141 @@ +"""Yes/No answer extraction + MME / POPE ratings, ported from VLMEvalKit. + +Ported from VLMEvalKit @ 8eca8e5: +- ``process_punctuation`` — ``vlmeval/smp/misc.py`` +- ``YOrN_Extraction`` — ``vlmeval/dataset/utils/yorn.py`` +- ``mme_rating`` — pure-python port of ``MME_rating`` + (``vlmeval/dataset/utils/yorn.py``, pandas-free) +- ``pope_rating`` — pure-python port of ``POPE_rating`` + (``vlmeval/dataset/utils/yorn.py``, pandas-free) + +No LLM judge: upstream retries 'Unknown' extractions through a judge model; +here 'Unknown' deterministically stays 'Unknown' and therefore never matches +the 'Yes'/'No' ground truth (counts as wrong / as a "No" prediction in the +POPE confusion matrix, identical to upstream's ``exact_matching`` policy). +""" + +import re +from collections import defaultdict + + +def process_punctuation(inText): + outText = inText + punct = [";", r"/", "[", "]", '"', "{", "}", "(", ")", "=", "+", "\\", "_", "-", ">", "<", "@", "`", ",", "?", "!"] + commaStrip = re.compile(r"(\d)(,)(\d)") + periodStrip = re.compile(r"(? dict[str, float]: + """Pure-python port of VLMEvalKit ``MME_rating``. + + ``records``: one dict per question with keys ``category``, ``image_key`` + (the pairing key — the two yes/no questions of one image share it) and + ``score`` (0.0 or 1.0). + + Per category: ``acc`` = mean score over all questions x100, ``acc_plus`` = + mean over images of (score_q1 * score_q2) x100 (upstream: ``val[0] * val[1]``); + category score = acc + acc_plus (max 200). Perception = sum over its 10 + categories (max 2000); cognition = sum over its 4 (max 800); total max 2800. + """ + stats: dict[str, dict[str, list[float]]] = defaultdict(lambda: defaultdict(list)) + for r in records: + stats[r["category"]][str(r["image_key"])].append(float(r["score"])) + + def acc(key, mode="normal"): + res = stats[key] + values = [] + for val in res.values(): + if mode == "normal": + values.extend(val) + elif mode == "plus": + # Upstream indexes val[0] * val[1] (asserting pairs implicitly). + # Tolerate unpaired images (subset scoring): they contribute 0 + # to acc_plus but still count in the denominator. + values.append(val[0] * val[1] if len(val) >= 2 else 0.0) + return sum(values) / len(values) * 100 if values else 0.0 + + scores = {k: acc(k) + acc(k, "plus") for k in stats} + + ret: dict[str, float] = {} + ret["perception"] = sum(scores.get(c, 0.0) for c in MME_PERCEPTION_CATEGORIES) + # VLMEvalKit calls this super-category 'reasoning'. + ret["cognition"] = sum(scores.get(c, 0.0) for c in MME_COGNITION_CATEGORIES) + ret.update(scores) + return ret + + +def _f1(y_true: list[int], y_pred: list[int]) -> tuple[float, float, float]: + """``cal_f1_score`` from VLMEvalKit ``POPE_rating`` — returns (f1, precision, recall).""" + tp = sum(1 for t, p in zip(y_true, y_pred) if t == 1 and p == 1) + fp = sum(1 for t, p in zip(y_true, y_pred) if t == 0 and p == 1) + fn = sum(1 for t, p in zip(y_true, y_pred) if t == 1 and p == 0) + + precision = tp / (tp + fp) if (tp + fp) != 0 else 0 + recall = tp / (tp + fn) if (tp + fn) != 0 else 0 + f1_score = 2 * (precision * recall) / (precision + recall) if (precision + recall) != 0 else 0 + return f1_score, precision, recall + + +def pope_rating(records: list[dict]) -> dict[str, dict[str, float]]: + """Pure-python port of VLMEvalKit ``POPE_rating``. + + ``records``: one dict per question with keys ``answer`` ('Yes'/'No'), + ``extracted`` ('Yes'/'No'/'Unknown') and ``category`` (may be a + comma-joined multi-category string; upstream explodes it). + + Returns ``{split: {f1, precision, recall, accuracy}}`` (all 0-1) with an + 'Overall' split plus one per exploded category. The POPE headline + ('Overall' column upstream) is the **F1 score**, not accuracy. + """ + exploded: list[tuple[str, str, str]] = [] + for r in records: + cats = str(r.get("category") or "").split(",") + for c in cats: + exploded.append((r["answer"], r["extracted"], c.strip())) + + def rate(rows): + y_true = [1 if a == "Yes" else 0 for a, _, _ in rows] + y_pred = [1 if e == "Yes" else 0 for _, e, _ in rows] + f1, precision, recall = _f1(y_true, y_pred) + acc = sum(1 for a, e, _ in rows if a == e) / len(rows) if rows else 0.0 + return {"f1": f1, "precision": precision, "recall": recall, "accuracy": acc} + + res = {"Overall": rate(exploded)} + for cat in sorted({c for _, _, c in exploded if c}): + res[cat] = rate([row for row in exploded if row[2] == cat]) + return res diff --git a/packages/pipette-scores/pipette_scores/types.py b/packages/pipette-scores/pipette_scores/types.py index 75e2209..9259c04 100644 --- a/packages/pipette-scores/pipette_scores/types.py +++ b/packages/pipette-scores/pipette_scores/types.py @@ -3,7 +3,7 @@ import enum from typing import Any, Union -from pydantic import BaseModel, ConfigDict, field_validator +from pydantic import BaseModel, ConfigDict, Field, SerializerFunctionWrapHandler, field_validator, model_serializer # --------------------------------------------------------------------------- # Eval IDs @@ -15,6 +15,41 @@ class EvalId(enum.StrEnum): IFSTRUCT = "ifstruct" GPQA_DIAMOND = "gpqa_diamond" MATH_500 = "math_500" + # VLM evals — string values must equal the pipette-mgmt `parameter_lmms_task` + # and the scores-server route segment. + MME = "mme" + MMSTAR = "mmstar" + OCRBENCH = "ocrbench" + MMMU_VAL = "mmmu_val" + REALWORLDQA = "realworldqa" + MMBENCH_EN_DEV = "mmbench_en_dev" + POPE = "pope" + # ASR evals: one per dataset of the Open ASR Leaderboard's English public + # sets (in its column order; the asr_esb_ prefix is historical, see + # dataset_catalog/asr_datasets.py) and of the FLEURS locales evaluated. + # Values equal the pipette-mgmt task name and the scores-server route segment. + ASR_ESB_AMI_CLEANED = "asr_esb_ami_cleaned" + ASR_ESB_EARNINGS22_CLEANED_AA_CHUNKED = "asr_esb_earnings22_cleaned_aa_chunked" + ASR_ESB_GIGASPEECH_CLEANED = "asr_esb_gigaspeech_cleaned" + ASR_ESB_LIBRISPEECH_TEST_CLEAN = "asr_esb_librispeech_test_clean" + ASR_ESB_LIBRISPEECH_TEST_OTHER = "asr_esb_librispeech_test_other" + ASR_ESB_SPGISPEECH = "asr_esb_spgispeech" + ASR_ESB_URGENT2024 = "asr_esb_urgent2024" + ASR_ESB_MONSOON_EN_IN = "asr_esb_monsoon_en_in" + ASR_ESB_VOXPOPULI_CLEANED_AA = "asr_esb_voxpopuli_cleaned_aa" + ASR_FLEURS_EN_US = "asr_fleurs_en_us" + ASR_FLEURS_ES_419 = "asr_fleurs_es_419" + ASR_FLEURS_PT_BR = "asr_fleurs_pt_br" + ASR_FLEURS_DE_DE = "asr_fleurs_de_de" + ASR_FLEURS_FR_FR = "asr_fleurs_fr_fr" + ASR_FLEURS_IT_IT = "asr_fleurs_it_it" + ASR_FLEURS_NL_NL = "asr_fleurs_nl_nl" + + +# The ASR evals in catalog order (for reports and builds), and +# as a set (for membership checks). +ASR_EVAL_ORDER: tuple[EvalId, ...] = tuple(e for e in EvalId if e.value.startswith("asr_")) +ASR_EVAL_IDS: frozenset[EvalId] = frozenset(ASR_EVAL_ORDER) # --------------------------------------------------------------------------- @@ -25,13 +60,33 @@ class EvalId(enum.StrEnum): class ChatMessage(BaseModel): model_config = ConfigDict(frozen=True) role: str - content: str + # String for text-only evals; list of OpenAI-style content blocks (text / + # image_url) for multimodal evals. + content: str | list[dict] + + +def _without_unset(handler: SerializerFunctionWrapHandler, value: BaseModel, *optional: str) -> dict[str, Any]: + """Serialize, leaving out the named optional fields when they are None, so a + payload that does not use them keeps its earlier wire shape exactly.""" + data = handler(value) + for name in optional: + if data.get(name) is None: + data.pop(name, None) + return data class EvalSample(BaseModel): model_config = ConfigDict(frozen=True) id: str messages: list[ChatMessage] + # Per-sample chat-template arguments the client passes with the request + # (ASR: Voxtral's transcription mode and language). Omitted when None. + chat_template_kwargs: dict[str, Any] | None = None + + @model_serializer(mode="wrap") + # No return annotation: pydantic would publish it as the OpenAPI schema. + def _serialize(self, handler: SerializerFunctionWrapHandler): + return _without_unset(handler, self, "chat_template_kwargs") # --------------------------------------------------------------------------- @@ -81,9 +136,49 @@ def _answer_is_option_letter(_cls, v: str) -> str: return normalized +class Math500Grader(enum.StrEnum): + """Which grader a MATH-500 dataset version is scored with (metadata ``grader``).""" + + # The AA-aligned score_math_generic that liquid_evals' math500_std track uses. + MATH_GENERIC = "math_generic" + # liquid_evals' plain math500 task: extract_math_answer, then verify_math_answer. + LIQUID_EVALS_MATH500 = "liquid_evals_math500" + + class Math500Sample(BaseDatasetSample): prompt: str - answer: str # free-form gold answer (LaTeX / number), graded by score_math_generic + answer: str # free-form gold answer (LaTeX / number) + # Set from the dataset's metadata.json; versions that predate the key use math_generic. + grader: Math500Grader = Math500Grader.MATH_GENERIC + + +class VlmSample(BaseDatasetSample): + """Shared sample shape for all VLM evals. + + ``raw_doc`` carries the benchmark-specific fields the scorer needs + (option columns, answer, category, index, ...) — see + ``scoring/vlm_upstream/PORTING_NOTES.md`` for the per-benchmark contract. + """ + + prompt: str + answer: Any + raw_doc: dict[str, Any] + + +class AsrSample(BaseDatasetSample): + """One ASR clip's ground truth (no audio: the scorer never reads it). + + ``scoring`` is the dataset's recipe from metadata.json (normalizer, aligner, + Open ASR exclusions); ``session``/``chunk_index`` place a chunk of chunked + long-form audio (Earnings22) in the session it is scored with. + """ + + reference: str + language: str # ISO 639-3 + session: str | None = None + chunk_index: int | None = None + scoring: dict[str, Any] + raw_doc: dict[str, Any] DatasetSample = Union[ @@ -91,6 +186,8 @@ class Math500Sample(BaseDatasetSample): IFStructSample, GPQADiamondSample, Math500Sample, + VlmSample, + AsrSample, ] @@ -102,6 +199,9 @@ class Math500Sample(BaseDatasetSample): class SampleCompletion(BaseModel): id: str completion: str + # How the client's generation ended, as plan-types names it ("eos", + # "truncated", "doom_loop", "failure", "unknown"); None when not reported. + stop_reason: str | None = None # --------------------------------------------------------------------------- @@ -114,6 +214,8 @@ class SampleScore(BaseModel): id: str is_correct: bool + # ASR: the sample's alignment (see ScoredSample.measures). None elsewhere. + measures: dict[str, Any] | None = None class ScoredSample(BaseModel): @@ -121,6 +223,18 @@ class ScoredSample(BaseModel): messages: list[ChatMessage] completion: str is_correct: bool + # ASR only: the sample's part in its scoring unit (docs/asr-evals.md §3: + # the unit's counts and normalized texts on one sample per unit, so summing + # them over samples gives the dataset's totals). Omitted when None. + measures: dict[str, Any] | None = None + # The sample's chat-template arguments as served (ASR voxtral: the + # transcription mode and language hint). Omitted when None. + chat_template_kwargs: dict[str, Any] | None = None + + @model_serializer(mode="wrap") + # No return annotation: pydantic would publish it as the OpenAPI schema. + def _serialize(self, handler: SerializerFunctionWrapHandler): + return _without_unset(handler, self, "measures", "chat_template_kwargs") # --------------------------------------------------------------------------- @@ -132,6 +246,17 @@ class ScoreRequest(BaseModel): eval_id: str dataset_name: str completions: list[SampleCompletion] + # The job's enable_thinking model flag; None when the job did not set it. + enable_thinking: bool | None = None + # The run's prompt profile. VLM evals (dataset_catalog/prompt_profiles.py), + # optional: grading is the same, but scored_samples carry the prompts it + # produced. ASR evals (dataset_catalog/asr_prompt_profiles.py), required: + # each completion is cleaned as that profile says. + prompt_profile: str | None = None + # The digest of the samples the run was given (`sha256:`, as + # /samples/digest reports it), if known: scoring refuses with 409 when + # this server no longer serves those samples. + samples_digest: str | None = Field(default=None, pattern=r"^sha256:[0-9a-f]{64}$") class ScoreResponse(BaseModel): @@ -145,3 +270,9 @@ class ScoreResponse(BaseModel): class SamplesResponse(BaseModel): samples: list[EvalSample] + + +class SamplesDigestResponse(BaseModel): + # hashing.samples_digest of what /samples serves for the same request. + digest: str + sample_count: int diff --git a/packages/pipette-scores/pyproject.toml b/packages/pipette-scores/pyproject.toml index 6dcaa06..59c3af0 100644 --- a/packages/pipette-scores/pyproject.toml +++ b/packages/pipette-scores/pyproject.toml @@ -14,7 +14,9 @@ dependencies = [ "pandas>=2.0.0", "pyarrow>=14.0.0", "langdetect>=1.0.9", - "nltk>=3.8.0", + # IFBench verdicts depend on the tokenizer (3.9.4 splits dashes into their own + # tokens); this matches the nltk that the internal liquid_evals environments lock. + "nltk==3.9.4", "immutabledict>=4.0.0", "emoji>=2.0.0", "syllapy>=0.7.2", @@ -23,15 +25,53 @@ dependencies = [ "setuptools>=69.0.0,<81", # syllapy imports pkg_resources (removed in setuptools 81) "tabulate>=0.9.0", "ifstruct", - "sympy>=1.12", - "pylatexenc>=2.10", + # MATH-500 verdicts depend on sympy's simplify and pylatexenc's LaTeX-to-text; + # these match liquid_evals' uv.lock, which its math500 task grades with. + "sympy==1.14.0", + "pylatexenc==2.10", + # ASR WER (scoring/asr/NOTICE.md): kaldialign is the Open ASR Leaderboard's + # pin (requirements/requirements_jobs.txt); num2words, which its multilingual + # normalizer spells numbers out with (FLEURS), it installs unpinned; it is + # pinned here at its latest release, which that install resolves to, so that + # the FLEURS numbers are reproducible. + "kaldialign==0.12.0", + "num2words==0.5.14", + # The vendored Open ASR normalizers' \p{...} classes are regex's Unicode + # tables. + "regex==2026.4.4", + # ASR samples decode each clip and re-encode it as 16 kHz mono WAV; pinned + # with its bundled libsndfile 1.2.2 (scoring/asr/NOTICE.md). + "soundfile==0.14.0", + # A clip not at 16 kHz mono (MonsoonASR is 44.1 kHz) is resampled by the + # dataset builder as datasets.Audio(sampling_rate=16000) does, through + # librosa's default soxr_hq (dataset_catalog/_asr_audio.py). + "soxr==1.1.0", + # `pipette-scores packages` and the server's PIPETTE_SCORES_PACKAGES index reads go through S3 (packages.py). + "boto3", ] [tool.uv.sources] ifstruct = { path = "../../vendor/ifstruct" } [project.optional-dependencies] -dev = ["pytest>=8.0", "pytest-asyncio>=0.24.0", "httpx>=0.24.0", "ruff>=0.8.0"] +# Offline VLM dataset builders + verifier only (scripts/build__dataset.py +# → _vlm_build download_tsv/maybe_resize, verify_vlm_datasets.py). Kept OUT of +# runtime deps so the production scores image (`uv sync --no-dev`) stays lean — +# the serving/scoring path (_vlm_common, the 7 scorers) never imports these. +build = [ + "requests>=2.31.0", + "pillow>=10.0", + # ASR dataset builders (scripts/build_asr_datasets.py): the languages' ISO 639 + # names and codes, fixed into each built dataset. + "pycountry==26.2.16", +] +dev = [ + "pytest>=8.0", + "pytest-asyncio>=0.24.0", + "httpx>=0.24.0", + "ruff>=0.8.0", + "pipette-scores[build]", +] [project.scripts] pipette-scores = "pipette_scores.cli:main" @@ -43,16 +83,17 @@ build-backend = "hatchling.build" [tool.hatch.build.targets.wheel] packages = ["pipette_scores"] -# Bundle the pristine upstream IFBench source (vendor/ifbench submodule) into -# the wheel so the installed package is self-contained and does not depend on -# the repo layout at runtime. The adapter (scoring/ifbench/upstream.py) finds -# these files inside site-packages; in editable/source installs it falls back -# to the submodule path at the repo root. +# Bundle the vendored IFBench checkers (vendor/ifbench-checkers, at the revision +# liquid_evals scores with) into the wheel so the installed package is +# self-contained and does not depend on the repo layout at runtime. The adapter +# (scoring/ifbench/upstream.py) finds these files inside site-packages; in +# editable/source installs it falls back to the vendored path at the repo root. [tool.hatch.build.targets.wheel.force-include] -"../../vendor/ifbench/evaluation_lib.py" = "pipette_scores/scoring/ifbench/_upstream/evaluation_lib.py" -"../../vendor/ifbench/instructions.py" = "pipette_scores/scoring/ifbench/_upstream/instructions.py" -"../../vendor/ifbench/instructions_registry.py" = "pipette_scores/scoring/ifbench/_upstream/instructions_registry.py" -"../../vendor/ifbench/instructions_util.py" = "pipette_scores/scoring/ifbench/_upstream/instructions_util.py" -"../../vendor/ifbench/LICENSE" = "pipette_scores/scoring/ifbench/_upstream/LICENSE" +"../../vendor/ifbench-checkers/evaluation_lib.py" = "pipette_scores/scoring/ifbench/_upstream/evaluation_lib.py" +"../../vendor/ifbench-checkers/instructions.py" = "pipette_scores/scoring/ifbench/_upstream/instructions.py" +"../../vendor/ifbench-checkers/instructions_registry.py" = "pipette_scores/scoring/ifbench/_upstream/instructions_registry.py" +"../../vendor/ifbench-checkers/instructions_util.py" = "pipette_scores/scoring/ifbench/_upstream/instructions_util.py" +"../../vendor/ifbench-checkers/LICENSE" = "pipette_scores/scoring/ifbench/_upstream/LICENSE" +"../../vendor/ifbench-checkers/README.md" = "pipette_scores/scoring/ifbench/_upstream/README.md" "../../NOTICE" = "pipette_scores/NOTICE" "../../THIRD_PARTY_LICENSES.md" = "pipette_scores/THIRD_PARTY_LICENSES.md" diff --git a/packages/pipette-scores/scripts/build_asr_datasets.py b/packages/pipette-scores/scripts/build_asr_datasets.py new file mode 100644 index 0000000..1c971be --- /dev/null +++ b/packages/pipette-scores/scripts/build_asr_datasets.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python +"""Materialize the ASR eval datasets (Open ASR Leaderboard English + FLEURS) from their pinned sources. + +Run from the repo root (build deps: ``uv sync --extra build``), writing OUTSIDE +the checkout (the builder refuses a path inside it): + + uv run --project packages/pipette-scores \\ + python packages/pipette-scores/scripts/build_asr_datasets.py \\ + --out /data/pipette-scores/datasets [EVAL_ID ...] + +With no EVAL_ID, builds every ASR eval (about 20 GB of audio downloaded; SPGISpeech +alone is 11.5 GB). Writes ``//2026.10.1/{test.parquet,metadata.json}``; +serve it with ``PIPETTE_SCORES_DATA_DIR=/..``. A full build must match the +catalog's clip count and ``expected_source_sha256`` where filled; a new +fingerprint is printed for the catalog. ``--limit N`` builds a smoke test under +``2026.10.1-limit`` (never the canonical version), which is never served. + +Never commit the output or bake it into an image: SPGISpeech and URGENT24 are +non-commercial and GigaSpeech gated (see docs/dataset-licenses.md). Sources and +pins: pipette_scores/dataset_catalog/asr_datasets.py; each set is read as the +Open ASR Leaderboard loads it (docs/asr-evals.md). +""" + +import argparse +import logging +import pathlib +import sys + +from pipette_scores.dataset_catalog import _asr_build, asr_datasets +from pipette_scores.types import EvalId + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("eval_ids", nargs="*", help="EvalIds to build (default: all ASR evals)") + parser.add_argument("--out", type=pathlib.Path, required=True, help="datasets root (…/datasets), outside the repo") + parser.add_argument("--version", default=None, help=f"default {asr_datasets.DATASET_VERSION}") + parser.add_argument( + "--limit", type=int, default=None, help="first N clips only: a smoke test, never served (no count check)" + ) + args = parser.parse_args() + # The log goes to stdout too, so each dataset's summary line lands after its own log. + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s", stream=sys.stdout) + for noisy in ("httpx", "huggingface_hub", "datasets", "filelock", "fsspec"): + logging.getLogger(noisy).setLevel(logging.WARNING) + version = args.version or ( + asr_datasets.DATASET_VERSION if args.limit is None else f"{asr_datasets.DATASET_VERSION}-limit{args.limit}" + ) + _asr_build.refuse_inside_checkout(args.out) + ids = [EvalId(e) for e in args.eval_ids] or list(asr_datasets.ASR_DATASETS) + for eval_id in ids: + dataset = asr_datasets.ASR_DATASETS[eval_id] + meta = _asr_build.build(dataset, args.out, version, limit=args.limit) + audio = meta["audio"] + if args.limit is not None: + pin = f"--limit {args.limit} smoke build, never served" + elif version != asr_datasets.DATASET_VERSION: + pin = f"version {version}, not checked against the pin" + elif dataset.expected_source_sha256: + pin = "matches the pin" + else: + pin = "NOT PINNED: add it to EXPECTED_SOURCE_SHA256" + # One line per dataset, flushed, so it cannot interleave with the logging. + print( + f"{eval_id.value}: {meta['num_samples']} clips, source_sha256 {meta['source_sha256']} ({pin}), " + f"{audio['total_seconds'] / 3600:.2f} h, max {audio['max_seconds']} s, " + f"{audio['clips_over_30_s']} over 30 s, {audio['audio_bytes'] / 1e6:.0f} MB, " + f"{audio['formats']} {audio['subtypes']} {audio['sample_rates']} Hz x{audio['channels']}", + flush=True, + ) + + +if __name__ == "__main__": + main() diff --git a/packages/pipette-scores/scripts/build_math_500_dataset.py b/packages/pipette-scores/scripts/build_math_500_dataset.py index f3120ce..f36822d 100644 --- a/packages/pipette-scores/scripts/build_math_500_dataset.py +++ b/packages/pipette-scores/scripts/build_math_500_dataset.py @@ -1,35 +1,52 @@ #!/usr/bin/env python """One-shot materializer for the MATH-500 eval dataset. -Run from the repo root: +Run from the repo root, once per dataset version: uv run --project packages/pipette-scores \ - python packages/pipette-scores/scripts/build_math_500_dataset.py + python packages/pipette-scores/scripts/build_math_500_dataset.py --version 2026.10.1 MATH-500 is third-party data, so (like gpqa_diamond) the emitted `datasets/math_500//` is gitignored and materialized at deploy from the pinned upstream revision — not committed. -Reproduces the Artificial Analysis MATH methodology via liquid_evals' -**standardized** track (`tasks/math500_std.py`): the `MathGenericFormatter` -chain-of-thought prompt asking for the final answer in `\\boxed{}`, graded by -`pipette_scores.scoring.math_500.math_generic` (the AA-aligned grader that the -`_std` task pairs with). The gold answer is the upstream `answer` column. - -Prompt + grader are taken from the same track on purpose: the plain -`tasks/math500.py` uses a different prompt and a bespoke grader, so mixing the -two would not reproduce the published AA number. +Both versions hold the same 500 problems with the upstream `answer` column as +gold. They differ in the liquid_evals track they reproduce: + +- `2026.06.1` — the Artificial Analysis methodology via liquid_evals' + **standardized** track (`tasks/math500_std.py`): the `MathGenericFormatter` + chain-of-thought prompt asking for the final answer in `\\boxed{}`, 5 repeats, + graded by `pipette_scores.scoring.math_500.math_generic` (the AA-aligned + grader that the `_std` task pairs with). +- `2026.10.1` — liquid_evals' **plain** `tasks/math500.py`, the MATH-500 on + Liquid's model cards: the problem, then "Please reason step by step, and put + your final answer within \\boxed{}.", 5 repeats, graded by liquid_evals' + `extract_math_answer` + `verify_math_answer` + (`pipette_scores.scoring.math_500.liquid_evals_parser`). One task run is one + pass over the 500 problems; the model-card number is the mean of 5 such + runs, and pass@1 over the 5 repeats is that mean. The task takes its gold + from `extract_math_answer(solution)`; the build checks that this equals + `answer` on every row. + +Each version's prompt and grader come from the same track on purpose: mixing +them would reproduce neither number. The grader is recorded as +`metadata.grader` (absent in 2026.06.1, which predates the key and so keeps +math_generic). """ import argparse import json import pathlib +from collections.abc import Callable +from dataclasses import dataclass import pyarrow as pa import pyarrow.parquet as pq from datasets import load_dataset from pipette_scores.dataset_catalog.math_500 import content_sha256 +from pipette_scores.scoring.math_500.liquid_evals_parser import extract_math_answer +from pipette_scores.types import Math500Grader # --- Upstream source (pinned for reproducibility) --- HF_DATASET = "HuggingFaceH4/MATH-500" @@ -37,34 +54,95 @@ # Pin the revision — without it a rebuild silently tracks the dataset's HEAD. HF_REVISION = "6e4ed1a2a79af7d8630a6b768ec859cb5af4d3be" # HuggingFaceH4/MATH-500 @ 2026-06 -# --- Expected output (the build asserts against these) --- EXPECTED_SAMPLES = 500 -# Fingerprint of the formatted rows at HF_REVISION. A mismatch means upstream -# drift, a formatter change, or a datasets-lib parsing change — fail before -# writing a bad blob rather than ship one. (Filled after the first build.) -EXPECTED_CONTENT_SHA256 = "03d8a41616e2b3f5e0c317e8de72d3a88b930186a0ef7d00686e90051004f588" -# --- Prompt formatting (AA methodology) --- +# --- Prompt formatting --- # The `MathGenericFormatter` template from liquid_evals' standardized math track # (`tasks/math500_std.py`): instruction, then the problem, then a trailing # reminder. Kept byte-identical to liquid-verifiers # `tasks/backends/math_generic.py` so the prompt matches the score_math_generic # grader it is paired with. -def format_row(doc: dict) -> dict: - """Build one {prompt, answer} row from a raw MATH-500 doc (AA methodology).""" - prompt = ( +def std_prompt(problem: str) -> str: + return ( "Solve the following math problem step by step. " "Put your answer inside \\boxed{}.\n\n" - f"{doc['problem']}\n\n" + f"{problem}\n\n" "Remember to put your answer inside \\boxed{}." ) - return {"prompt": prompt, "answer": doc["answer"]} + + +# liquid_evals `tasks/math500.py` prepare_prompt (d5534cf, line 49): the single +# user message; the task sends no system prompt. +def liquid_evals_prompt(problem: str) -> str: + return f"{problem}\n\nPlease reason step by step, and put your final answer within \\boxed{{}}." + + +@dataclass(frozen=True) +class DatasetVersion: + prompt: Callable[[str], str] + repeats: int + # Fingerprint of the formatted rows at HF_REVISION. A mismatch means + # upstream drift, a formatter change, or a datasets-lib parsing change — + # fail before writing a bad blob rather than ship one. + content_sha256: str + # metadata.grader; None leaves the key out. + grader: Math500Grader | None + # The gold must equal liquid_evals' extract_math_answer(solution). + check_gold_extraction: bool + notes: str + + +VERSIONS = { + "2026.06.1": DatasetVersion( + prompt=std_prompt, + repeats=5, + content_sha256="03d8a41616e2b3f5e0c317e8de72d3a88b930186a0ef7d00686e90051004f588", + grader=None, + check_gold_extraction=False, + notes=( + "MATH-500 (Hendrycks MATH 500-problem subset), Artificial Analysis methodology: " + "5 repeats per question, pass@1. Prompt + grader are the standardized (_std) track " + "from liquid_evals tasks/math500_std.py: the MathGenericFormatter chain-of-thought " + "prompt (instruction / problem / trailing reminder, final answer in \\boxed{}) graded " + "by pipette_scores.scoring.math_500.math_generic (boxed extraction -> PRM800K grader -> " + "robust_boxed). Gold answer is the upstream `answer` column. The N logical questions " + "are served as N*5 #k attempt ids at load time." + ), + ), + "2026.10.1": DatasetVersion( + prompt=liquid_evals_prompt, + repeats=5, + content_sha256="5a0da41f1322ab852cfe7ea2e51fca9277f9142cc64a011f07dd597bb6d5eaa2", + grader=Math500Grader.LIQUID_EVALS_MATH500, + check_gold_extraction=True, + notes=( + "MATH-500 (Hendrycks MATH 500-problem subset), liquid_evals' plain math500 track " + "(tasks/math500.py @ d5534cf), the MATH-500 Liquid reports on its model cards: 5 repeats " + "per question, pass@1. One math500 task run is one pass over the 500 problems, and the " + "model-card number is the mean of 5 runs; pass@1 over the 5 repeats is that mean. The N " + "logical questions are served as N*5 #k attempt ids at load time. Prompt: one user " + "message, no system prompt -- the problem, then 'Please reason step by step, and put " + "your final answer within \\boxed{}.' " + "Graded by liquid_evals' extract_math_answer + verify_math_answer (mathd/sympy " + "equivalence on the extracted answer, then on each of the response's last 150 lines), " + "vendored verbatim as pipette_scores.scoring.math_500.liquid_evals_parser and selected " + "by `grader`. Gold answer is the upstream `answer` column, identical on all 500 rows to " + "the task's extract_math_answer(solution). liquid_evals shuffles the problems " + "(random_state=777); order does not affect the score." + ), + ), +} + + +def format_row(doc: dict, version: DatasetVersion) -> dict: + """Build one {prompt, answer} row from a raw MATH-500 doc.""" + return {"prompt": version.prompt(doc["problem"]), "answer": doc["answer"]} def main() -> None: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--version", default="2026.06.1", help="Dataset version dir name") + parser.add_argument("--version", default="2026.06.1", choices=sorted(VERSIONS), help="Dataset version") parser.add_argument( "--out-dir", type=pathlib.Path, @@ -72,18 +150,26 @@ def main() -> None: help="datasets/math_500 root (version subdir is created under it)", ) args = parser.parse_args() + version = VERSIONS[args.version] dataset = load_dataset(HF_DATASET, split=HF_SPLIT, revision=HF_REVISION) - rows = [format_row(doc) for doc in dataset] + if version.check_gold_extraction: + differ = [doc["unique_id"] for doc in dataset if extract_math_answer(doc["solution"]) != doc["answer"]] + if differ: + raise SystemExit( + f"`answer` differs from extract_math_answer(solution) on {len(differ)} rows " + f"(e.g. {differ[:3]}); liquid_evals grades against the latter." + ) + rows = [format_row(doc, version) for doc in dataset] if len(rows) != EXPECTED_SAMPLES: raise SystemExit(f"expected {EXPECTED_SAMPLES} MATH-500 rows, got {len(rows)} — wrong split/revision?") digest = content_sha256([(r["prompt"], r["answer"]) for r in rows]) - if digest != EXPECTED_CONTENT_SHA256: + if digest != version.content_sha256: raise SystemExit( - f"content hash {digest} != expected {EXPECTED_CONTENT_SHA256} — upstream " + f"content hash {digest} != expected {version.content_sha256} — upstream " f"data at {HF_REVISION} or the formatting changed; review before updating " - "EXPECTED_CONTENT_SHA256." + f"the {args.version} content_sha256." ) out = args.out_dir / args.version @@ -95,20 +181,13 @@ def main() -> None: "dataset": args.version, "source_dataset": HF_DATASET, "num_samples": len(rows), - "repeats": 5, + "repeats": version.repeats, "content_sha256": digest, + **({"grader": version.grader.value} if version.grader else {}), # Informational — actual eval temperature is a client-side policy. "generation_params": {"temperature": 0.6}, "source": {"hf_dataset": HF_DATASET, "split": HF_SPLIT, "hf_revision": HF_REVISION}, - "notes": ( - "MATH-500 (Hendrycks MATH 500-problem subset), Artificial Analysis methodology: " - "5 repeats per question, pass@1. Prompt + grader are the standardized (_std) track " - "from liquid_evals tasks/math500_std.py: the MathGenericFormatter chain-of-thought " - "prompt (instruction / problem / trailing reminder, final answer in \\boxed{}) graded " - "by pipette_scores.scoring.math_500.math_generic (boxed extraction -> PRM800K grader -> " - "robust_boxed). Gold answer is the upstream `answer` column. The N logical questions " - "are served as N*5 #k attempt ids at load time." - ), + "notes": version.notes, } (out / "metadata.json").write_text(json.dumps(metadata, indent=2) + "\n") diff --git a/packages/pipette-scores/scripts/build_mmbench_en_dev_dataset.py b/packages/pipette-scores/scripts/build_mmbench_en_dev_dataset.py new file mode 100644 index 0000000..1f2f4af --- /dev/null +++ b/packages/pipette-scores/scripts/build_mmbench_en_dev_dataset.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python +"""One-shot materializer for the MMBench EN dev (v1.1) eval dataset. + +Run from the repo root: + + uv run --project packages/pipette-scores \ + python packages/pipette-scores/scripts/build_mmbench_en_dev_dataset.py + +Downloads VLMEvalKit's pinned ``MMBench_DEV_EN_V11.tsv`` (~52 MB; URL + MD5 from +VLMEvalKit @ 8eca8e5) and writes ``datasets/mmbench_en_dev/2026.10.1/`` +(test.parquet + metadata.json). Gitignored, materialized at deploy — never +committed. MMBench images may not be redistributed in transformed form (see +docs/dataset-licenses.md). + +TSV columns: index, question, hint, A, B, C, D, answer, category, image, +l2-category, split. Prompt = VLMEvalKit MCQ format +(``Hint:`` line included when the row has one). + +The TSV contains ALL circular-rotation rows (rotations of one base question +share ``index % 1e6``); every row is kept so circular scoring works. Rotation +rows store an index pointer in the ``image`` cell instead of base64 — the +builder resolves those to the actual bytes (VLMEvalKit image-dedup scheme). +""" + +from pipette_scores.dataset_catalog._vlm_build import BenchmarkSpec, run_cli + +# --- Upstream source (pinned for reproducibility; VLMEvalKit @ 8eca8e5) --- +URL = "https://opencompass.openxlab.space/utils/benchmarks/MMBench/MMBench_DEV_EN_V11.tsv" +MD5 = "30c05be8f2f347a50be25aa067248184" + +# --- Expected output (the build asserts against these) --- +# Total row count incl. circular rotations. (Filled after the first build.) +EXPECTED_SAMPLES = 4876 # MMBench_DEV_EN_V11.tsv: 1292 base questions incl. circular rotations +# Fingerprint of the built rows. (Filled after the first build.) +EXPECTED_CONTENT_SHA256 = "06d4cc91d3fa972f706731623e3cdf46c2cf10dfaa6f9366e81458b7358b2b3a" + +SPEC = BenchmarkSpec( + eval_id="mmbench_en_dev", + vlmevalkit_name="MMBench_DEV_EN_V11", + url=URL, + md5=MD5, + prompt_style="mcq", + expected_samples=EXPECTED_SAMPLES, + expected_content_sha256=EXPECTED_CONTENT_SHA256, + license="Apache-2.0 (repository)", + license_notes=( + "The MMBench repository is Apache-2.0, but the images come from public " + "datasets and the internet with uncleared rights. You must self-assess " + "redistribution and commercial-use risk." + ), +) + +if __name__ == "__main__": + run_cli(SPEC, description=__doc__) diff --git a/packages/pipette-scores/scripts/build_mme_dataset.py b/packages/pipette-scores/scripts/build_mme_dataset.py new file mode 100644 index 0000000..54a5744 --- /dev/null +++ b/packages/pipette-scores/scripts/build_mme_dataset.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python +"""One-shot materializer for the MME eval dataset (full set). + +Run from the repo root: + + uv run --project packages/pipette-scores \ + python packages/pipette-scores/scripts/build_mme_dataset.py + +Downloads VLMEvalKit's pinned ``MME.tsv`` (~48 MB; URL + MD5 from VLMEvalKit +@ 8eca8e5) and writes ``datasets/mme/2026.10.1/`` (test.parquet + metadata.json). +Like the other third-party datasets, the output is gitignored and materialized +at deploy — never committed. MME may not be redistributed in transformed form +(see docs/dataset-licenses.md). + +TSV columns: index, category, image_path, question, answer, image. The prompt +is the raw question (ends with "Please answer yes or no."); raw_doc keeps +``image_path`` — the acc_plus pairing key shared by the two yes/no questions +of each image (see scoring/vlm_upstream/PORTING_NOTES.md). +""" + +from pipette_scores.dataset_catalog._vlm_build import BenchmarkSpec, run_cli + +# --- Upstream source (pinned for reproducibility; VLMEvalKit @ 8eca8e5) --- +URL = "https://opencompass.openxlab.space/utils/VLMEval/MME.tsv" +MD5 = "b36b43c3f09801f5d368627fb92187c3" + +# --- Expected output (the build asserts against these) --- +EXPECTED_SAMPLES = 2374 +# Fingerprint of the built rows (prompt/target/raw_doc/image digests). A +# mismatch means upstream drift or a formatting change — fail before writing a +# bad blob. (Filled after the first build.) +EXPECTED_CONTENT_SHA256 = "a9cbb72ac15c431eea45521f1fbb06da5d7374c63ac8c92eb517b5460d00613a" + +SPEC = BenchmarkSpec( + eval_id="mme", + vlmevalkit_name="MME", + url=URL, + md5=MD5, + prompt_style="bare", + expected_samples=EXPECTED_SAMPLES, + expected_content_sha256=EXPECTED_CONTENT_SHA256, + license="none (no declared license)", + license_notes=( + "MME has no declared license; the authors' stated terms are " + "academic-research-only / non-commercial use. You must self-assess " + "whether your use (especially any commercial use) complies with the " + "authors' terms before running this benchmark." + ), +) + +if __name__ == "__main__": + run_cli(SPEC, description=__doc__) diff --git a/packages/pipette-scores/scripts/build_mmmu_val_dataset.py b/packages/pipette-scores/scripts/build_mmmu_val_dataset.py new file mode 100644 index 0000000..348bdb7 --- /dev/null +++ b/packages/pipette-scores/scripts/build_mmmu_val_dataset.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python +"""One-shot materializer for the MMMU validation-split eval dataset. + +Run from the repo root: + + uv run --project packages/pipette-scores \ + python packages/pipette-scores/scripts/build_mmmu_val_dataset.py + +Downloads VLMEvalKit's pinned ``MMMU_DEV_VAL.tsv`` (~81 MB; URL + MD5 from +VLMEvalKit @ 8eca8e5) and writes ``datasets/mmmu_val/2026.10.1/`` (test.parquet + +metadata.json). Gitignored, materialized at deploy — never committed. MMMU +images may not be redistributed in transformed form (see +docs/dataset-licenses.md). + +The TSV holds 1050 rows (150 dev + 900 validation); only the 900 VAL rows are +kept. Multi-image rows store a stringified list in the ``image`` cell; the +question text keeps its ```` placeholders unchanged — the prompt +loader (dataset_catalog/_vlm_common.py) interleaves images at those positions, +matching VLMEvalKit ``MMMUDataset.split_MMMU``. Option columns run A-I and are +absent (NaN) for open questions, which the scorer rewrites via MMMU_preproc. +""" + +import pandas as pd + +from pipette_scores.dataset_catalog._vlm_build import BenchmarkSpec, run_cli + +# --- Upstream source (pinned for reproducibility; VLMEvalKit @ 8eca8e5) --- +URL = "https://opencompass.openxlab.space/utils/VLMEval/MMMU_DEV_VAL.tsv" +MD5 = "585e8ad75e73f75dcad265dfd0417d64" + +# --- Expected output (the build asserts against these) --- +EXPECTED_SAMPLES = 900 # VAL rows only; the 150 DEV rows are excluded +# Fingerprint of the built rows. (Filled after the first build.) +EXPECTED_CONTENT_SHA256 = "3a3e615964a3ab63988cf441a153e38250b91cdcadbcd615d315b88ae3202ee4" + + +def keep_validation(df: pd.DataFrame) -> pd.DataFrame: + out = df[df["split"] == "validation"].reset_index(drop=True) + if len(out) != EXPECTED_SAMPLES: + raise SystemExit(f"expected {EXPECTED_SAMPLES} MMMU validation rows, got {len(out)} — TSV layout changed?") + return out + + +SPEC = BenchmarkSpec( + eval_id="mmmu_val", + vlmevalkit_name="MMMU_DEV_VAL", + url=URL, + md5=MD5, + prompt_style="mcq", + expected_samples=EXPECTED_SAMPLES, + expected_content_sha256=EXPECTED_CONTENT_SHA256, + filter_fn=keep_validation, + license="Apache-2.0 (dataset card)", + license_notes=( + "The MMMU dataset card is Apache-2.0, but the images come from " + "textbooks and online sources with uncleared rights, and the card " + "carries a takedown clause. You must self-assess redistribution and " + "commercial-use risk." + ), +) + +if __name__ == "__main__": + run_cli(SPEC, description=__doc__) diff --git a/packages/pipette-scores/scripts/build_mmstar_dataset.py b/packages/pipette-scores/scripts/build_mmstar_dataset.py new file mode 100644 index 0000000..a925732 --- /dev/null +++ b/packages/pipette-scores/scripts/build_mmstar_dataset.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python +"""One-shot materializer for the MMStar eval dataset (full set). + +Run from the repo root: + + uv run --project packages/pipette-scores \ + python packages/pipette-scores/scripts/build_mmstar_dataset.py + +Downloads VLMEvalKit's pinned ``MMStar.tsv`` (~60 MB; URL + MD5 from +VLMEvalKit @ 8eca8e5) and writes ``datasets/mmstar/2026.10.1/`` (test.parquet + +metadata.json). Gitignored, materialized at deploy — never committed. MMStar +may not be redistributed in transformed form (see docs/dataset-licenses.md). + +TSV columns: index, question, answer, category, l2_category, bench, image, +A, B, C, D. Prompt = VLMEvalKit MCQ format (no hint column in this TSV). +""" + +from pipette_scores.dataset_catalog._vlm_build import BenchmarkSpec, run_cli + +# --- Upstream source (pinned for reproducibility; VLMEvalKit @ 8eca8e5) --- +URL = "https://opencompass.openxlab.space/utils/VLMEval/MMStar.tsv" +MD5 = "e1ecd2140806c1b1bbf54b43372efb9e" + +# --- Expected output (the build asserts against these) --- +EXPECTED_SAMPLES = 1500 +# Fingerprint of the built rows. (Filled after the first build.) +EXPECTED_CONTENT_SHA256 = "9851cbed9513819f5aee26bbe2b92ea98b984fef59289ab36b085c13183f49f0" + +SPEC = BenchmarkSpec( + eval_id="mmstar", + vlmevalkit_name="MMStar", + url=URL, + md5=MD5, + prompt_style="mcq", + expected_samples=EXPECTED_SAMPLES, + expected_content_sha256=EXPECTED_CONTENT_SHA256, + license="none (no declared license)", + license_notes=( + "MMStar declares no license of its own and aggregates content from " + "source benchmarks including SEED-Bench (CC-BY-NC) and ScienceQA " + "(CC-BY-NC-SA), whose terms it inherits. You must self-assess whether " + "your use complies with those non-commercial terms." + ), +) + +if __name__ == "__main__": + run_cli(SPEC, description=__doc__) diff --git a/packages/pipette-scores/scripts/build_ocrbench_dataset.py b/packages/pipette-scores/scripts/build_ocrbench_dataset.py new file mode 100644 index 0000000..13f26bb --- /dev/null +++ b/packages/pipette-scores/scripts/build_ocrbench_dataset.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python +"""One-shot materializer for the OCRBench eval dataset (full set). + +Run from the repo root: + + uv run --project packages/pipette-scores \ + python packages/pipette-scores/scripts/build_ocrbench_dataset.py + +Downloads VLMEvalKit's pinned ``OCRBench.tsv`` (~114 MB; URL + MD5 from +VLMEvalKit @ 8eca8e5) and writes ``datasets/ocrbench/2026.10.1/`` (test.parquet + +metadata.json). Gitignored, materialized at deploy — never committed. +OCRBench's image sources may not be redistributed in transformed form (see +docs/dataset-licenses.md). + +TSV columns: index, image, question, answer, category. Prompt = the raw +question. ``answer`` is a stringified list of acceptable strings; ``category`` +is one of the 10 OCRBench categories, exact case (incl. the case-sensitive +``Handwritten Mathematical Expression Recognition``) — both carried verbatim +in raw_doc for the scorer. +""" + +from pipette_scores.dataset_catalog._vlm_build import BenchmarkSpec, run_cli + +# --- Upstream source (pinned for reproducibility; VLMEvalKit @ 8eca8e5) --- +URL = "https://opencompass.openxlab.space/utils/VLMEval/OCRBench.tsv" +MD5 = "e953d98a987cc6e26ef717b61260b778" + +# --- Expected output (the build asserts against these) --- +EXPECTED_SAMPLES = 1000 # OCRBench has exactly 1000 questions (Final Score denominator) +# Fingerprint of the built rows. (Filled after the first build.) +EXPECTED_CONTENT_SHA256 = "b46dd26966effff2319ab30520d479063cce5c217617cb047a7f2515ab4010fe" + +SPEC = BenchmarkSpec( + eval_id="ocrbench", + vlmevalkit_name="OCRBench", + url=URL, + md5=MD5, + prompt_style="bare", + expected_samples=EXPECTED_SAMPLES, + expected_content_sha256=EXPECTED_CONTENT_SHA256, + license="MIT (benchmark code); images: mixed, ~29 source OCR datasets", + license_notes=( + "The OCRBench repository code is MIT-licensed, but the images aggregate " + "roughly 29 OCR datasets with mixed and partly research-only terms. You " + "must self-assess whether your use complies with each source dataset's " + "terms." + ), +) + +if __name__ == "__main__": + run_cli(SPEC, description=__doc__) diff --git a/packages/pipette-scores/scripts/build_pope_dataset.py b/packages/pipette-scores/scripts/build_pope_dataset.py new file mode 100644 index 0000000..ced9d44 --- /dev/null +++ b/packages/pipette-scores/scripts/build_pope_dataset.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python +"""One-shot materializer for the POPE eval dataset (full set). + +Run from the repo root: + + uv run --project packages/pipette-scores \ + python packages/pipette-scores/scripts/build_pope_dataset.py + +Downloads VLMEvalKit's pinned ``POPE.tsv`` (~1.1 GB — the largest of the 7; +URL + MD5 from VLMEvalKit @ 8eca8e5) and writes ``datasets/pope/2026.10.1/`` +(test.parquet + metadata.json). Gitignored, materialized at deploy — never +committed. POPE's COCO/Flickr images may not be redistributed (see +docs/dataset-licenses.md). + +TSV columns: image, question, answer, category, index. Prompt = the raw +question ("Is there a ... in the image?"). ``category`` is the subset name +(adversarial / popular / random; may be comma-joined — the scorer explodes +it) and is carried verbatim in raw_doc. +""" + +from pipette_scores.dataset_catalog._vlm_build import BenchmarkSpec, run_cli + +# --- Upstream source (pinned for reproducibility; VLMEvalKit @ 8eca8e5) --- +URL = "https://opencompass.openxlab.space/utils/VLMEval/POPE.tsv" +MD5 = "c12f5acb142f2ef1f85a26ba2fbe41d5" + +# --- Expected output (the build asserts against these) --- +# Row count after VLMEvalKit's merge of overlapping subset questions +# (comma-joined categories). (Filled after the first build.) +EXPECTED_SAMPLES = 5127 # POPE.tsv rows (overlapping subset rows merged with comma-joined categories) +# Fingerprint of the built rows. (Filled after the first build.) +EXPECTED_CONTENT_SHA256 = "b47995b5ea4310771c31be26d9487fe8d3d9f75ae53f09cce8806c9147c680f0" + +SPEC = BenchmarkSpec( + eval_id="pope", + vlmevalkit_name="POPE", + url=URL, + md5=MD5, + prompt_style="bare", + expected_samples=EXPECTED_SAMPLES, + expected_content_sha256=EXPECTED_CONTENT_SHA256, + license="MIT (annotations); images: COCO val2014 / Flickr per-photo terms", + license_notes=( + "POPE annotations are MIT-licensed, but the images are COCO val2014 " + "photos governed by Flickr per-photo terms that do not permit " + "redistribution. Do not redistribute the built dataset." + ), +) + +if __name__ == "__main__": + run_cli(SPEC, description=__doc__) diff --git a/packages/pipette-scores/scripts/build_realworldqa_dataset.py b/packages/pipette-scores/scripts/build_realworldqa_dataset.py new file mode 100644 index 0000000..d781c38 --- /dev/null +++ b/packages/pipette-scores/scripts/build_realworldqa_dataset.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python +"""One-shot materializer for the RealWorldQA eval dataset (full set). + +Run from the repo root: + + uv run --project packages/pipette-scores \ + python packages/pipette-scores/scripts/build_realworldqa_dataset.py + +Downloads VLMEvalKit's pinned ``RealWorldQA.tsv`` (~184 MB; URL + MD5 from +VLMEvalKit @ 8eca8e5) and writes ``datasets/realworldqa/2026.10.1/`` (test.parquet ++ metadata.json). Gitignored, materialized at deploy — never committed. + +LICENSING: RealWorldQA is CC-BY-ND-4.0. The parquet this script produces is +Adapted Material under that license and MUST NOT be redistributed — build it +locally, use it locally (see docs/dataset-licenses.md). + +TSV columns: image, answer, index, A, B, C, D, question. Prompt = VLMEvalKit +MCQ format. +""" + +from pipette_scores.dataset_catalog._vlm_build import BenchmarkSpec, run_cli + +# --- Upstream source (pinned for reproducibility; VLMEvalKit @ 8eca8e5) --- +URL = "https://opencompass.openxlab.space/utils/VLMEval/RealWorldQA.tsv" +MD5 = "4de008f55dc4fd008ca9e15321dc44b7" + +# --- Expected output (the build asserts against these) --- +EXPECTED_SAMPLES = 765 +# Fingerprint of the built rows. (Filled after the first build.) +EXPECTED_CONTENT_SHA256 = "b05826e7058eb131c6ed787c5271e49454e911a01645f30feb71c7681bb1bb38" + +_ND_WARNING = ( + "*** RealWorldQA is CC-BY-ND-4.0 (NoDerivatives). The parquet file just\n" + "*** built is Adapted Material and MUST NOT be redistributed, shared,\n" + "*** uploaded, or served outside your organization in ANY form. Keep it\n" + "*** on the machine that runs the eval." +) + +SPEC = BenchmarkSpec( + eval_id="realworldqa", + vlmevalkit_name="RealWorldQA", + url=URL, + md5=MD5, + prompt_style="mcq", + expected_samples=EXPECTED_SAMPLES, + expected_content_sha256=EXPECTED_CONTENT_SHA256, + license="CC-BY-ND-4.0", + license_notes=( + "CC-BY-ND-4.0 prohibits sharing adaptations. The built parquet " + "(repacked images + reformatted prompts) is Adapted Material and must " + "not be redistributed in any form." + ), + extra_notice=_ND_WARNING, +) + +if __name__ == "__main__": + run_cli(SPEC, description=__doc__) diff --git a/packages/pipette-scores/scripts/build_vlm_datasets.sbatch b/packages/pipette-scores/scripts/build_vlm_datasets.sbatch new file mode 100644 index 0000000..0ac0b3d --- /dev/null +++ b/packages/pipette-scores/scripts/build_vlm_datasets.sbatch @@ -0,0 +1,50 @@ +#!/bin/bash +#SBATCH --job-name=build-vlm-datasets +#SBATCH --account=liquidai +#SBATCH --partition=defq +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=8 +#SBATCH --mem=64G +#SBATCH --time=06:00:00 +#SBATCH --output=logs/pipette/OUT_%x.%j +# +# Build all 7 VLM benchmark datasets (full sets) from VLMEvalKit's pinned +# upstream TSVs. CPU-only, I/O bound — one job loops the 7 builder scripts. +# +# From the pipette-scores repo root: +# mkdir -p logs/pipette +# sbatch packages/pipette-scores/scripts/build_vlm_datasets.sbatch +# +# Extra flags are forwarded to every builder, e.g.: +# sbatch packages/pipette-scores/scripts/build_vlm_datasets.sbatch --insecure --max-image-px 512 +# +# (--insecure skips TLS verification for opencompass.openxlab.space, whose +# certificate has been observed expired; integrity is still enforced via the +# MD5s pinned from VLMEvalKit.) +# +# Output: datasets//2026.10.1/{test.parquet,metadata.json}. LICENSING: the +# built datasets must never be committed or redistributed — see +# docs/dataset-licenses.md. + +set -euo pipefail + +# Resolve the repo root. Under slurm the batch script executes from a spooled +# COPY (e.g. /cm/local/apps/slurm/...), so BASH_SOURCE cannot locate the repo — +# use $SLURM_SUBMIT_DIR (the sbatch invocation dir, i.e. the repo root per the +# usage above). Fall back to BASH_SOURCE for direct local execution. +if [ -n "${SLURM_SUBMIT_DIR:-}" ]; then + PROJECT_ROOT="$SLURM_SUBMIT_DIR" +else + PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +fi +cd "$PROJECT_ROOT" + +SCRIPTS_DIR="packages/pipette-scores/scripts" + +for eval in mme mmstar ocrbench mmmu_val realworldqa mmbench_en_dev pope; do + echo "=== building ${eval} ===" + uv run --no-sync --project packages/pipette-scores \ + python "${SCRIPTS_DIR}/build_${eval}_dataset.py" "$@" +done + +echo "=== all 7 VLM datasets built ===" diff --git a/packages/pipette-scores/scripts/gen_math_500_liquid_evals_parity.py b/packages/pipette-scores/scripts/gen_math_500_liquid_evals_parity.py new file mode 100644 index 0000000..1868024 --- /dev/null +++ b/packages/pipette-scores/scripts/gen_math_500_liquid_evals_parity.py @@ -0,0 +1,247 @@ +#!/usr/bin/env python +"""Regenerate the liquid_evals MATH-500 grader parity fixture. + +Runs the ORIGINAL liquid_evals parser on the cases below and records its +outputs in tests/fixtures/math_500_liquid_evals_parity.json, which +tests/test_math_500_liquid_evals_parity.py checks the vendored copy +(pipette_scores.scoring.math_500.liquid_evals_parser) against. Run from the +repo root against a liquid_evals clone; the clone is only read (`git show`): + + uv run --project packages/pipette-scores \\ + python packages/pipette-scores/scripts/gen_math_500_liquid_evals_parity.py \\ + --liquid-evals /path/to/liquid_evals --rev d5534cf + +The original parser needs only sympy and pylatexenc, which this package pins +to the versions liquid_evals locks. +""" + +import argparse +import importlib.util +import json +import pathlib +import subprocess +import tempfile + +PARSER_PATH = "liquid_evals/parser.py" +FIXTURE = pathlib.Path(__file__).resolve().parents[1] / "tests" / "fixtures" / "math_500_liquid_evals_parity.json" + +# extract_math_answer inputs: each pattern in its priority order, and the +# brace counting of the \boxed{} scan. +EXTRACT_CASES = [ + "The answer is \\boxed{42}.", + "\\boxed{\\frac{1}{2}}", + "\\boxed{\\frac{\\sqrt{3}}{2}}", + "first \\boxed{3}, then corrected: \\boxed{5}", + "\\boxed{\\boxed{7}}", + "\\boxed{\\left( 3, \\frac{\\pi}{2} \\right)}", + "\\boxed{x \\in [-2, 7]}", + "unbalanced \\boxed{3 so the result is 4", + "\\boxed{}", + "empty box \\boxed{} then 12 apples", + "\\boxed {9} has a space", + "\\fbox{5} is not a boxed pattern", + "\\boxed{5\\text{ cm}}", + "\\boxed{90^\\circ}", + "$\\boxed{\\$18.90}$", + "Work.\n#### 18", + "Work.\n### Final Answer\n5", + "### 6\nmore text\n### 7", + "ANSWER: 7", + "answer: x = 3\nmore", + "The answer is [3, 4]", + "the answer is {x+1}", + "So the answer is (B).", + "Hence the answer is 12. Done", + "The final answer is 9", + "Final answer: 2/3", + "Answer: -4", + "We get 3.5 apples\n\n", + "The total is 1,000 dollars", + "no numbers here", + "", + "\\boxed{1}\\boxed{2}", +] + +# (given, ground truth) pairs for check_math_result and its parts: mathd and +# sympy equivalence, units, percents, tuples, intervals, text answers, and the +# guards that keep sympy off long or unbounded input. +CHECK_CASES = [ + ("42", "42"), + ("\\frac{1}{2}", "\\frac12"), + ("0.5", "\\frac{1}{2}"), + ("1/2", "\\frac{1}{2}"), + ("2/4", "\\frac{1}{2}"), + ("\\dfrac{3}{4}", "\\frac34"), + ("\\tfrac{3}{4}", "0.75"), + ("10\\%", "10"), + ("10%", "10"), + ("5 \\text{ cm}", "5"), + ("5\\text{ cm}\\text{ m}", "5"), + ("5 cm", "5"), + ("12 inches", "12"), + ("3 feet", "3"), + ("90^\\circ", "90"), + ("90^{\\circ}", "90"), + ("\\$5", "5"), + ("$5.00", "5"), + ("x=3", "3"), + ("k = 3", "3"), + ("y = 2x + 1", "2x+1"), + ("(1,2)", "(1, 2)"), + ("[1,2)", "(1,2)"), + ("(1,2,3)", "(1,2)"), + ("(-\\infty, 3]", "(-\\infty,3]"), + ("\\left( 3, \\frac{\\pi}{2} \\right)", "\\left( 3, \\frac{\\pi}{2} \\right)"), + ("(3, \\pi/2)", "\\left( 3, \\frac{\\pi}{2} \\right)"), + ("1,000", "1000"), + ("1,000,000", "10^6"), + ("3 million", "3000000"), + ("7 3/4", "\\frac{31}{4}"), + ("\\sqrt{2}/2", "\\frac{\\sqrt{2}}{2}"), + ("2\\sqrt3", "2\\sqrt{3}"), + ("2\\sqrt", "2\\sqrt{3}"), + ("\\frac", "\\frac{1}{2}"), + ("\\pi", "\\pi"), + ("2\\pi", "\\pi*2"), + ("\\infty", "\\infty"), + ("x^2+2x+1", "(x+1)^2"), + ("x^{2}+2x+1", "(x+1)^2"), + ("2^10", "1024"), + ("1024", "2^{10}"), + ("abc", "abd"), + ("\\text{Monday}", "\\text{Monday}"), + ("monday", "\\text{Monday}"), + ("Monday", "\\text{(C)}"), + ("C", "\\text{(C)}"), + ("(C)", "\\text{(C)}"), + ("1 and 2", "1, 2"), + ("1 or 2", "1,2"), + ("\\{1,2\\}", "1,2"), + ("10", "10.0"), + ("3.0", "3"), + ("-3", "- 3"), + (".5", "0.5"), + ("\\!1\\!000", "1000"), + ("\\frac{1}{3}", "0.333"), + ("0.333", "\\frac{1}{3}"), + ("1e400", "1"), + ("nan", "nan"), + ("", "5"), + ("5", ""), + ("The answer is 4", "4"), + ("4", "The answer is 4"), + ("x = \\frac{-b}{2a}", "-\\frac{b}{2a}"), + ("+".join(["x"] * 1600), "1600x"), + (" ".join(["\\alpha"] * 101), "1"), + ("\\begin{pmatrix} 1 \\\\ 2 \\end{pmatrix}", "\\begin{pmatrix} 1 \\\\ 2 \\end{pmatrix}"), +] + + +def _lines(n: int) -> str: + return "\n".join(f"step {i} of the work" for i in range(n)) + + +# (raw response, gold) for the tasks/math500.py pipeline: remove_thinking_tags, +# extract_math_answer, verify_math_answer. Covers the fallback over the last +# 150 lines and its counting: blank lines count toward the 150, but the cap is +# only checked on a non-blank line. +VERIFY_CASES = [ + ("So the answer is \\boxed{42}.", "42"), + ("So the answer is \\boxed{41}.", "42"), + ("\\boxed{\\frac{\\sqrt{3}}{2}}", "\\frac{\\sqrt{3}}{2}"), + ("\\boxed{\\dfrac{\\sqrt3}{2}}", "\\frac{\\sqrt{3}}{2}"), + ("\\boxed{\\boxed{7}}", "7"), + ("\\boxed{5\\text{ cm}}", "5"), + ("\\boxed{5 \\text{cm}}", "5"), + ("\\boxed{90^\\circ}", "90^\\circ"), + ("\\boxed{(3, \\frac{\\pi}{2})}", "\\left( 3, \\frac{\\pi}{2} \\right)"), + ("\\boxed{\\text{(C)}}", "\\text{(C)}"), + ("\\boxed{C}", "\\text{(C)}"), + ("The answer is 7", "7"), + ("Therefore, the final answer is 12", "12"), + ("Final answer: 5", "5"), + ("We compute.\nFinal answer: 5", "5"), + ("We compute 3 + 4.\nIt is 7 apples", "7"), + ("no answer here at all", "7"), + ("", "7"), + ("Hmm.\nMonday", "\\text{Monday}"), + ("\\boxed{3}\nThe answer is 4", "4"), + ("\\boxed{3}\n4", "4"), + ("4\n" + _lines(148) + "\n\\boxed{3}", "4"), + ("4\n" + _lines(149) + "\n\\boxed{3}", "4"), + ("4\n\n" + _lines(148) + "\n\\boxed{3}", "4"), + ("4\n\n\n" + _lines(148) + "\n\\boxed{3}", "4"), + ("maybe \\boxed{7}\nSo \\boxed{42}", "42"), + ("maybe \\boxed{42}\nSo \\boxed{7}", "42"), + ("reasoning \\boxed{42}\nThe answer is \\boxed{42}", "42"), + ("reasoning so far \\boxed{42} wait", "42"), + ("Partial work, so \\boxed{42}? Check the sign of", "42"), + ("Start.long reasoning \\boxed{42} that was cut", "42"), + ("[THINK]x\\boxed{1}[/THINK]\\boxed{2}", "2"), + ("analysisWe need 2+2.assistantfinal\\boxed{4}", "4"), + ("\\boxed{0.5}", "\\frac{1}{2}"), + ("\\boxed{2/4}", "\\frac{1}{2}"), + ("\\boxed{x^2 + 2x + 1}", "(x+1)^2"), + ("\\boxed{1,\\!000}", "1000"), + ("\\boxed{\\$18.90}", "\\$18.90"), + ("\\boxed{18.9}", "\\$18.90"), +] + + +def _load_original(repo: pathlib.Path, rev: str, tmp: pathlib.Path): + source = subprocess.run( + ["git", "-C", str(repo), "show", f"{rev}:{PARSER_PATH}"], check=True, capture_output=True, text=True + ).stdout + commit = subprocess.run( + ["git", "-C", str(repo), "rev-parse", f"{rev}^{{commit}}"], check=True, capture_output=True, text=True + ).stdout.strip() + blob = subprocess.run( + ["git", "-C", str(repo), "rev-parse", f"{rev}:{PARSER_PATH}"], check=True, capture_output=True, text=True + ).stdout.strip() + path = tmp / "liquid_evals_parser_original.py" + path.write_text(source) + spec = importlib.util.spec_from_file_location("liquid_evals_parser_original", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module, commit, blob + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--liquid-evals", type=pathlib.Path, required=True, help="liquid_evals clone (read only)") + parser.add_argument("--rev", default="d5534cf", help="liquid_evals revision to read parser.py at") + args = parser.parse_args() + + with tempfile.TemporaryDirectory() as tmp: + original, commit, blob = _load_original(args.liquid_evals, args.rev, pathlib.Path(tmp)) + + def verify(raw: str, gold: str) -> dict: + response = original.remove_thinking_tags(raw) + predicted = original.extract_math_answer(response) + correct = original.verify_math_answer(gold, predicted, response) + return {"raw": raw, "gold": gold, "predicted": predicted, "correct": bool(correct)} + + fixture = { + "source": {"repo": "Liquid4All/liquid_evals", "commit": commit, "path": PARSER_PATH, "blob": blob}, + "extract": [{"text": t, "expected": original.extract_math_answer(t)} for t in EXTRACT_CASES], + "check": [ + { + "given": given, + "truth": truth, + "mathd_normalize_given": original.mathd_normalize_answer(given), + "normalize_given": original._normalize(given), + "mathd": original.grade_answer_mathd(given, truth), + "sympy": original.grade_answer_sympy(given, truth), + "check": original.check_math_result(given, truth), + } + for given, truth in CHECK_CASES + ], + "verify": [verify(raw, gold) for raw, gold in VERIFY_CASES], + } + FIXTURE.parent.mkdir(parents=True, exist_ok=True) + FIXTURE.write_text(json.dumps(fixture, indent=1, ensure_ascii=False) + "\n") + print(f"Wrote {FIXTURE} from {PARSER_PATH} @ {commit}") + + +if __name__ == "__main__": + main() diff --git a/packages/pipette-scores/scripts/setup_asr_reference_env.sh b/packages/pipette-scores/scripts/setup_asr_reference_env.sh new file mode 100755 index 0000000..9b727a9 --- /dev/null +++ b/packages/pipette-scores/scripts/setup_asr_reference_env.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Build a throwaway environment holding the PUBLIC reference implementations the +# ASR scoring and audio are checked against (tests/test_asr_public_reference.py): +# +# - huggingface/open_asr_leaderboard at a pinned commit: its own normalizers +# (English, and the multilingual one FLEURS is scored with, which spells +# numbers out with num2words 0.5.14), row filter, chunk merge, compound-pair +# step and kaldialign WER (normalizer/eval_utils.py, normalizer/data_utils.py); +# - datasets 2.19.0 with librosa and soundfile: datasets.Audio(sampling_rate=16000) +# decoding, as the leaderboard's prepare_data applies it. +# +# None of this is a dependency of pipette-scores. Usage: +# +# packages/pipette-scores/scripts/setup_asr_reference_env.sh /tmp/asr-reference +# PIPETTE_ASR_REFERENCE_ENV=/tmp/asr-reference uv run pytest \ +# packages/pipette-scores/tests/test_asr_public_reference.py +# +# Needs uv, git and network access. +set -euo pipefail + +OPEN_ASR_LEADERBOARD_COMMIT=62a457c5c32d0de4df77e9470308aae7d45b055f + +dir="${1:?usage: $0 DIR}" +mkdir -p "$dir" +dir="$(cd "$dir" && pwd)" + +uv venv --quiet --python 3.12 "$dir/venv" +uv pip install --quiet --python "$dir/venv/bin/python" \ + "datasets==2.19.0" "librosa==1.0.0" "soxr==1.1.0" "soundfile==0.14.0" \ + "kaldialign==0.12.0" "num2words==0.5.14" "regex==2026.4.4" "numpy" + +if [[ ! -d "$dir/open_asr_leaderboard/.git" ]]; then + git clone --quiet https://github.com/huggingface/open_asr_leaderboard "$dir/open_asr_leaderboard" +fi +git -C "$dir/open_asr_leaderboard" fetch --quiet origin +git -C "$dir/open_asr_leaderboard" checkout --quiet --detach "$OPEN_ASR_LEADERBOARD_COMMIT" + +echo "reference environment ready: PIPETTE_ASR_REFERENCE_ENV=$dir" diff --git a/packages/pipette-scores/scripts/split_asr_dataset.py b/packages/pipette-scores/scripts/split_asr_dataset.py new file mode 100644 index 0000000..f1dd9c1 --- /dev/null +++ b/packages/pipette-scores/scripts/split_asr_dataset.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python +"""Lay out built ASR datasets for git-LFS: a ``test.parquet`` over the per-file +bound is rewritten as ordered parts ``test-00000-of-0000N.parquet``, … (cut +between row groups, rows in order); one under it is copied as it is. +``metadata.json`` is copied byte for byte: its ``content_sha256`` is a function of +the rows, not of the files, so the parts carry the build's fingerprint, and the +server reads them as one table (_asr_common.parquet_files). + + uv run --project packages/pipette-scores \\ + python packages/pipette-scores/scripts/split_asr_dataset.py \\ + --src /data/pipette-scores/datasets --out /datasets [EVAL_ID ...] + +Writes ``///`` from ``///`` and then +runs the server's integrity checks on the result (verify_asr_datasets.py re-hashes +every clip). The default bound, 1.9 GB, stays under GitHub's smallest git-LFS +per-file limit (2 GB; 5 GB on Enterprise Cloud). +""" + +import argparse +import pathlib +import shutil +import sys + +from pipette_scores.dataset_catalog import _asr_common, asr_datasets +from pipette_scores.types import EvalId + +DEFAULT_MAX_PART_BYTES = 1_900_000_000 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("eval_ids", nargs="*", help="EvalIds to lay out (default: every ASR eval built under --src)") + parser.add_argument("--src", type=pathlib.Path, required=True, help="built datasets root (…/datasets)") + parser.add_argument("--out", type=pathlib.Path, required=True, help="datasets root to write (…/datasets)") + parser.add_argument("--version", default=asr_datasets.DATASET_VERSION) + parser.add_argument("--max-part-bytes", type=int, default=DEFAULT_MAX_PART_BYTES) + args = parser.parse_args() + ids = [EvalId(e) for e in args.eval_ids] or [ + e for e in asr_datasets.ASR_DATASETS if (args.src / e.value / args.version).is_dir() + ] + for eval_id in ids: + src = args.src / eval_id.value / args.version + out = args.out / eval_id.value / args.version + if out.exists() and any(out.iterdir()): + raise SystemExit(f"{out} is not empty; remove it first") + files = _asr_common.split_parts(src / _asr_common.SINGLE_FILE, out, args.max_part_bytes) + shutil.copyfile(src / "metadata.json", out / "metadata.json") + rows = _asr_common.verified_rows(out) + sizes = ", ".join(f"{f.stat().st_size / 1e9:.2f}" for f in files) + print(f"{eval_id.value}: {len(rows)} clips in {len(files)} file(s) of {sizes} GB, checks pass", flush=True) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/packages/pipette-scores/scripts/verify_asr_datasets.py b/packages/pipette-scores/scripts/verify_asr_datasets.py new file mode 100644 index 0000000..63adbe5 --- /dev/null +++ b/packages/pipette-scores/scripts/verify_asr_datasets.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python +"""Offline integrity check of materialized ASR datasets: every clip's audio bytes +against its recorded sha256, then the checks the server makes before it scores or +serves a dataset (content_sha256 against metadata.json and, for the canonical +version, source_sha256 against the catalog's expected_source_sha256; each sample id its position; the +scoring recipe, language and clip count the catalog's; no smoke build). A set +split into test--of-.parquet parts is checked as the one table they hold. + + uv run --project packages/pipette-scores \\ + python packages/pipette-scores/scripts/verify_asr_datasets.py --root /data/pipette-scores/datasets +""" + +import argparse +import hashlib +import pathlib +import sys + +import pyarrow.parquet as pq + +from pipette_scores.dataset_catalog import _asr_common, asr_datasets + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--root", type=pathlib.Path, required=True, help="datasets root (…/datasets)") + parser.add_argument("--version", default=asr_datasets.DATASET_VERSION) + args = parser.parse_args() + failed = 0 + for eval_id, dataset in asr_datasets.ASR_DATASETS.items(): + d = args.root / eval_id.value / args.version + if not d.is_dir(): + continue + bad = clips = 0 + files: list[pathlib.Path] = [] + digest = None + try: + files = _asr_common.parquet_files(d) + for path in files: + file = pq.ParquetFile(path) + clips += file.metadata.num_rows + for batch in file.iter_batches(columns=["audio", "audio_sha256"], batch_size=256): + pairs = zip(batch.column("audio").to_pylist(), batch.column("audio_sha256").to_pylist()) + bad += sum(hashlib.sha256(audio).hexdigest() != digest for audio, digest in pairs) + digest = _asr_common.source_sha256(_asr_common.verified_rows(d)) + problem = "" + # OSError and ValueError cover a missing file, bad JSON and a corrupt parquet. + except (OSError, ValueError) as e: + problem = str(e) + pinned = "pinned" if dataset.expected_source_sha256 else "NOT PINNED" + status = "ok" if not bad and not problem else "FAILED" + failed += status != "ok" + layout = f"{len(files)} parts" if len(files) > 1 else (files[0].name if files else "no parquet") + print(f"{eval_id.value}: {status} ({clips} clips in {layout}, {bad} audio mismatches, {pinned} {digest})") + if problem: + print(f" {problem}") + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/packages/pipette-scores/scripts/verify_vlm_datasets.py b/packages/pipette-scores/scripts/verify_vlm_datasets.py new file mode 100644 index 0000000..020095b --- /dev/null +++ b/packages/pipette-scores/scripts/verify_vlm_datasets.py @@ -0,0 +1,275 @@ +#!/usr/bin/env python +"""Integrity verifier for the 7 built VLM benchmark datasets. + +Run from the repo root after `build_vlm_datasets.sbatch` (or the individual +builders): + + uv run --no-sync --project packages/pipette-scores \ + python packages/pipette-scores/scripts/verify_vlm_datasets.py \ + [--datasets-root datasets] [--version 2026.10.1] [--evals mme,pope,...] + +Checks, per eval: + - metadata.json present with num_samples / source / license fields + - parquet row count == metadata num_samples (and == the known full-set size + where one is pinned below) + - sample_ids unique and non-empty; prompts non-empty; raw_doc parses as JSON + and carries the per-eval fields the scorers need (PORTING_NOTES.md) + - EVERY image decodes: PIL verify() (header pass) + full load() (pixel pass) + - image_i columns are populated exactly for i < num_images + - the image-inclusive content_sha256, recomputed from the parquet, equals + metadata.json's and, for a full original-resolution build, the builder + script's pinned EXPECTED_CONTENT_SHA256 (so a substituted image fails) + - loader round-trip: dataset_catalog.load_eval_samples / load_prompt_samples + return the same count and well-formed multimodal messages (data-URI mime) + - mmbench_en_dev: circular rotation groups (index % 1e6) are complete enough + to score (every group's rotations all present) + - pope: every exploded category is one of random/popular/adversarial + +Exit code 0 = all checks pass; 1 = any failure (summary printed either way). +""" + +import argparse +import importlib +import io +import json +import pathlib +import sys +from collections import Counter, defaultdict + +import pyarrow.parquet as pq +from PIL import Image + +from pipette_scores.dataset_catalog._vlm_build import content_sha256 + +# Known full-set sizes (None = not pinned yet; report only). +EXPECTED_ROWS: dict[str, int | None] = { + "mme": 2374, + "mmstar": 1500, + "ocrbench": 1000, + "mmmu_val": 900, + "realworldqa": 765, + "mmbench_en_dev": 4876, + "pope": 5127, +} + +# raw_doc fields each scorer consumes (see scoring/vlm_upstream/PORTING_NOTES.md). +REQUIRED_RAW_DOC_KEYS: dict[str, set[str]] = { + "mme": {"category"}, # + image_path or question_id (pairing key, checked below) + "mmstar": {"answer"}, + "ocrbench": {"category"}, + "mmmu_val": {"answer"}, + "realworldqa": {"answer"}, + "mmbench_en_dev": {"answer", "index"}, + "pope": {"answer", "category"}, +} + +POPE_CATEGORIES = {"random", "popular", "adversarial"} + + +def _builder_pin(eval_id: str) -> str: + """The image-inclusive digest the eval's builder script pins: a trusted + value from outside the dataset artifact.""" + sys.path.insert(0, str(pathlib.Path(__file__).parent)) + try: + return importlib.import_module(f"build_{eval_id}_dataset").SPEC.expected_content_sha256 + finally: + sys.path.pop(0) + + +def fail(errors: list[str], msg: str) -> None: + errors.append(msg) + print(f" FAIL: {msg}") + + +def verify_eval(eval_id: str, root: pathlib.Path, version: str) -> list[str]: + errors: list[str] = [] + d = root / eval_id / version + print(f"\n=== {eval_id} ({d}) ===") + meta_path = d / "metadata.json" + pq_path = d / "test.parquet" + if not meta_path.exists() or not pq_path.exists(): + fail(errors, f"missing {meta_path.name if not meta_path.exists() else pq_path.name}") + return errors + + meta = json.loads(meta_path.read_text()) + for key in ("num_samples", "source", "license"): + if key not in meta: + fail(errors, f"metadata.json missing '{key}'") + + table = pq.read_table(pq_path) + n = table.num_rows + print(f" rows: {n}") + if meta.get("num_samples") != n: + fail(errors, f"metadata num_samples={meta.get('num_samples')} != parquet rows={n}") + expected = EXPECTED_ROWS.get(eval_id) + if expected is not None and n != expected: + fail(errors, f"expected full-set size {expected}, got {n}") + + cols = table.column_names + image_cols = sorted((c for c in cols if c.startswith("image_")), key=lambda c: int(c.split("_")[1])) + rows = table.to_pylist() + + # --- id/prompt/raw_doc checks --- + ids = [r["sample_id"] for r in rows] + if len(set(ids)) != len(ids): + fail(errors, f"{len(ids) - len(set(ids))} duplicate sample_ids") + if any(not i for i in ids): + fail(errors, "empty sample_id present") + empty_prompts = sum(1 for r in rows if not (r["prompt"] or "").strip()) + if empty_prompts: + fail(errors, f"{empty_prompts} empty prompts") + + required = REQUIRED_RAW_DOC_KEYS[eval_id] + bad_raw, missing_keys = 0, Counter() + mme_missing_pair_key = 0 + for r in rows: + try: + doc = json.loads(r["raw_doc"]) + except (TypeError, json.JSONDecodeError): + bad_raw += 1 + continue + for k in required: + if k not in doc: + missing_keys[k] += 1 + if eval_id == "mme" and not (doc.get("image_path") or doc.get("question_id")): + mme_missing_pair_key += 1 + if bad_raw: + fail(errors, f"{bad_raw} raw_doc values are not valid JSON") + for k, c in missing_keys.items(): + fail(errors, f"raw_doc missing '{k}' in {c} rows") + if mme_missing_pair_key: + fail(errors, f"{mme_missing_pair_key} MME rows lack a pairing key (image_path/question_id)") + + # --- image integrity: decode every image fully --- + corrupt = 0 + fmt_counter: Counter = Counter() + num_images_mismatch = 0 + for idx, r in enumerate(rows): + n_img = r["num_images"] + for i, col in enumerate(image_cols): + blob = r[col] + if i < n_img: + if blob is None: + num_images_mismatch += 1 + continue + try: + img = Image.open(io.BytesIO(blob)) + img.verify() # header/CRC pass + img = Image.open(io.BytesIO(blob)) + img.load() # full pixel decode + fmt_counter[img.format] += 1 + except Exception as e: # noqa: BLE001 — any decode failure is corruption + corrupt += 1 + if corrupt <= 3: + print(f" corrupt image: row {idx} ({r['sample_id']}) {col}: {e}") + elif blob is not None: + num_images_mismatch += 1 + total_imgs = sum(fmt_counter.values()) + print(f" images decoded: {total_imgs} formats: {dict(fmt_counter)}") + if corrupt: + fail(errors, f"{corrupt} corrupt/undecodable images") + if num_images_mismatch: + fail(errors, f"{num_images_mismatch} rows where image_i population != num_images") + + # --- content: the image-inclusive digest, against the build's and the pin --- + # A decodable image is not the right image: only the digest over every + # image's bytes tells a substituted one apart. The serving loader checks + # the image-free metadata digest alone. + digest = content_sha256([{**r, "_images": [r[col] for col in image_cols[: r["num_images"]]]} for r in rows]) + if digest != meta.get("content_sha256"): + fail(errors, f"content_sha256 {digest} != metadata.json's {meta.get('content_sha256')}") + pinned = _builder_pin(eval_id) + if meta.get("max_image_px") is None and meta.get("limit") is None and pinned: + if digest != pinned: + fail(errors, f"content_sha256 {digest} != the builder's pinned {pinned}") + else: + print(" content_sha256 matches the builder's pin") + else: + print(" note: a resized or partial build; content_sha256 checked against metadata.json only") + + # --- per-eval structural checks --- + if eval_id == "mmbench_en_dev": + groups: dict[int, list[dict]] = defaultdict(list) + for r in rows: + doc = json.loads(r["raw_doc"]) + groups[int(doc["index"]) % 1_000_000].append(doc) + sizes = Counter(len(v) for v in groups.values()) + print(f" circular groups: {len(groups)} group-size distribution: {dict(sizes)}") + # Upstream MMBench_DEV_EN_V11.tsv itself ships some groups with fewer + # rotations than options (28 groups) — VLMEvalKit scores those as-is, + # so option-count shortfall is + # informational. A build-side row loss instead shows up as + # NON-CONSECUTIVE or duplicated rotation offsets: fail only on that. + short, broken = 0, 0 + for g in groups.values(): + offsets = sorted(int(doc["index"]) // 1_000_000 for doc in g) + if offsets != list(range(len(offsets))): + broken += 1 + continue + n_opts = sum(1 for k in ("A", "B", "C", "D") if (g[0].get(k) or "") != "") + if len(g) != n_opts: + short += 1 + if short: + print(f" note: {short} groups have fewer rotations than options (upstream ships these)") + if broken: + fail( + errors, + f"{broken} circular groups with non-consecutive/duplicate rotation offsets (build-side row loss)", + ) + if eval_id == "pope": + bad_cat = Counter() + for r in rows: + doc = json.loads(r["raw_doc"]) + for c in str(doc.get("category", "")).split(","): + if c.strip() and c.strip() not in POPE_CATEGORIES: + bad_cat[c.strip()] += 1 + if bad_cat: + fail(errors, f"unexpected POPE categories: {dict(bad_cat)}") + + # --- loader round-trip --- + from pipette_scores import dataset_catalog + from pipette_scores.types import EvalId + + eid = EvalId(eval_id) + scoring_samples = dataset_catalog.load_eval_samples(eid, version, root_dir=root.parent) + prompt_samples = dataset_catalog.load_prompt_samples(eid, version, root_dir=root.parent) + if len(scoring_samples) != n or len(prompt_samples) != n: + fail(errors, f"loader round-trip counts: eval={len(scoring_samples)} prompt={len(prompt_samples)} != {n}") + else: + first = prompt_samples[0] + content = first.messages[0].content + blocks = content if isinstance(content, list) else [] + img_blocks = [b for b in blocks if b.get("type") == "image_url"] + ok_uri = all(b["image_url"]["url"].startswith("data:image/") for b in img_blocks) + if not img_blocks or not ok_uri: + fail(errors, f"prompt sample malformed: {len(img_blocks)} image blocks, data-URI ok={ok_uri}") + else: + print(f" loader round-trip OK ({len(img_blocks)} image block(s) in first sample)") + + if not errors: + print(" PASS") + return errors + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--datasets-root", default="datasets") + ap.add_argument("--version", default="2026.10.1") + ap.add_argument("--evals", default=",".join(EXPECTED_ROWS)) + args = ap.parse_args() + + root = pathlib.Path(args.datasets_root).resolve() + all_errors: dict[str, list[str]] = {} + for eval_id in args.evals.split(","): + eval_id = eval_id.strip() + all_errors[eval_id] = verify_eval(eval_id, root, args.version) + + print("\n================ SUMMARY ================") + failed = {k: v for k, v in all_errors.items() if v} + for eval_id in all_errors: + print(f" {eval_id:16s} {'FAIL (' + str(len(all_errors[eval_id])) + ')' if eval_id in failed else 'PASS'}") + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/packages/pipette-scores/tests/asr_public_reference.py b/packages/pipette-scores/tests/asr_public_reference.py new file mode 100644 index 0000000..4fd9927 --- /dev/null +++ b/packages/pipette-scores/tests/asr_public_reference.py @@ -0,0 +1,590 @@ +"""The corpus the ASR scoring and audio are checked on, and the call into the public references. + +Not a test module (tests/test_asr_public_reference.py uses it). The corpus: + +- a synthetic part, generated here deterministically: English sentences rich in + what the normalizers rewrite (numbers, currency, ordinals, abbreviations, + contractions, British spellings, fillers, bracketed noise markers, compounds, + Unicode punctuation and diacritics), sentences in every FLEURS language + evaluated (with digits, number words, compounds and bracketed spans), and + their hypotheses as perturbed references (words dropped, inserted, + substituted, compounds split and joined, neighbouring words cut at another + point, numbers spelled out or written as digits, case and punctuation changed, + empty answers, Earnings22-style chunked calls); +- a real part: every test transcript of each FLEURS locale (google/fleurs' + ``transcription``, the reference, and ``raw_transcription``, at the pinned + revision), and, when built datasets are found under + ``PIPETTE_SCORES_DATA_DIR``, every reference of each, with the same + perturbations, and a few of its clips. + +`run_reference` answers it with the reference environment's code +(asr_public_reference_runner.py); every input carries a name, which the answer +echoes, and the answers must be exactly the inputs' names in order. +``python tests/asr_public_reference.py --write-fixture`` (with +``PIPETTE_ASR_REFERENCE_ENV`` set) records the answers on the synthetic part in +fixtures/asr_public_reference.json, which the default test run compares against +without the reference environment. +""" + +from __future__ import annotations + +import argparse +import base64 +import hashlib +import io +import json +import math +import os +import pathlib +import random +import subprocess +import sys +import tempfile +from typing import Any + +HERE = pathlib.Path(__file__).resolve().parent +RUNNER = HERE / "asr_public_reference_runner.py" +FIXTURE = HERE / "fixtures" / "asr_public_reference.json" + +# --- synthetic English ------------------------------------------------------- + +_ENGLISH = [ + "Mr. Smith won't go, um, at 10:00 a.m. on the 3rd of May.", + "It costs twenty five dollars, or $25.50 with tax, about 3.5% more.", + "The colour of the Centre's theatre was grey; they organised a travelling show.", + "Dr. O'Neil's 2nd favourite is the jewellery box from the U.S.A., e.g. the 1990s model.", + "Uh, hmm, mm, I think, uh-huh, that's right.", + "[noise] We'll see you (laughs) at the {breath} meeting.", + "IGNORE TIME SEGMENT IN SCORING", + " [noise]", + "", + "The football white paper was e-mailed to the re-elected board.", + "Revenue grew five percent this quarter to $1.2 billion, up from $1,150,000,000.", + "She ran 5km in 21 minutes and lifted 100kg; a 5mm gap remained.", + "One hundred and five people, 1/2 of them under 18, met on 7 July 2021.", + "Call 555-0199 or e-mail info@example.com before 5 p.m.", + "“Café” and “naïve” are spelled with diacritics — Œuvre, too…", + "We'd've gone if y'all hadn't said it's 11 o'clock.", + "The St. Louis Cardinals won 4-3 in the 9th inning, Gov. Brown said.", + "Our EBITDA margin was 23.4 percent, versus twenty-two point one percent last year.", + "Thirty-three thousand three hundred and thirty-three, 33,333 and 33333.", + 'I said, "Don\'t!" — but he did; it was okay, OK, ok.', + "First, second, 21st, 102nd and the 1,000th customer.", + "£20, €5, ¥300 and 15 cents.", + "Mmm, mhm, mm-hmm.", + "The analyse, analyze; catalogue, catalog; programme, program question.", + "Q3 2024 results: EPS of $0.87, guidance of $3.40 to $3.50.", +] +_FILLERS = ["um", "uh", "the", "and", "really", "twenty", "a", "of"] +_NUMBER_WORDS = { + "5": "five", + "25": "twenty five", + "3rd": "third", + "10:00": "ten o'clock", + "100kg": "one hundred kilograms", + "1/2": "half", + "18": "eighteen", + "2nd": "second", + "4-3": "four three", +} + + +def _perturb(text: str, rng: random.Random) -> str: + words = text.split() + op = rng.randrange(11) + if op == 0 or not words: + return text + if op == 1: + del words[rng.randrange(len(words))] + elif op == 2: + words.insert(rng.randrange(len(words) + 1), rng.choice(_FILLERS)) + elif op == 3: + words[rng.randrange(len(words))] = rng.choice(_FILLERS) + elif op == 4: + # Split a long word: a compound split the leaderboard's merge_compounds forgives. + long = [i for i, w in enumerate(words) if len(w) > 6] + if long: + i = rng.choice(long) + cut = rng.randrange(3, len(words[i]) - 2) + words[i : i + 1] = [words[i][:cut], words[i][cut:]] + elif op == 5 and len(words) > 1: + # Join two words. + i = rng.randrange(len(words) - 1) + words[i : i + 2] = [words[i] + words[i + 1]] + elif op == 6: + words = [_NUMBER_WORDS.get(w.strip(".,;"), w) for w in words] + elif op == 7: + return text.lower() if rng.random() < 0.5 else text.upper() + elif op == 8: + return "".join(c for c in text if c.isalnum() or c.isspace()) + elif op == 9: + return "" + elif op == 10: + i = rng.randrange(len(words)) + words = words[: i + 1] + words[: i + 1] + words[i + 1 :] + return " ".join(words) + + +def synthetic_english(n: int = 1200, seed: int = 0) -> list[tuple[str, str]]: + rng = random.Random(seed) + pairs = [] + for k in range(n): + reference = _ENGLISH[k % len(_ENGLISH)] + if k >= len(_ENGLISH): + # Recombine sentences so lengths and contexts vary. + reference = " ".join(rng.sample(_ENGLISH, rng.randrange(1, 4))) + hypothesis = reference + for _ in range(rng.randrange(0, 4)): + hypothesis = _perturb(hypothesis, rng) + pairs.append((reference, hypothesis)) + return pairs + + +def synthetic_calls(n_calls: int = 6, seed: int = 1) -> list[dict[str, Any]]: + """Earnings22-style rows: each chunk carries its call's transcript, the call + (parent_id) and its chunk_index; chunks out of order, some answers blank.""" + rng = random.Random(seed) + rows = [] + for call in range(n_calls): + transcript = " ".join(rng.choice(_ENGLISH) for _ in range(rng.randrange(8, 20))) + answer = transcript + for _ in range(3): + answer = _perturb(answer, rng) + words = answer.split() + n = rng.randrange(2, 7) + cuts = sorted(rng.sample(range(1, max(len(words), n + 1)), n - 1)) if len(words) > n else [] + pieces = [" ".join(words[a:b]) for a, b in zip([0, *cuts], [*cuts, len(words)])] + pieces += [""] * (n - len(pieces)) + chunks = [ + { + "text": transcript, + "pred_text": f" {piece} " if rng.random() < 0.9 else " ", + "parent_id": f"call-{call}", + "chunk_index": i, + } + for i, piece in enumerate(pieces) + ] + rng.shuffle(chunks) + rows += chunks + return rows + + +# --- synthetic FLEURS -------------------------------------------------------- + +_FLEURS_TEXT: dict[str, list[str]] = { + "eng": _ENGLISH, + "spa": [ + "El niño comió 25 manzanas, ¿verdad?", + "Mañana, a las 10:30, llegará el tren de Málaga.", + "En 1999 había 10 000 habitantes (según el censo) y veinte escuelas.", + ], + "por": [ + "A população é de 1.234.567 habitantes; não é?", + "Ela disse: «Obrigada!» às três horas.", + "O gabinete afirmou que 19 dos feridos eram agentes policiais [ruído] em 2006.", + ], + "deu": [ + "Löwenrudel agieren ähnlich wie Wolfs- oder Hunderudel.", + "Die Donaudampfschifffahrtsgesellschaft fährt um 9 Uhr ab — pünktlich!", + "Rund 1000000 Menschen sahen am 3. Oktober zwanzig Schiffe (laut Polizei).", + ], + "fra": [ + "L'été dernier, j'ai payé 12,50 € pour un café à Noël.", + "Où est-ce qu'il a mis les clés ?", + "En 1900, vingt-cinq des 25 voyageurs avaient « environ » 30 ans.", + ], + "ita": [ + "Perché l'università è chiusa il 25 aprile?", + "È così: più o meno 3 ore.", + "Nel dopoguerra, a partire dal 2009, venti visti (separati) bastavano.", + ], + "nld": [ + "Het is één uur 's nachts, zei de buurman.", + "De fietsenmaker repareerde twee-en-twintig banden.", + "Volgens Angel (2006) kunnen 10 000 organisaties 11e of 12e eeuwse modellen gebruiken.", + ], +} + +# Number words a hypothesis may write for a reference's digits, or the reverse +# (fixed here, so that the corpus does not depend on num2words). +_FLEURS_NUMBERS: dict[str, dict[str, str]] = { + "eng": {"25": "twenty five", "3rd": "third", "18": "eighteen"}, + "spa": {"25": "veinticinco", "1999": "mil novecientos noventa y nueve", "veinte": "20"}, + "por": {"19": "dezenove", "2006": "dois mil e seis", "três": "3"}, + "deu": {"9": "neun", "zwanzig": "20", "3.": "dritten"}, + "fra": {"25": "vingt-cinq", "1900": "mille neuf cents", "30": "trente", "vingt-cinq": "25"}, + "ita": {"25": "venticinque", "2009": "duemilanove", "venti": "20", "3": "tre"}, + "nld": {"2006": "tweeduizend zes", "één": "1", "10": "tien"}, +} + + +def _spell_numbers(text: str, language: str, rng: random.Random) -> str: + """Some of the text's numbers written the other way: digits as words, or words as digits.""" + words = _FLEURS_NUMBERS[language] + return " ".join( + words[w.strip(".,;")] if w.strip(".,;") in words and rng.random() < 0.7 else w for w in text.split() + ) + + +def _resegment(text: str, rng: random.Random) -> str: + """Two neighbouring words cut at another point ("zum beispiel" as "zumbei + spiel"): what the leaderboard's normalize_compound_pairs forgives.""" + words = text.split() + pairs = [i for i in range(len(words) - 1) if len(words[i]) + len(words[i + 1]) > 4] + if not pairs: + return text + i = rng.choice(pairs) + joined = words[i] + words[i + 1] + cut = rng.choice([c for c in range(1, len(joined)) if c != len(words[i])]) + words[i : i + 2] = [joined[:cut], joined[cut:]] + return " ".join(words) + + +def fleurs_config(eval_id: str) -> str: + """The leaderboard's config name for a FLEURS eval's language (``fleurs_``), + which the reference reads the language code from, as its run_eval_ml.py does.""" + from pipette_scores.dataset_catalog import asr_datasets + from pipette_scores.scoring.asr.normalize import OPEN_ASR_LANGUAGE_CODES + from pipette_scores.types import EvalId + + return f"fleurs_{OPEN_ASR_LANGUAGE_CODES[asr_datasets.ASR_DATASETS[EvalId(eval_id)].language]}" + + +def synthetic_fleurs(seed: int = 2, per_language: int = 60) -> list[dict[str, Any]]: + from pipette_scores.dataset_catalog import asr_datasets + + rng = random.Random(seed) + datasets = [] + for locale, language, _ in asr_datasets.FLEURS_LOCALES: + texts = _FLEURS_TEXT[language] + pairs = [] + for k in range(per_language): + reference = texts[k % len(texts)] if k < len(texts) else " ".join(rng.sample(texts, len(texts))) + hypothesis = _spell_numbers(reference, language, rng) if rng.random() < 0.5 else reference + for _ in range(rng.randrange(0, 4)): + hypothesis = _perturb(hypothesis, rng) + if rng.random() < 0.3: + hypothesis = _resegment(hypothesis, rng) + pairs.append((reference, hypothesis)) + eval_id = f"asr_fleurs_{locale}" + datasets.append({"eval_id": eval_id, "config": fleurs_config(eval_id), "language": language, "pairs": pairs}) + return datasets + + +# --- synthetic audio --------------------------------------------------------- + + +def synthetic_clips() -> list[bytes]: + """Clips at the rates, channel counts, subtypes and containers a source may + ship: 8 to 48 kHz, mono to three channels, 16/24-bit, float, WAV and FLAC, + odd lengths. Their samples are deterministic (their bytes are not: a float + WAV written by libsndfile carries a timestamp).""" + import numpy as np + import soundfile as sf + + rng = np.random.default_rng(3) + clips = [] + for rate, channels, subtype, fmt, seconds in [ + (16000, 1, "PCM_16", "WAV", 1.0), + (16000, 2, "PCM_16", "WAV", 0.7), + (44100, 1, "PCM_16", "WAV", 2.0), + (44100, 1, "PCM_16", "WAV", 0.50001), + (44100, 1, "PCM_16", "WAV", 0.50003), # 22,051 frames: 8000.36 at 16 kHz, so the length rule matters + (44100, 2, "FLOAT", "WAV", 1.3), + (48000, 2, "PCM_24", "WAV", 0.9), + (48000, 3, "PCM_16", "WAV", 0.4), + (22050, 1, "PCM_16", "FLAC", 1.1), + (24000, 1, "PCM_16", "FLAC", 0.6), + (32000, 1, "FLOAT", "WAV", 0.33), + (8000, 1, "PCM_16", "WAV", 1.7), + (11025, 2, "PCM_16", "FLAC", 0.8), + ]: + n = int(seconds * rate) + t = np.arange(n) / rate + tone = 0.2 * np.sin(2 * np.pi * 440 * t)[:, None] * (1 + np.arange(channels))[None, :] / channels + data = np.clip(tone + 0.05 * rng.standard_normal((n, channels)), -1, 1) + buffer = io.BytesIO() + sf.write(buffer, data, rate, format=fmt, subtype=subtype) + clips.append(buffer.getvalue()) + return clips + + +# --- real data --------------------------------------------------------------- + + +def built_datasets(root: pathlib.Path) -> list[tuple[str, pathlib.Path]]: + """The built ASR datasets under a datasets root: (eval_id, the latest + version's directory), refusing a root that is missing, holds none, or holds + an ``asr_*`` directory that is no catalog eval or has no built version.""" + from pipette_scores.dataset_catalog import asr_datasets + from pipette_scores.types import EvalId + + if not root.is_dir(): + raise ValueError(f"PIPETTE_SCORES_DATA_DIR: {root} is not a directory") + found = [] + for eval_dir in sorted(root.glob("asr_*")): + try: + asr_datasets.ASR_DATASETS[EvalId(eval_dir.name)] + except (ValueError, KeyError): + raise ValueError(f"PIPETTE_SCORES_DATA_DIR: {eval_dir} is no ASR eval of the catalog") from None + versions = sorted(v for v in eval_dir.iterdir() if (v / "metadata.json").is_file()) + if not versions: + raise ValueError(f"PIPETTE_SCORES_DATA_DIR: {eval_dir} holds no built version (no metadata.json)") + found.append((eval_dir.name, versions[-1])) + if not found: + raise ValueError(f"PIPETTE_SCORES_DATA_DIR: {root} holds no built ASR dataset (asr_*//)") + return found + + +def data_roots(value: str | None) -> list[pathlib.Path]: + """``PIPETTE_SCORES_DATA_DIR``'s roots (several joined with os.pathsep), + each a data dir holding ``datasets/`` or a datasets root. Unset or empty: + none. Set: every root must hold at least one built ASR dataset + (`built_datasets`), so that a mistyped path fails instead of checking + nothing.""" + roots = [pathlib.Path(v) for v in (value or "").split(os.pathsep) if v] + roots = [r / "datasets" if (r / "datasets").is_dir() else r for r in roots] + for root in roots: + built_datasets(root) + return roots + + +def real_corpus(data_dir: pathlib.Path, clips_per_dataset: int = 24) -> dict[str, Any]: + """Every reference of each built dataset under ``data_dir`` (its latest + version), perturbed into hypotheses, and a few clips of each, all named + ``:[:]``.""" + import pyarrow.parquet as pq + + from pipette_scores.dataset_catalog import _asr_common, asr_datasets + from pipette_scores.types import EvalId + + english: dict[str, list[dict[str, Any]]] = {} + fleurs: dict[str, dict[str, Any]] = {} + clips: dict[str, bytes] = {} + rng = random.Random(4) + for eval_id, version in built_datasets(data_dir): + dataset = asr_datasets.ASR_DATASETS[EvalId(eval_id)] + name = f"{data_dir}:{eval_id}" + files = _asr_common.parquet_files(version) + columns = ["reference", "session", "chunk_index"] + rows = [r for f in files for r in pq.read_table(f, columns=columns).to_pylist()] + if not rows: + raise ValueError(f"{version}: no rows") + if dataset.family == "open_asr_leaderboard": + manifest = [] + sessions: dict[str, list[dict[str, Any]]] = {} + for row in rows: + if row["session"] is None: + hypothesis = row["reference"] + for _ in range(rng.randrange(0, 3)): + hypothesis = _perturb(hypothesis, rng) + manifest.append({"text": row["reference"], "pred_text": hypothesis}) + else: + sessions.setdefault(row["session"], []).append(row) + for session, chunks in sessions.items(): + answer = chunks[0]["reference"] + for _ in range(8): + answer = _perturb(answer, rng) + words = answer.split() + size = -(-len(words) // len(chunks)) + for chunk in chunks: + k = chunk["chunk_index"] + piece = " ".join(words[k * size : (k + 1) * size]) + manifest.append( + {"text": chunk["reference"], "pred_text": piece, "parent_id": session, "chunk_index": k} + ) + english[name] = manifest + else: + pairs = [] + for row in rows: + hypothesis = row["reference"] + for _ in range(rng.randrange(0, 3)): + hypothesis = _perturb(hypothesis, rng) + pairs.append((row["reference"], hypothesis)) + fleurs[name] = { + "eval_id": eval_id, + "config": fleurs_config(eval_id), + "language": dataset.language, + "pairs": pairs, + } + audio = pq.ParquetFile(files[0]).read_row_group(0, columns=["audio"]).column("audio").to_pylist() + clips.update({f"{name}:{k}": clip for k, clip in enumerate(audio[:clips_per_dataset])}) + return {"open_asr": english, "fleurs": fleurs, "clips": clips} + + +def fleurs_transcripts(seed: int = 5) -> dict[str, dict[str, Any]]: + """Every test-split transcript of each FLEURS locale evaluated, from the + pinned google/fleurs revision's ``data//test.tsv``: its column 3, + ``transcription`` (the reference), named ``fleurs_tsv:``, and its + column 2, ``raw_transcription`` (cased, punctuated, with its parentheses), + named ``fleurs_tsv_raw:``; each perturbed into hypotheses (numbers + among them written the other way).""" + from huggingface_hub import hf_hub_download + + from pipette_scores.dataset_catalog import asr_datasets + + rng = random.Random(seed) + datasets = {} + for locale, language, expected in asr_datasets.FLEURS_LOCALES: + dataset = asr_datasets.ASR_DATASETS[asr_datasets.EvalId(f"asr_fleurs_{locale}")] + assert dataset.reference_column == "transcription", dataset.eval_id + path = hf_hub_download( + asr_datasets.FLEURS_REPO, f"data/{locale}/test.tsv", repo_type="dataset", revision=dataset.revision + ) + rows = [line.split("\t") for line in pathlib.Path(path).read_text(encoding="utf-8").splitlines() if line] + assert len(rows) == expected, (locale, len(rows)) + for prefix, column in (("fleurs_tsv", 3), ("fleurs_tsv_raw", 2)): + pairs = [] + for row in rows: + hypothesis = _spell_numbers(row[column], language, rng) + for _ in range(rng.randrange(0, 3)): + hypothesis = _perturb(hypothesis, rng) + pairs.append((row[column], hypothesis)) + datasets[f"{prefix}:{dataset.eval_id.value}"] = { + "eval_id": dataset.eval_id.value, + "config": fleurs_config(dataset.eval_id.value), + "language": language, + "pairs": pairs, + } + return datasets + + +def synthetic_inputs() -> dict[str, dict[str, Any]]: + """The synthetic part, named: open_asr ``synthetic`` (the English pairs) and + ``synthetic_calls``, fleurs ``synthetic:``, clips ``synthetic:``.""" + return { + "open_asr": { + "synthetic": [{"text": r, "pred_text": h} for r, h in synthetic_english()], + "synthetic_calls": synthetic_calls(), + }, + "fleurs": {f"synthetic:{d['eval_id']}": d for d in synthetic_fleurs()}, + "clips": {f"synthetic:{k}": clip for k, clip in enumerate(synthetic_clips())}, + } + + +# --- the reference call ------------------------------------------------------ + + +def reference_env() -> pathlib.Path | None: + value = os.environ.get("PIPETTE_ASR_REFERENCE_ENV") + return pathlib.Path(value) if value else None + + +def by_name(names: list[str], answers: list[dict[str, Any]], kind: str) -> dict[str, dict[str, Any]]: + """The reference's answers keyed by the names they echo, which must be + exactly ``names``, in order: one answer per input, none lost or added.""" + if len(answers) != len(names): + raise AssertionError(f"{kind}: {len(answers)} answers for {len(names)} inputs") + echoed = [a.get("name") for a in answers] + if echoed != names: + raise AssertionError(f"{kind}: the answers are for {echoed}, not {names}") + return {a["name"]: a for a in answers} + + +def run_reference( + env: pathlib.Path, + open_asr: dict[str, list[dict[str, Any]]], + fleurs: dict[str, dict[str, Any]], + clips: dict[str, bytes], +) -> dict[str, Any]: + """The reference's answers, each keyed by its input's name.""" + request = { + "open_asr": [{"name": name, "rows": rows} for name, rows in open_asr.items()], + "fleurs": [{"name": name, **{k: d[k] for k in ("config", "language", "pairs")}} for name, d in fleurs.items()], + "audio": [{"name": name, "bytes": base64.b64encode(c).decode("ascii")} for name, c in clips.items()], + } + with tempfile.TemporaryDirectory() as tmp: + request_path, answer_path = pathlib.Path(tmp) / "request.json", pathlib.Path(tmp) / "answer.json" + request_path.write_text(json.dumps(request, ensure_ascii=False), encoding="utf-8") + subprocess.run( + [str(env / "venv" / "bin" / "python"), str(RUNNER), str(env), str(request_path), str(answer_path)], + check=True, + stdout=subprocess.DEVNULL, + ) + answer = json.loads(answer_path.read_text(encoding="utf-8")) + return { + "versions": answer["versions"], + "open_asr": by_name(list(open_asr), answer["open_asr"], "open_asr"), + "fleurs": by_name(list(fleurs), answer["fleurs"], "fleurs"), + "audio": by_name(list(clips), answer["audio"], "audio"), + } + + +def reference_array(answer: dict[str, Any]) -> Any: + """The samples of one audio answer, as the reference decoded them.""" + import numpy as np + + return np.frombuffer(base64.b64decode(answer["data"]), dtype=answer["dtype"]).reshape(answer["shape"]) + + +def audio_summary(array: Any) -> dict[str, Any]: + """What the fixture records of a decoded clip: its dtype, its shape and four + fixed +-1 projections of its samples (each correctly rounded, math.fsum), so + that equal samples give equal numbers on any platform, and a resampler that + is off anywhere in the clip moves them far past floating-point rounding.""" + import numpy as np + + weights = np.random.default_rng(6).integers(0, 2, size=(4, array.shape[-1])) * 2 - 1 + samples = np.asarray(array, dtype=np.float64) + return { + "dtype": str(array.dtype), + "shape": list(array.shape), + "projections": [math.fsum(row * samples) for row in weights.astype(np.float64)], + } + + +def digest(texts: list[str]) -> str: + return hashlib.sha256("\x1e".join(texts).encode("utf-8")).hexdigest() + + +def synthetic_fixture(answer: dict[str, Any]) -> dict[str, Any]: + """What fixtures/asr_public_reference.json records of the reference's + answers (keyed by name) on the synthetic corpus.""" + inputs = synthetic_inputs() + english, calls = answer["open_asr"]["synthetic"], answer["open_asr"]["synthetic_calls"] + return { + "note": ( + "Answers of the public reference implementations on the synthetic corpus of " + "tests/asr_public_reference.py; regenerate with `python tests/asr_public_reference.py --write-fixture`." + ), + "corpus_sha256": corpus_digest(), + "open_asr_english": {k: english[k] for k in ("kept", "units", "totals", "wer")}, + "open_asr_calls": {k: calls[k] for k in ("kept", "units", "totals", "wer")}, + "fleurs": { + d["eval_id"]: { + "totals": answer["fleurs"][name]["totals"], + "references_sha256": digest(answer["fleurs"][name]["references"]), + "hypotheses_sha256": digest(answer["fleurs"][name]["hypotheses"]), + } + for name, d in inputs["fleurs"].items() + }, + "audio": {name: audio_summary(reference_array(answer["audio"][name])) for name in inputs["clips"]}, + } + + +def corpus_digest() -> str: + english = synthetic_english() + calls = synthetic_calls() + fleurs = synthetic_fleurs() + return hashlib.sha256(json.dumps([english, calls, fleurs], ensure_ascii=False).encode("utf-8")).hexdigest() + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--write-fixture", action="store_true") + args = parser.parse_args() + env = reference_env() + if env is None: + raise SystemExit("set PIPETTE_ASR_REFERENCE_ENV (scripts/setup_asr_reference_env.sh)") + inputs = synthetic_inputs() + answer = run_reference(env, inputs["open_asr"], inputs["fleurs"], inputs["clips"]) + fixture = synthetic_fixture(answer) + if args.write_fixture: + FIXTURE.write_text(json.dumps(fixture, indent=1, ensure_ascii=False) + "\n", encoding="utf-8") + print(f"wrote {FIXTURE}") + else: + json.dump(fixture, sys.stdout, indent=1, ensure_ascii=False) + + +if __name__ == "__main__": + main() diff --git a/packages/pipette-scores/tests/asr_public_reference_runner.py b/packages/pipette-scores/tests/asr_public_reference_runner.py new file mode 100644 index 0000000..5ee8b59 --- /dev/null +++ b/packages/pipette-scores/tests/asr_public_reference_runner.py @@ -0,0 +1,128 @@ +"""Answer a corpus with the PUBLIC reference implementations (run by the reference interpreter). + +Not a test module: tests/asr_public_reference.py runs it with the Python of the +environment scripts/setup_asr_reference_env.sh builds, as + + /venv/bin/python asr_public_reference_runner.py + +Each input carries a ``name``, which its answer echoes (the caller pairs them +by name). Everything computed here is the reference code's own, unchanged: + +- ``open_asr``: huggingface/open_asr_leaderboard ``normalizer/data_utils`` + (``normalizer``, ``is_target_text_in_range``: the ``prepare_data`` filter) and + ``normalizer/eval_utils`` (``merge_chunked_manifest``, ``align_manifests``, + ``score_manifests``, ``_align_pair``), English, ``num_workers=1``; +- ``fleurs``: the same leaderboard code as its multilingual scoring runs it + (``score_results(multilingual=True, language=)``): ``eval_utils._normalize`` + (its English normalizer for "en", else ``data_utils.ml_normalizer`` with the + language), ``_align_pair`` (``normalize_compound_pairs``, then kaldialign) and + ``align_manifests``; the language code read from the leaderboard config name + (``fleurs_``) as its ``run_eval_ml.py`` reads it; +- ``audio``: ``datasets.Audio(sampling_rate=16000).decode_example`` of datasets + 2.19.0 (soundfile, librosa). +""" + +from __future__ import annotations + +import base64 +import json +import pathlib +import re +import sys + + +def _open_asr(manifests: list[dict]) -> list[dict]: + from normalizer import data_utils, eval_utils + + answers = [] + for named in manifests: + manifest = named["rows"] + kept = [] + for row in manifest: + # prepare_data: norm_text = normalizer(text), filtered by is_target_text_in_range. + if not data_utils.is_target_text_in_range(data_utils.normalizer(row["text"])): + continue + kept.append({"duration": None, "time": None, **row}) + merged = eval_utils.merge_chunked_manifest(kept) + [totals] = eval_utils.align_manifests([merged], language="en", num_workers=1) + [score] = eval_utils.score_manifests([kept], language="en", num_workers=1) + pairs = [eval_utils._align_pair((d["text"], d["pred_text"], "en", False)) for d in merged] + answers.append( + { + "name": named["name"], + "kept": len(kept), + "units": len(merged), + "totals": {k: totals[k] for k in ("ins", "del", "sub", "total", "ref_len", "err_rate")}, + "wer": score["wer"], + "pairs": [[p["ins"], p["del"], p["sub"], p["ref_len"]] for p in pairs], + } + ) + return answers + + +def _fleurs(datasets: list[dict]) -> list[dict]: + from normalizer import eval_utils + + answers = [] + for dataset in datasets: + # run_eval_ml.py: the language from the config name (fleurs_de -> de). + match = re.search(r"_([a-z]{2})(?:_test)?$", dataset["config"]) + if match is None: + raise SystemExit(f"no language code in config {dataset['config']!r}") + language = match.group(1) + manifest = [{"text": r, "pred_text": h} for r, h in dataset["pairs"]] + [totals] = eval_utils.align_manifests([manifest], language=language, multilingual=True, num_workers=1) + pairs = [eval_utils._align_pair((r, h, language, True)) for r, h in dataset["pairs"]] + answers.append( + { + "name": dataset["name"], + "language": language, + "references": [eval_utils._normalize(r, language) for r, _ in dataset["pairs"]], + "hypotheses": [eval_utils._normalize(h, language) for _, h in dataset["pairs"]], + "totals": {k: totals[k] for k in ("ins", "del", "sub", "total", "ref_len", "err_rate")}, + "pairs": [[p["ins"], p["del"], p["sub"], p["ref_len"]] for p in pairs], + } + ) + return answers + + +def _audio(clips: list[dict]) -> list[dict]: + from datasets import Audio + + feature = Audio(sampling_rate=16000) + answers = [] + for clip in clips: + decoded = feature.decode_example({"path": None, "bytes": base64.b64decode(clip["bytes"])}) + array = decoded["array"] + answers.append( + { + "name": clip["name"], + "sampling_rate": decoded["sampling_rate"], + "dtype": str(array.dtype), + "shape": list(array.shape), + "data": base64.b64encode(array.tobytes()).decode("ascii"), + } + ) + return answers + + +def main() -> None: + env, request_path, answer_path = (pathlib.Path(a) for a in sys.argv[1:4]) + sys.path.insert(0, str(env / "open_asr_leaderboard")) + request = json.loads(request_path.read_text(encoding="utf-8")) + import datasets + import kaldialign # noqa: F401 + import num2words # noqa: F401 + from importlib.metadata import version + + answer = { + "versions": {"datasets": datasets.__version__, "num2words": version("num2words")}, + "open_asr": _open_asr(request.get("open_asr", [])), + "fleurs": _fleurs(request.get("fleurs", [])), + "audio": _audio(request.get("audio", [])), + } + answer_path.write_text(json.dumps(answer, ensure_ascii=False), encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/packages/pipette-scores/tests/fixtures/asr_public_reference.json b/packages/pipette-scores/tests/fixtures/asr_public_reference.json new file mode 100644 index 0000000..8345d0b --- /dev/null +++ b/packages/pipette-scores/tests/fixtures/asr_public_reference.json @@ -0,0 +1,274 @@ +{ + "note": "Answers of the public reference implementations on the synthetic corpus of tests/asr_public_reference.py; regenerate with `python tests/asr_public_reference.py --write-fixture`.", + "corpus_sha256": "4ed9ae14192ac2194f3aa9681f65d3dc2a386b9335931545b741e5786655dab9", + "open_asr_english": { + "kept": 1116, + "units": 1116, + "totals": { + "ins": 1708, + "del": 3674, + "sub": 520, + "total": 5902, + "ref_len": 21530, + "err_rate": 0.27412912215513235 + }, + "wer": 27.41 + }, + "open_asr_calls": { + "kept": 28, + "units": 6, + "totals": { + "ins": 37, + "del": 79, + "sub": 6, + "total": 122, + "ref_len": 668, + "err_rate": 0.18263473053892215 + }, + "wer": 18.26 + }, + "fleurs": { + "asr_fleurs_en_us": { + "totals": { + "ins": 427, + "del": 1228, + "sub": 183, + "total": 1838, + "ref_len": 8158, + "err_rate": 0.22530031870556508 + }, + "references_sha256": "2bcbeaed6769641dc888746a47aac1f220eeaa6c122419024f2ecc5b29c15649", + "hypotheses_sha256": "3f70739372b44e9658dfebca9db37283e514a4e14a660cd5aecbf102b434ccc3" + }, + "asr_fleurs_es_419": { + "totals": { + "ins": 130, + "del": 327, + "sub": 19, + "total": 476, + "ref_len": 1668, + "err_rate": 0.2853717026378897 + }, + "references_sha256": "0c804f87e924c52ee1fd48b3b475a65b722749e0abac82c57974715fce661e97", + "hypotheses_sha256": "52cdaa81abd273ad14dd3d0ef8bc83c8976a0ea47c5e7a883cbcc637293088e9" + }, + "asr_fleurs_pt_br": { + "totals": { + "ins": 142, + "del": 417, + "sub": 37, + "total": 596, + "ref_len": 2248, + "err_rate": 0.26512455516014233 + }, + "references_sha256": "c29a08def9eb45cf6d460aba54cc0a3d393415c13ad620685138c6c4b0cb0e80", + "hypotheses_sha256": "9dd0e3200af83bd6d15749f2789d6afc508c5ee352cdc41039d19968d03722e2" + }, + "asr_fleurs_de_de": { + "totals": { + "ins": 101, + "del": 277, + "sub": 9, + "total": 387, + "ref_len": 1432, + "err_rate": 0.2702513966480447 + }, + "references_sha256": "4dc7274cb4ab0fbc25929717f76b2788123f1a6987ea7f55ccfaaf4ea4549855", + "hypotheses_sha256": "2a3de7e3fa5be82d81bbffbae0c1d0a8ee796eba74f1cf86efbfd98dcca8cf2b" + }, + "asr_fleurs_fr_fr": { + "totals": { + "ins": 216, + "del": 376, + "sub": 64, + "total": 656, + "ref_len": 1975, + "err_rate": 0.3321518987341772 + }, + "references_sha256": "286931fb35d17925889d1a2bd65f71183215eaac619c786aecede829823acac2", + "hypotheses_sha256": "befde7102c0834519d26c0963df363e013cb50d7a7c249db44aa35d9290f2549" + }, + "asr_fleurs_it_it": { + "totals": { + "ins": 150, + "del": 83, + "sub": 17, + "total": 250, + "ref_len": 1378, + "err_rate": 0.18142235123367198 + }, + "references_sha256": "e3f65ca8a9d9e0a4ae08b7691ace144b5ffa6bec75871337899a432e7a0425b4", + "hypotheses_sha256": "779824def44ffcc57d7f5bf6542bd83916b1d5fb1016a60ab2c7c2c4e6a8afa4" + }, + "asr_fleurs_nl_nl": { + "totals": { + "ins": 167, + "del": 148, + "sub": 29, + "total": 344, + "ref_len": 1556, + "err_rate": 0.2210796915167095 + }, + "references_sha256": "e89dd142482956e0b33c52e627708fd56895fd2b7be8c2016f0c33a4b39e21d3", + "hypotheses_sha256": "03ac5373128321a26b2a9f464216479abd13d998a5939a31e450e0acfd11c9e2" + } + }, + "audio": { + "synthetic:0": { + "dtype": "float64", + "shape": [ + 16000 + ], + "projections": [ + 9.31597900390625, + 38.38427734375, + 3.419921875, + -31.8055419921875 + ] + }, + "synthetic:1": { + "dtype": "float64", + "shape": [ + 11200 + ], + "projections": [ + 10.7384033203125, + 18.618316650390625, + 10.40509033203125, + 2.439971923828125 + ] + }, + "synthetic:2": { + "dtype": "float64", + "shape": [ + 32000 + ], + "projections": [ + 42.33514472353272, + -28.475582922052126, + -23.47223713324638, + -4.708073227840941 + ] + }, + "synthetic:3": { + "dtype": "float64", + "shape": [ + 8000 + ], + "projections": [ + 6.849297082284465, + 0.26745599252171814, + 14.89071828336455, + 14.703600517008454 + ] + }, + "synthetic:4": { + "dtype": "float64", + "shape": [ + 8001 + ], + "projections": [ + 7.431317881273571, + 3.724427258304786, + 20.48470800247742, + 13.133156515366863 + ] + }, + "synthetic:5": { + "dtype": "float64", + "shape": [ + 20800 + ], + "projections": [ + 23.709535166337446, + 8.775410203852516, + -7.841983656333468, + -2.9995739548976417 + ] + }, + "synthetic:6": { + "dtype": "float64", + "shape": [ + 14400 + ], + "projections": [ + 6.0521195631008595, + 29.99399705673568, + -1.5129033240955323, + -5.974555250955746 + ] + }, + "synthetic:7": { + "dtype": "float64", + "shape": [ + 6400 + ], + "projections": [ + 10.723752788384445, + -0.7437152013881132, + 12.310199774918146, + 0.36512000404763967 + ] + }, + "synthetic:8": { + "dtype": "float64", + "shape": [ + 17600 + ], + "projections": [ + 27.201634862663923, + 25.561785096448148, + -2.4057043149659876, + -22.83474291258608 + ] + }, + "synthetic:9": { + "dtype": "float64", + "shape": [ + 9600 + ], + "projections": [ + 14.287457935512066, + 16.104517214000225, + 28.006347393617034, + -9.179147629067302 + ] + }, + "synthetic:10": { + "dtype": "float64", + "shape": [ + 5280 + ], + "projections": [ + 5.262140964157879, + -2.915385398082435, + 0.893868244253099, + -22.660271869972348 + ] + }, + "synthetic:11": { + "dtype": "float64", + "shape": [ + 27200 + ], + "projections": [ + 43.310360291274264, + -6.386169803561643, + -11.929985500639305, + -28.81510943523608 + ] + }, + "synthetic:12": { + "dtype": "float64", + "shape": [ + 12800 + ], + "projections": [ + 12.030276922741905, + 18.970606460003182, + 6.993227287428454, + -6.154400517931208 + ] + } + } +} diff --git a/packages/pipette-scores/tests/fixtures/math_500_liquid_evals_parity.json b/packages/pipette-scores/tests/fixtures/math_500_liquid_evals_parity.json new file mode 100644 index 0000000..1f2c3f1 --- /dev/null +++ b/packages/pipette-scores/tests/fixtures/math_500_liquid_evals_parity.json @@ -0,0 +1,988 @@ +{ + "source": { + "repo": "Liquid4All/liquid_evals", + "commit": "d5534cfa17713b9fb2d1fe6626763d411380e9a8", + "path": "liquid_evals/parser.py", + "blob": "cd04c90839e366eb51d1071166d3727dcdcf0c99" + }, + "extract": [ + { + "text": "The answer is \\boxed{42}.", + "expected": "42" + }, + { + "text": "\\boxed{\\frac{1}{2}}", + "expected": "\\frac{1}{2}" + }, + { + "text": "\\boxed{\\frac{\\sqrt{3}}{2}}", + "expected": "\\frac{\\sqrt{3}}{2}" + }, + { + "text": "first \\boxed{3}, then corrected: \\boxed{5}", + "expected": "5" + }, + { + "text": "\\boxed{\\boxed{7}}", + "expected": "\\boxed{7}" + }, + { + "text": "\\boxed{\\left( 3, \\frac{\\pi}{2} \\right)}", + "expected": "\\left( 3, \\frac{\\pi}{2} \\right)" + }, + { + "text": "\\boxed{x \\in [-2, 7]}", + "expected": "x \\in [-2, 7]" + }, + { + "text": "unbalanced \\boxed{3 so the result is 4", + "expected": "4" + }, + { + "text": "\\boxed{}", + "expected": "" + }, + { + "text": "empty box \\boxed{} then 12 apples", + "expected": "12" + }, + { + "text": "\\boxed {9} has a space", + "expected": "9" + }, + { + "text": "\\fbox{5} is not a boxed pattern", + "expected": "5" + }, + { + "text": "\\boxed{5\\text{ cm}}", + "expected": "5\\text{ cm}" + }, + { + "text": "\\boxed{90^\\circ}", + "expected": "90^\\circ" + }, + { + "text": "$\\boxed{\\$18.90}$", + "expected": "\\$18.90" + }, + { + "text": "Work.\n#### 18", + "expected": "18" + }, + { + "text": "Work.\n### Final Answer\n5", + "expected": "Final Answer" + }, + { + "text": "### 6\nmore text\n### 7", + "expected": "7" + }, + { + "text": "ANSWER: 7", + "expected": "7" + }, + { + "text": "answer: x = 3\nmore", + "expected": "x = 3" + }, + { + "text": "The answer is [3, 4]", + "expected": "3, 4" + }, + { + "text": "the answer is {x+1}", + "expected": "x+1" + }, + { + "text": "So the answer is (B).", + "expected": "B" + }, + { + "text": "Hence the answer is 12. Done", + "expected": "12" + }, + { + "text": "The final answer is 9", + "expected": "9" + }, + { + "text": "Final answer: 2/3", + "expected": "2/3" + }, + { + "text": "Answer: -4", + "expected": "-4" + }, + { + "text": "We get 3.5 apples\n\n", + "expected": "3.5" + }, + { + "text": "The total is 1,000 dollars", + "expected": "000" + }, + { + "text": "no numbers here", + "expected": "" + }, + { + "text": "", + "expected": "" + }, + { + "text": "\\boxed{1}\\boxed{2}", + "expected": "2" + } + ], + "check": [ + { + "given": "42", + "truth": "42", + "mathd_normalize_given": "42", + "normalize_given": "42", + "mathd": true, + "sympy": true, + "check": true + }, + { + "given": "\\frac{1}{2}", + "truth": "\\frac12", + "mathd_normalize_given": "\\frac{1}{2}", + "normalize_given": "1/2", + "mathd": true, + "sympy": true, + "check": true + }, + { + "given": "0.5", + "truth": "\\frac{1}{2}", + "mathd_normalize_given": "\\frac{1}{2}", + "normalize_given": "0.5", + "mathd": true, + "sympy": true, + "check": true + }, + { + "given": "1/2", + "truth": "\\frac{1}{2}", + "mathd_normalize_given": "\\frac{1}{2}", + "normalize_given": "1/2", + "mathd": true, + "sympy": true, + "check": true + }, + { + "given": "2/4", + "truth": "\\frac{1}{2}", + "mathd_normalize_given": "\\frac{2}{4}", + "normalize_given": "2/4", + "mathd": false, + "sympy": false, + "check": false + }, + { + "given": "\\dfrac{3}{4}", + "truth": "\\frac34", + "mathd_normalize_given": "\\frac{3}{4}", + "normalize_given": "3/4", + "mathd": true, + "sympy": true, + "check": true + }, + { + "given": "\\tfrac{3}{4}", + "truth": "0.75", + "mathd_normalize_given": "\\frac{3}{4}", + "normalize_given": "3/4", + "mathd": false, + "sympy": true, + "check": true + }, + { + "given": "10\\%", + "truth": "10", + "mathd_normalize_given": "10", + "normalize_given": "10", + "mathd": true, + "sympy": true, + "check": true + }, + { + "given": "10%", + "truth": "10", + "mathd_normalize_given": "10", + "normalize_given": "10", + "mathd": true, + "sympy": true, + "check": true + }, + { + "given": "5 \\text{ cm}", + "truth": "5", + "mathd_normalize_given": "5", + "normalize_given": "5", + "mathd": true, + "sympy": true, + "check": true + }, + { + "given": "5\\text{ cm}\\text{ m}", + "truth": "5", + "mathd_normalize_given": "5\\text{ cm}\\text{ m}", + "normalize_given": "5m", + "mathd": false, + "sympy": false, + "check": false + }, + { + "given": "5 cm", + "truth": "5", + "mathd_normalize_given": "5cm", + "normalize_given": "5", + "mathd": false, + "sympy": true, + "check": true + }, + { + "given": "12 inches", + "truth": "12", + "mathd_normalize_given": "12inches", + "normalize_given": "12", + "mathd": false, + "sympy": true, + "check": true + }, + { + "given": "3 feet", + "truth": "3", + "mathd_normalize_given": "3feet", + "normalize_given": "3", + "mathd": false, + "sympy": true, + "check": true + }, + { + "given": "90^\\circ", + "truth": "90", + "mathd_normalize_given": "90", + "normalize_given": "90", + "mathd": true, + "sympy": true, + "check": true + }, + { + "given": "90^{\\circ}", + "truth": "90", + "mathd_normalize_given": "90", + "normalize_given": "90^∘", + "mathd": true, + "sympy": false, + "check": true + }, + { + "given": "\\$5", + "truth": "5", + "mathd_normalize_given": "5", + "normalize_given": "5", + "mathd": true, + "sympy": true, + "check": true + }, + { + "given": "$5.00", + "truth": "5", + "mathd_normalize_given": "$5.00", + "normalize_given": "5", + "mathd": false, + "sympy": true, + "check": true + }, + { + "given": "x=3", + "truth": "3", + "mathd_normalize_given": "3", + "normalize_given": "x=3", + "mathd": true, + "sympy": false, + "check": true + }, + { + "given": "k = 3", + "truth": "3", + "mathd_normalize_given": "3", + "normalize_given": "k=3", + "mathd": true, + "sympy": false, + "check": true + }, + { + "given": "y = 2x + 1", + "truth": "2x+1", + "mathd_normalize_given": "2x+1", + "normalize_given": "y=2x+1", + "mathd": true, + "sympy": false, + "check": true + }, + { + "given": "(1,2)", + "truth": "(1, 2)", + "mathd_normalize_given": "(1,2)", + "normalize_given": "(1,2)", + "mathd": true, + "sympy": true, + "check": true + }, + { + "given": "[1,2)", + "truth": "(1,2)", + "mathd_normalize_given": "[1,2)", + "normalize_given": "[1,2)", + "mathd": false, + "sympy": false, + "check": false + }, + { + "given": "(1,2,3)", + "truth": "(1,2)", + "mathd_normalize_given": "(1,2,3)", + "normalize_given": "(1,2,3)", + "mathd": false, + "sympy": false, + "check": false + }, + { + "given": "(-\\infty, 3]", + "truth": "(-\\infty,3]", + "mathd_normalize_given": "(-\\infty,3]", + "normalize_given": "(-inf,3]", + "mathd": true, + "sympy": true, + "check": true + }, + { + "given": "\\left( 3, \\frac{\\pi}{2} \\right)", + "truth": "\\left( 3, \\frac{\\pi}{2} \\right)", + "mathd_normalize_given": "(3,\\frac{\\pi}{2})", + "normalize_given": "(3,pi/2)", + "mathd": true, + "sympy": true, + "check": true + }, + { + "given": "(3, \\pi/2)", + "truth": "\\left( 3, \\frac{\\pi}{2} \\right)", + "mathd_normalize_given": "(3,\\pi/2)", + "normalize_given": "(3,pi/2)", + "mathd": false, + "sympy": true, + "check": true + }, + { + "given": "1,000", + "truth": "1000", + "mathd_normalize_given": "1,000", + "normalize_given": "1000", + "mathd": false, + "sympy": true, + "check": true + }, + { + "given": "1,000,000", + "truth": "10^6", + "mathd_normalize_given": "1,000,000", + "normalize_given": "1000000", + "mathd": false, + "sympy": false, + "check": false + }, + { + "given": "3 million", + "truth": "3000000", + "mathd_normalize_given": "3million", + "normalize_given": "3*10^6", + "mathd": false, + "sympy": false, + "check": false + }, + { + "given": "7 3/4", + "truth": "\\frac{31}{4}", + "mathd_normalize_given": "\\frac{73}{4}", + "normalize_given": "7+3/4", + "mathd": false, + "sympy": true, + "check": true + }, + { + "given": "\\sqrt{2}/2", + "truth": "\\frac{\\sqrt{2}}{2}", + "mathd_normalize_given": "\\sqrt{2}/2", + "normalize_given": "sqrt(2)/2", + "mathd": false, + "sympy": true, + "check": true + }, + { + "given": "2\\sqrt3", + "truth": "2\\sqrt{3}", + "mathd_normalize_given": "2\\sqrt{3}", + "normalize_given": "2sqrt(3)", + "mathd": true, + "sympy": true, + "check": true + }, + { + "given": "2\\sqrt", + "truth": "2\\sqrt{3}", + "mathd_normalize_given": "2\\sqrt", + "normalize_given": "2\\sqrt", + "mathd": false, + "sympy": false, + "check": false + }, + { + "given": "\\frac", + "truth": "\\frac{1}{2}", + "mathd_normalize_given": "\\frac", + "normalize_given": "%s/%s", + "mathd": false, + "sympy": false, + "check": false + }, + { + "given": "\\pi", + "truth": "\\pi", + "mathd_normalize_given": "\\pi", + "normalize_given": "pi", + "mathd": true, + "sympy": true, + "check": true + }, + { + "given": "2\\pi", + "truth": "\\pi*2", + "mathd_normalize_given": "2\\pi", + "normalize_given": "2pi", + "mathd": false, + "sympy": true, + "check": true + }, + { + "given": "\\infty", + "truth": "\\infty", + "mathd_normalize_given": "\\infty", + "normalize_given": "inf", + "mathd": true, + "sympy": true, + "check": true + }, + { + "given": "x^2+2x+1", + "truth": "(x+1)^2", + "mathd_normalize_given": "x^2+2x+1", + "normalize_given": "x^2+2x+1", + "mathd": false, + "sympy": true, + "check": true + }, + { + "given": "x^{2}+2x+1", + "truth": "(x+1)^2", + "mathd_normalize_given": "x^{2}+2x+1", + "normalize_given": "x^2+2x+1", + "mathd": false, + "sympy": true, + "check": true + }, + { + "given": "2^10", + "truth": "1024", + "mathd_normalize_given": "2^10", + "normalize_given": "2^10", + "mathd": false, + "sympy": false, + "check": false + }, + { + "given": "1024", + "truth": "2^{10}", + "mathd_normalize_given": "1024", + "normalize_given": "1024", + "mathd": false, + "sympy": false, + "check": false + }, + { + "given": "abc", + "truth": "abd", + "mathd_normalize_given": "abc", + "normalize_given": "abc", + "mathd": false, + "sympy": false, + "check": false + }, + { + "given": "\\text{Monday}", + "truth": "\\text{Monday}", + "mathd_normalize_given": "Monday", + "normalize_given": "mon", + "mathd": true, + "sympy": true, + "check": true + }, + { + "given": "monday", + "truth": "\\text{Monday}", + "mathd_normalize_given": "monday", + "normalize_given": "mon", + "mathd": false, + "sympy": true, + "check": true + }, + { + "given": "Monday", + "truth": "\\text{(C)}", + "mathd_normalize_given": "Monday", + "normalize_given": "mon", + "mathd": false, + "sympy": false, + "check": false + }, + { + "given": "C", + "truth": "\\text{(C)}", + "mathd_normalize_given": "C", + "normalize_given": "c", + "mathd": false, + "sympy": true, + "check": true + }, + { + "given": "(C)", + "truth": "\\text{(C)}", + "mathd_normalize_given": "(C)", + "normalize_given": "(c)", + "mathd": true, + "sympy": true, + "check": true + }, + { + "given": "1 and 2", + "truth": "1, 2", + "mathd_normalize_given": "1and2", + "normalize_given": "1,2", + "mathd": false, + "sympy": true, + "check": true + }, + { + "given": "1 or 2", + "truth": "1,2", + "mathd_normalize_given": "1or2", + "normalize_given": "1,2", + "mathd": false, + "sympy": true, + "check": true + }, + { + "given": "\\{1,2\\}", + "truth": "1,2", + "mathd_normalize_given": "\\{1,2\\}", + "normalize_given": "1,2", + "mathd": false, + "sympy": true, + "check": true + }, + { + "given": "10", + "truth": "10.0", + "mathd_normalize_given": "10", + "normalize_given": "10", + "mathd": false, + "sympy": true, + "check": true + }, + { + "given": "3.0", + "truth": "3", + "mathd_normalize_given": "3.0", + "normalize_given": "3", + "mathd": false, + "sympy": true, + "check": true + }, + { + "given": "-3", + "truth": "- 3", + "mathd_normalize_given": "-3", + "normalize_given": "-3", + "mathd": true, + "sympy": true, + "check": true + }, + { + "given": ".5", + "truth": "0.5", + "mathd_normalize_given": "\\frac{1}{2}", + "normalize_given": ".5", + "mathd": true, + "sympy": true, + "check": true + }, + { + "given": "\\!1\\!000", + "truth": "1000", + "mathd_normalize_given": "1000", + "normalize_given": "1000", + "mathd": true, + "sympy": true, + "check": true + }, + { + "given": "\\frac{1}{3}", + "truth": "0.333", + "mathd_normalize_given": "\\frac{1}{3}", + "normalize_given": "1/3", + "mathd": false, + "sympy": false, + "check": false + }, + { + "given": "0.333", + "truth": "\\frac{1}{3}", + "mathd_normalize_given": "0.333", + "normalize_given": "0.333", + "mathd": false, + "sympy": false, + "check": false + }, + { + "given": "1e400", + "truth": "1", + "mathd_normalize_given": "1e400", + "normalize_given": "1e400", + "mathd": false, + "sympy": false, + "check": false + }, + { + "given": "nan", + "truth": "nan", + "mathd_normalize_given": "nan", + "normalize_given": "nan", + "mathd": true, + "sympy": true, + "check": true + }, + { + "given": "", + "truth": "5", + "mathd_normalize_given": "", + "normalize_given": "", + "mathd": false, + "sympy": false, + "check": false + }, + { + "given": "5", + "truth": "", + "mathd_normalize_given": "5", + "normalize_given": "5", + "mathd": false, + "sympy": false, + "check": false + }, + { + "given": "The answer is 4", + "truth": "4", + "mathd_normalize_given": "Theansweris4", + "normalize_given": "theansweris4", + "mathd": false, + "sympy": false, + "check": false + }, + { + "given": "4", + "truth": "The answer is 4", + "mathd_normalize_given": "4", + "normalize_given": "4", + "mathd": false, + "sympy": false, + "check": false + }, + { + "given": "x = \\frac{-b}{2a}", + "truth": "-\\frac{b}{2a}", + "mathd_normalize_given": "\\frac{-b}{2a}", + "normalize_given": "x=-b/2a", + "mathd": false, + "sympy": false, + "check": false + }, + { + "given": "x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x", + "truth": "1600x", + "mathd_normalize_given": "x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x", + "normalize_given": "x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x+x", + "mathd": false, + "sympy": false, + "check": false + }, + { + "given": "\\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha \\alpha", + "truth": "1", + "mathd_normalize_given": "\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha", + "normalize_given": "\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha\\alpha", + "mathd": false, + "sympy": false, + "check": false + }, + { + "given": "\\begin{pmatrix} 1 \\\\ 2 \\end{pmatrix}", + "truth": "\\begin{pmatrix} 1 \\\\ 2 \\end{pmatrix}", + "mathd_normalize_given": "\\begin{pmatrix}1\\2\\end{pmatrix}", + "normalize_given": "[1;2]", + "mathd": true, + "sympy": true, + "check": true + } + ], + "verify": [ + { + "raw": "So the answer is \\boxed{42}.", + "gold": "42", + "predicted": "42", + "correct": true + }, + { + "raw": "So the answer is \\boxed{41}.", + "gold": "42", + "predicted": "41", + "correct": false + }, + { + "raw": "\\boxed{\\frac{\\sqrt{3}}{2}}", + "gold": "\\frac{\\sqrt{3}}{2}", + "predicted": "\\frac{\\sqrt{3}}{2}", + "correct": true + }, + { + "raw": "\\boxed{\\dfrac{\\sqrt3}{2}}", + "gold": "\\frac{\\sqrt{3}}{2}", + "predicted": "\\dfrac{\\sqrt3}{2}", + "correct": true + }, + { + "raw": "\\boxed{\\boxed{7}}", + "gold": "7", + "predicted": "\\boxed{7}", + "correct": true + }, + { + "raw": "\\boxed{5\\text{ cm}}", + "gold": "5", + "predicted": "5\\text{ cm}", + "correct": true + }, + { + "raw": "\\boxed{5 \\text{cm}}", + "gold": "5", + "predicted": "5 \\text{cm}", + "correct": true + }, + { + "raw": "\\boxed{90^\\circ}", + "gold": "90^\\circ", + "predicted": "90^\\circ", + "correct": true + }, + { + "raw": "\\boxed{(3, \\frac{\\pi}{2})}", + "gold": "\\left( 3, \\frac{\\pi}{2} \\right)", + "predicted": "(3, \\frac{\\pi}{2})", + "correct": true + }, + { + "raw": "\\boxed{\\text{(C)}}", + "gold": "\\text{(C)}", + "predicted": "\\text{(C)}", + "correct": true + }, + { + "raw": "\\boxed{C}", + "gold": "\\text{(C)}", + "predicted": "C", + "correct": true + }, + { + "raw": "The answer is 7", + "gold": "7", + "predicted": "7", + "correct": true + }, + { + "raw": "Therefore, the final answer is 12", + "gold": "12", + "predicted": "12", + "correct": true + }, + { + "raw": "Final answer: 5", + "gold": "5", + "predicted": "5", + "correct": true + }, + { + "raw": "We compute.\nFinal answer: 5", + "gold": "5", + "predicted": "5", + "correct": true + }, + { + "raw": "We compute 3 + 4.\nIt is 7 apples", + "gold": "7", + "predicted": "7", + "correct": true + }, + { + "raw": "no answer here at all", + "gold": "7", + "predicted": "", + "correct": false + }, + { + "raw": "", + "gold": "7", + "predicted": "", + "correct": false + }, + { + "raw": "Hmm.\nMonday", + "gold": "\\text{Monday}", + "predicted": "", + "correct": true + }, + { + "raw": "\\boxed{3}\nThe answer is 4", + "gold": "4", + "predicted": "3", + "correct": false + }, + { + "raw": "\\boxed{3}\n4", + "gold": "4", + "predicted": "3", + "correct": true + }, + { + "raw": "4\nstep 0 of the work\nstep 1 of the work\nstep 2 of the work\nstep 3 of the work\nstep 4 of the work\nstep 5 of the work\nstep 6 of the work\nstep 7 of the work\nstep 8 of the work\nstep 9 of the work\nstep 10 of the work\nstep 11 of the work\nstep 12 of the work\nstep 13 of the work\nstep 14 of the work\nstep 15 of the work\nstep 16 of the work\nstep 17 of the work\nstep 18 of the work\nstep 19 of the work\nstep 20 of the work\nstep 21 of the work\nstep 22 of the work\nstep 23 of the work\nstep 24 of the work\nstep 25 of the work\nstep 26 of the work\nstep 27 of the work\nstep 28 of the work\nstep 29 of the work\nstep 30 of the work\nstep 31 of the work\nstep 32 of the work\nstep 33 of the work\nstep 34 of the work\nstep 35 of the work\nstep 36 of the work\nstep 37 of the work\nstep 38 of the work\nstep 39 of the work\nstep 40 of the work\nstep 41 of the work\nstep 42 of the work\nstep 43 of the work\nstep 44 of the work\nstep 45 of the work\nstep 46 of the work\nstep 47 of the work\nstep 48 of the work\nstep 49 of the work\nstep 50 of the work\nstep 51 of the work\nstep 52 of the work\nstep 53 of the work\nstep 54 of the work\nstep 55 of the work\nstep 56 of the work\nstep 57 of the work\nstep 58 of the work\nstep 59 of the work\nstep 60 of the work\nstep 61 of the work\nstep 62 of the work\nstep 63 of the work\nstep 64 of the work\nstep 65 of the work\nstep 66 of the work\nstep 67 of the work\nstep 68 of the work\nstep 69 of the work\nstep 70 of the work\nstep 71 of the work\nstep 72 of the work\nstep 73 of the work\nstep 74 of the work\nstep 75 of the work\nstep 76 of the work\nstep 77 of the work\nstep 78 of the work\nstep 79 of the work\nstep 80 of the work\nstep 81 of the work\nstep 82 of the work\nstep 83 of the work\nstep 84 of the work\nstep 85 of the work\nstep 86 of the work\nstep 87 of the work\nstep 88 of the work\nstep 89 of the work\nstep 90 of the work\nstep 91 of the work\nstep 92 of the work\nstep 93 of the work\nstep 94 of the work\nstep 95 of the work\nstep 96 of the work\nstep 97 of the work\nstep 98 of the work\nstep 99 of the work\nstep 100 of the work\nstep 101 of the work\nstep 102 of the work\nstep 103 of the work\nstep 104 of the work\nstep 105 of the work\nstep 106 of the work\nstep 107 of the work\nstep 108 of the work\nstep 109 of the work\nstep 110 of the work\nstep 111 of the work\nstep 112 of the work\nstep 113 of the work\nstep 114 of the work\nstep 115 of the work\nstep 116 of the work\nstep 117 of the work\nstep 118 of the work\nstep 119 of the work\nstep 120 of the work\nstep 121 of the work\nstep 122 of the work\nstep 123 of the work\nstep 124 of the work\nstep 125 of the work\nstep 126 of the work\nstep 127 of the work\nstep 128 of the work\nstep 129 of the work\nstep 130 of the work\nstep 131 of the work\nstep 132 of the work\nstep 133 of the work\nstep 134 of the work\nstep 135 of the work\nstep 136 of the work\nstep 137 of the work\nstep 138 of the work\nstep 139 of the work\nstep 140 of the work\nstep 141 of the work\nstep 142 of the work\nstep 143 of the work\nstep 144 of the work\nstep 145 of the work\nstep 146 of the work\nstep 147 of the work\n\\boxed{3}", + "gold": "4", + "predicted": "3", + "correct": true + }, + { + "raw": "4\nstep 0 of the work\nstep 1 of the work\nstep 2 of the work\nstep 3 of the work\nstep 4 of the work\nstep 5 of the work\nstep 6 of the work\nstep 7 of the work\nstep 8 of the work\nstep 9 of the work\nstep 10 of the work\nstep 11 of the work\nstep 12 of the work\nstep 13 of the work\nstep 14 of the work\nstep 15 of the work\nstep 16 of the work\nstep 17 of the work\nstep 18 of the work\nstep 19 of the work\nstep 20 of the work\nstep 21 of the work\nstep 22 of the work\nstep 23 of the work\nstep 24 of the work\nstep 25 of the work\nstep 26 of the work\nstep 27 of the work\nstep 28 of the work\nstep 29 of the work\nstep 30 of the work\nstep 31 of the work\nstep 32 of the work\nstep 33 of the work\nstep 34 of the work\nstep 35 of the work\nstep 36 of the work\nstep 37 of the work\nstep 38 of the work\nstep 39 of the work\nstep 40 of the work\nstep 41 of the work\nstep 42 of the work\nstep 43 of the work\nstep 44 of the work\nstep 45 of the work\nstep 46 of the work\nstep 47 of the work\nstep 48 of the work\nstep 49 of the work\nstep 50 of the work\nstep 51 of the work\nstep 52 of the work\nstep 53 of the work\nstep 54 of the work\nstep 55 of the work\nstep 56 of the work\nstep 57 of the work\nstep 58 of the work\nstep 59 of the work\nstep 60 of the work\nstep 61 of the work\nstep 62 of the work\nstep 63 of the work\nstep 64 of the work\nstep 65 of the work\nstep 66 of the work\nstep 67 of the work\nstep 68 of the work\nstep 69 of the work\nstep 70 of the work\nstep 71 of the work\nstep 72 of the work\nstep 73 of the work\nstep 74 of the work\nstep 75 of the work\nstep 76 of the work\nstep 77 of the work\nstep 78 of the work\nstep 79 of the work\nstep 80 of the work\nstep 81 of the work\nstep 82 of the work\nstep 83 of the work\nstep 84 of the work\nstep 85 of the work\nstep 86 of the work\nstep 87 of the work\nstep 88 of the work\nstep 89 of the work\nstep 90 of the work\nstep 91 of the work\nstep 92 of the work\nstep 93 of the work\nstep 94 of the work\nstep 95 of the work\nstep 96 of the work\nstep 97 of the work\nstep 98 of the work\nstep 99 of the work\nstep 100 of the work\nstep 101 of the work\nstep 102 of the work\nstep 103 of the work\nstep 104 of the work\nstep 105 of the work\nstep 106 of the work\nstep 107 of the work\nstep 108 of the work\nstep 109 of the work\nstep 110 of the work\nstep 111 of the work\nstep 112 of the work\nstep 113 of the work\nstep 114 of the work\nstep 115 of the work\nstep 116 of the work\nstep 117 of the work\nstep 118 of the work\nstep 119 of the work\nstep 120 of the work\nstep 121 of the work\nstep 122 of the work\nstep 123 of the work\nstep 124 of the work\nstep 125 of the work\nstep 126 of the work\nstep 127 of the work\nstep 128 of the work\nstep 129 of the work\nstep 130 of the work\nstep 131 of the work\nstep 132 of the work\nstep 133 of the work\nstep 134 of the work\nstep 135 of the work\nstep 136 of the work\nstep 137 of the work\nstep 138 of the work\nstep 139 of the work\nstep 140 of the work\nstep 141 of the work\nstep 142 of the work\nstep 143 of the work\nstep 144 of the work\nstep 145 of the work\nstep 146 of the work\nstep 147 of the work\nstep 148 of the work\n\\boxed{3}", + "gold": "4", + "predicted": "3", + "correct": false + }, + { + "raw": "4\n\nstep 0 of the work\nstep 1 of the work\nstep 2 of the work\nstep 3 of the work\nstep 4 of the work\nstep 5 of the work\nstep 6 of the work\nstep 7 of the work\nstep 8 of the work\nstep 9 of the work\nstep 10 of the work\nstep 11 of the work\nstep 12 of the work\nstep 13 of the work\nstep 14 of the work\nstep 15 of the work\nstep 16 of the work\nstep 17 of the work\nstep 18 of the work\nstep 19 of the work\nstep 20 of the work\nstep 21 of the work\nstep 22 of the work\nstep 23 of the work\nstep 24 of the work\nstep 25 of the work\nstep 26 of the work\nstep 27 of the work\nstep 28 of the work\nstep 29 of the work\nstep 30 of the work\nstep 31 of the work\nstep 32 of the work\nstep 33 of the work\nstep 34 of the work\nstep 35 of the work\nstep 36 of the work\nstep 37 of the work\nstep 38 of the work\nstep 39 of the work\nstep 40 of the work\nstep 41 of the work\nstep 42 of the work\nstep 43 of the work\nstep 44 of the work\nstep 45 of the work\nstep 46 of the work\nstep 47 of the work\nstep 48 of the work\nstep 49 of the work\nstep 50 of the work\nstep 51 of the work\nstep 52 of the work\nstep 53 of the work\nstep 54 of the work\nstep 55 of the work\nstep 56 of the work\nstep 57 of the work\nstep 58 of the work\nstep 59 of the work\nstep 60 of the work\nstep 61 of the work\nstep 62 of the work\nstep 63 of the work\nstep 64 of the work\nstep 65 of the work\nstep 66 of the work\nstep 67 of the work\nstep 68 of the work\nstep 69 of the work\nstep 70 of the work\nstep 71 of the work\nstep 72 of the work\nstep 73 of the work\nstep 74 of the work\nstep 75 of the work\nstep 76 of the work\nstep 77 of the work\nstep 78 of the work\nstep 79 of the work\nstep 80 of the work\nstep 81 of the work\nstep 82 of the work\nstep 83 of the work\nstep 84 of the work\nstep 85 of the work\nstep 86 of the work\nstep 87 of the work\nstep 88 of the work\nstep 89 of the work\nstep 90 of the work\nstep 91 of the work\nstep 92 of the work\nstep 93 of the work\nstep 94 of the work\nstep 95 of the work\nstep 96 of the work\nstep 97 of the work\nstep 98 of the work\nstep 99 of the work\nstep 100 of the work\nstep 101 of the work\nstep 102 of the work\nstep 103 of the work\nstep 104 of the work\nstep 105 of the work\nstep 106 of the work\nstep 107 of the work\nstep 108 of the work\nstep 109 of the work\nstep 110 of the work\nstep 111 of the work\nstep 112 of the work\nstep 113 of the work\nstep 114 of the work\nstep 115 of the work\nstep 116 of the work\nstep 117 of the work\nstep 118 of the work\nstep 119 of the work\nstep 120 of the work\nstep 121 of the work\nstep 122 of the work\nstep 123 of the work\nstep 124 of the work\nstep 125 of the work\nstep 126 of the work\nstep 127 of the work\nstep 128 of the work\nstep 129 of the work\nstep 130 of the work\nstep 131 of the work\nstep 132 of the work\nstep 133 of the work\nstep 134 of the work\nstep 135 of the work\nstep 136 of the work\nstep 137 of the work\nstep 138 of the work\nstep 139 of the work\nstep 140 of the work\nstep 141 of the work\nstep 142 of the work\nstep 143 of the work\nstep 144 of the work\nstep 145 of the work\nstep 146 of the work\nstep 147 of the work\n\\boxed{3}", + "gold": "4", + "predicted": "3", + "correct": true + }, + { + "raw": "4\n\n\nstep 0 of the work\nstep 1 of the work\nstep 2 of the work\nstep 3 of the work\nstep 4 of the work\nstep 5 of the work\nstep 6 of the work\nstep 7 of the work\nstep 8 of the work\nstep 9 of the work\nstep 10 of the work\nstep 11 of the work\nstep 12 of the work\nstep 13 of the work\nstep 14 of the work\nstep 15 of the work\nstep 16 of the work\nstep 17 of the work\nstep 18 of the work\nstep 19 of the work\nstep 20 of the work\nstep 21 of the work\nstep 22 of the work\nstep 23 of the work\nstep 24 of the work\nstep 25 of the work\nstep 26 of the work\nstep 27 of the work\nstep 28 of the work\nstep 29 of the work\nstep 30 of the work\nstep 31 of the work\nstep 32 of the work\nstep 33 of the work\nstep 34 of the work\nstep 35 of the work\nstep 36 of the work\nstep 37 of the work\nstep 38 of the work\nstep 39 of the work\nstep 40 of the work\nstep 41 of the work\nstep 42 of the work\nstep 43 of the work\nstep 44 of the work\nstep 45 of the work\nstep 46 of the work\nstep 47 of the work\nstep 48 of the work\nstep 49 of the work\nstep 50 of the work\nstep 51 of the work\nstep 52 of the work\nstep 53 of the work\nstep 54 of the work\nstep 55 of the work\nstep 56 of the work\nstep 57 of the work\nstep 58 of the work\nstep 59 of the work\nstep 60 of the work\nstep 61 of the work\nstep 62 of the work\nstep 63 of the work\nstep 64 of the work\nstep 65 of the work\nstep 66 of the work\nstep 67 of the work\nstep 68 of the work\nstep 69 of the work\nstep 70 of the work\nstep 71 of the work\nstep 72 of the work\nstep 73 of the work\nstep 74 of the work\nstep 75 of the work\nstep 76 of the work\nstep 77 of the work\nstep 78 of the work\nstep 79 of the work\nstep 80 of the work\nstep 81 of the work\nstep 82 of the work\nstep 83 of the work\nstep 84 of the work\nstep 85 of the work\nstep 86 of the work\nstep 87 of the work\nstep 88 of the work\nstep 89 of the work\nstep 90 of the work\nstep 91 of the work\nstep 92 of the work\nstep 93 of the work\nstep 94 of the work\nstep 95 of the work\nstep 96 of the work\nstep 97 of the work\nstep 98 of the work\nstep 99 of the work\nstep 100 of the work\nstep 101 of the work\nstep 102 of the work\nstep 103 of the work\nstep 104 of the work\nstep 105 of the work\nstep 106 of the work\nstep 107 of the work\nstep 108 of the work\nstep 109 of the work\nstep 110 of the work\nstep 111 of the work\nstep 112 of the work\nstep 113 of the work\nstep 114 of the work\nstep 115 of the work\nstep 116 of the work\nstep 117 of the work\nstep 118 of the work\nstep 119 of the work\nstep 120 of the work\nstep 121 of the work\nstep 122 of the work\nstep 123 of the work\nstep 124 of the work\nstep 125 of the work\nstep 126 of the work\nstep 127 of the work\nstep 128 of the work\nstep 129 of the work\nstep 130 of the work\nstep 131 of the work\nstep 132 of the work\nstep 133 of the work\nstep 134 of the work\nstep 135 of the work\nstep 136 of the work\nstep 137 of the work\nstep 138 of the work\nstep 139 of the work\nstep 140 of the work\nstep 141 of the work\nstep 142 of the work\nstep 143 of the work\nstep 144 of the work\nstep 145 of the work\nstep 146 of the work\nstep 147 of the work\n\\boxed{3}", + "gold": "4", + "predicted": "3", + "correct": true + }, + { + "raw": "maybe \\boxed{7}\nSo \\boxed{42}", + "gold": "42", + "predicted": "42", + "correct": true + }, + { + "raw": "maybe \\boxed{42}\nSo \\boxed{7}", + "gold": "42", + "predicted": "7", + "correct": false + }, + { + "raw": "reasoning \\boxed{42}\nThe answer is \\boxed{42}", + "gold": "42", + "predicted": "42", + "correct": true + }, + { + "raw": "reasoning so far \\boxed{42} wait", + "gold": "42", + "predicted": "", + "correct": false + }, + { + "raw": "Partial work, so \\boxed{42}? Check the sign of", + "gold": "42", + "predicted": "42", + "correct": true + }, + { + "raw": "Start.long reasoning \\boxed{42} that was cut", + "gold": "42", + "predicted": "", + "correct": false + }, + { + "raw": "[THINK]x\\boxed{1}[/THINK]\\boxed{2}", + "gold": "2", + "predicted": "2", + "correct": true + }, + { + "raw": "analysisWe need 2+2.assistantfinal\\boxed{4}", + "gold": "4", + "predicted": "4", + "correct": true + }, + { + "raw": "\\boxed{0.5}", + "gold": "\\frac{1}{2}", + "predicted": "0.5", + "correct": true + }, + { + "raw": "\\boxed{2/4}", + "gold": "\\frac{1}{2}", + "predicted": "2/4", + "correct": false + }, + { + "raw": "\\boxed{x^2 + 2x + 1}", + "gold": "(x+1)^2", + "predicted": "x^2 + 2x + 1", + "correct": true + }, + { + "raw": "\\boxed{1,\\!000}", + "gold": "1000", + "predicted": "1,\\!000", + "correct": true + }, + { + "raw": "\\boxed{\\$18.90}", + "gold": "\\$18.90", + "predicted": "\\$18.90", + "correct": true + }, + { + "raw": "\\boxed{18.9}", + "gold": "\\$18.90", + "predicted": "18.9", + "correct": true + } + ] +} diff --git a/packages/pipette-scores/tests/test_api.py b/packages/pipette-scores/tests/test_api.py index b89a776..68b83f2 100644 --- a/packages/pipette-scores/tests/test_api.py +++ b/packages/pipette-scores/tests/test_api.py @@ -1,13 +1,23 @@ +import hashlib +import json from unittest.mock import patch import pytest from fastapi.testclient import TestClient -from pipette_scores.api.app import _APP_VERSION, app +import asyncio +import threading +import time + +from pipette_scores import dataset_catalog, packages +from pipette_scores.api.app import _APP_VERSION, _DIGESTS, _SERVED, _samples_digest_or_error, app +from pipette_scores.hashing import samples_digest from pipette_scores.types import ( EvalSample, ChatMessage, IFStructSample, + SampleScore, + VlmSample, ) @@ -36,6 +46,17 @@ def client(): return TestClient(app) +@pytest.fixture(autouse=True) +def _fresh_digests(): + # Digests are memoized per (eval, dataset, profile); the tests patch the + # loader under the same keys with different samples. + _DIGESTS.clear() + _SERVED.clear() + yield + _DIGESTS.clear() + _SERVED.clear() + + # --------------------------------------------------------------------------- # GET /health # --------------------------------------------------------------------------- @@ -97,12 +118,276 @@ def test_success(self, client): assert len(body["samples"]) == 1 assert body["samples"][0]["messages"][0]["content"] == "Q?" + def test_prompt_profile_is_passed_to_the_loader(self, client): + prompts = [EvalSample(id="1", messages=[ChatMessage(role="user", content="Q?")])] + with patch("pipette_scores.api.app._load_prompts_cached", return_value=prompts) as load: + resp = client.get("/evals/mmstar/datasets/test/samples?prompt_profile=internvl") + assert resp.status_code == 200 + load.assert_called_once_with("mmstar", "test", "internvl") + + @pytest.mark.parametrize("profile", ["glm_v", "kimi_vl", "nemotron_think", "phi4_rv_think", "mimo_vl"]) + def test_the_new_vlm_prompt_profiles_are_served(self, client, profile): + prompts = [EvalSample(id="1", messages=[ChatMessage(role="user", content="Q?")])] + with patch("pipette_scores.api.app._load_prompts_cached", return_value=prompts) as load: + resp = client.get(f"/evals/mmstar/datasets/test/samples?prompt_profile={profile}") + assert resp.status_code == 200 + load.assert_called_once_with("mmstar", "test", profile) + + def test_unknown_prompt_profile_returns_400(self, client): + resp = client.get("/evals/mmstar/datasets/test/samples?prompt_profile=nope") + assert resp.status_code == 400 + assert "Unknown prompt profile: nope" in resp.json()["detail"] + + def test_prompt_profile_on_a_text_eval_returns_400(self, client): + resp = client.get("/evals/ifstruct/datasets/test/samples?prompt_profile=internvl") + assert resp.status_code == 400 + + +# --------------------------------------------------------------------------- +# GET /evals/{eval_id}/datasets/{dataset_name}/samples/digest +# --------------------------------------------------------------------------- + +VLM_SAMPLES = [ + EvalSample( + id="a1", + messages=[ + ChatMessage( + role="user", + content=[ + {"type": "image_url", "image_url": {"url": "data:image/png;base64,iVBORw0KGgo="}}, + {"type": "text", "text": "Which mood? 心情\nA. Sad\nB. Happy"}, + ], + ) + ], + ), + EvalSample(id="b2", messages=[ChatMessage(role="user", content="Is there a dog?")]), +] + + +class TestSamplesDigest: + def test_the_digest_is_that_of_the_served_body(self, client): + with ( + patch("pipette_scores.api.app._load_prompts_cached", return_value=VLM_SAMPLES), + patch("pipette_scores.api.app._build_prompts", return_value=VLM_SAMPLES), + ): + digest = client.get("/evals/mmstar/datasets/test/samples/digest") + served = client.get("/evals/mmstar/datasets/test/samples") + assert digest.status_code == 200 + # Recomputed from the HTTP body alone, as a client holding a copy could. + lines = "".join( + json.dumps(s, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + "\n" + for s in served.json()["samples"] + ) + expected = f"sha256:{hashlib.sha256(lines.encode()).hexdigest()}" + assert digest.json() == {"digest": expected, "sample_count": 2} + assert served.headers["X-Samples-Digest"] == expected + + def test_the_header_is_hashed_from_the_samples_served(self, client): + # /samples hashes the list it serves; nothing is built a second time. + with ( + patch("pipette_scores.api.app._load_prompts_cached", return_value=VLM_SAMPLES), + patch("pipette_scores.api.app._build_prompts") as build, + ): + served = client.get("/evals/mmstar/datasets/test/samples") + assert served.headers["X-Samples-Digest"] == samples_digest(VLM_SAMPLES) + build.assert_not_called() + + def test_the_digest_route_keeps_no_samples(self, client): + # A probe asks for every pinned set every minute: the route builds the + # samples to hash them and holds on to the digest alone. + with ( + patch("pipette_scores.api.app._load_prompts_cached") as cached, + patch("pipette_scores.api.app._build_prompts", return_value=VLM_SAMPLES) as build, + ): + resp = client.get("/evals/mmstar/datasets/test/samples/digest?prompt_profile=qwen3_5") + assert resp.status_code == 200 + build.assert_called_once_with("mmstar", "test", "qwen3_5") + cached.assert_not_called() + + def test_the_digest_is_computed_once(self, client): + with ( + patch("pipette_scores.api.app._build_prompts", return_value=VLM_SAMPLES) as build, + patch("pipette_scores.api.app.samples_digest", wraps=samples_digest) as digest, + ): + first = client.get("/evals/mmstar/datasets/test/samples/digest").json() + second = client.get("/evals/mmstar/datasets/test/samples/digest").json() + assert first == second + assert build.call_count == 1 + assert digest.call_count == 1 + + def test_concurrent_requests_build_a_cold_set_once(self): + # After a deploy every pinned set is cold, and a cache probes them at + # once: one build per set, and one cold build at a time in the process. + builds, running, peak = [], [0], [0] + guard = threading.Lock() + + def slow_build(eval_id, dataset_name, prompt_profile): + with guard: + builds.append(prompt_profile) + running[0] += 1 + peak[0] = max(peak[0], running[0]) + time.sleep(0.05) + with guard: + running[0] -= 1 + return VLM_SAMPLES + + async def probe_all(): + return await asyncio.gather( + *(_samples_digest_or_error("mmstar", "test", profile) for profile in [None] * 4 + ["qwen3_5"] * 4) + ) + + with patch("pipette_scores.api.app._build_prompts", side_effect=slow_build): + digests = asyncio.run(probe_all()) + assert sorted(builds, key=str) == [None, "qwen3_5"] + assert peak[0] == 1 + assert len({d for d, _ in digests}) == 1 + + def test_a_served_list_is_hashed_once(self, client): + # The cached list /samples serves is the same object every time. + with ( + patch("pipette_scores.api.app._load_prompts_cached", return_value=VLM_SAMPLES), + patch("pipette_scores.api.app.samples_digest", wraps=samples_digest) as digest, + ): + first = client.get("/evals/mmstar/datasets/test/samples") + second = client.get("/evals/mmstar/datasets/test/samples") + assert first.headers["X-Samples-Digest"] == second.headers["X-Samples-Digest"] + assert digest.call_count == 1 + + def test_a_malformed_samples_digest_is_a_bad_request(self, client): + with patch("pipette_scores.api.app._build_prompts") as build: + resp = client.post( + "/score", + json={"eval_id": "mmstar", "dataset_name": "test", "completions": [], "samples_digest": "garbage"}, + ) + assert resp.status_code == 422 + build.assert_not_called() + + def test_the_header_is_always_the_bodys(self, client): + # A digest cached for another body never rides on this one. + _DIGESTS[("mmstar", "test", None)] = ("sha256:" + "0" * 64, 2) + with patch("pipette_scores.api.app._load_prompts_cached", return_value=VLM_SAMPLES): + served = client.get("/evals/mmstar/datasets/test/samples") + digest = client.get("/evals/mmstar/datasets/test/samples/digest") + assert served.headers["X-Samples-Digest"] == samples_digest(VLM_SAMPLES) + assert digest.json()["digest"] == samples_digest(VLM_SAMPLES) + + def test_a_subclasss_own_fields_are_hashed_as_served(self, client): + # The response writes samples through the declared EvalSample, so a + # loader's subclass field never reaches the body, nor the digest. + class Extended(EvalSample): + generation_params: dict = {"temperature": 0.0} + + extended = [Extended(id=s.id, messages=s.messages) for s in VLM_SAMPLES] + with patch("pipette_scores.api.app._load_prompts_cached", return_value=extended): + served = client.get("/evals/mmstar/datasets/test/samples") + assert "generation_params" not in served.text + assert served.headers["X-Samples-Digest"] == samples_digest(VLM_SAMPLES) + + @pytest.mark.parametrize( + ("path", "status"), + [ + ("/evals/nonexistent/datasets/test/samples/digest", 404), + ("/evals/ifstruct/datasets/does-not-exist/samples/digest", 404), + ("/evals/mmstar/datasets/test/samples/digest?prompt_profile=nope", 400), + ("/evals/ifstruct/datasets/test/samples/digest?prompt_profile=internvl", 400), + ], + ) + def test_it_refuses_what_samples_refuses(self, client, path, status): + assert client.get(path).status_code == status + + +class TestSamplesDigestFromThePackagesIndex: + @pytest.fixture + def store(self, tmp_path, monkeypatch): + monkeypatch.setattr(dataset_catalog, "_DEFAULT_ROOT", tmp_path) + for eval_id in ("mmstar", "asr_fleurs_nl_nl"): + (tmp_path / "datasets" / eval_id / "test").mkdir(parents=True) + monkeypatch.setattr("pipette_scores.api.app._PACKAGES", packages.LocalStore(tmp_path / "packages")) + return tmp_path / "packages" + + def _index(self, store, eval_id: str, digests: dict[str | None, str], dataset: str = "test") -> None: + path = store / packages.index_key(_APP_VERSION, eval_id, dataset) + path.parent.mkdir(parents=True) + profiles = [{"prompt_profile": p, "samples_digest": d, "sample_count": 2} for p, d in digests.items()] + path.write_text(json.dumps({"eval_id": eval_id, "dataset_version": dataset, "profiles": profiles})) + + def test_a_set_the_index_lists_is_never_built(self, client, store): + digests = {None: "sha256:" + "1" * 64, "qwen3_5": "sha256:" + "2" * 64} + self._index(store, "mmstar", digests) + self._index(store, "asr_fleurs_nl_nl", {"voxtral": "sha256:" + "3" * 64}) + with ( + patch("pipette_scores.api.app._build_prompts", side_effect=AssertionError("built")), + patch("pipette_scores.api.app.dataset_catalog.asr_prompt_digests", side_effect=AssertionError("built")), + ): + for profile, digest in digests.items(): + query = f"?prompt_profile={profile}" if profile else "" + resp = client.get(f"/evals/mmstar/datasets/test/samples/digest{query}") + assert resp.json() == {"digest": digest, "sample_count": 2} + resp = client.get("/evals/asr_fleurs_nl_nl/datasets/test/samples/digest?prompt_profile=voxtral") + assert resp.json() == {"digest": "sha256:" + "3" * 64, "sample_count": 2} + + def test_an_indexed_set_this_server_lacks_is_not_found(self, client, store): + self._index(store, "mmstar", {None: "sha256:" + "1" * 64}, "not-synced") + self._index(store, "asr_fleurs_nl_nl", {"voxtral": "sha256:" + "3" * 64}, "not-synced") + assert client.get("/evals/mmstar/datasets/not-synced/samples/digest").status_code == 404 + resp = client.get("/evals/asr_fleurs_nl_nl/datasets/not-synced/samples/digest?prompt_profile=voxtral") + assert resp.status_code == 404 + + def test_a_set_without_an_index_is_built(self, client, store): + with patch("pipette_scores.api.app._build_prompts", return_value=VLM_SAMPLES) as build: + resp = client.get("/evals/mmstar/datasets/test/samples/digest") + assert resp.json() == {"digest": samples_digest(VLM_SAMPLES), "sample_count": 2} + build.assert_called_once_with("mmstar", "test", None) + + def test_an_unreadable_index_is_an_error_not_a_build(self, client, store): + path = store / packages.index_key(_APP_VERSION, "mmstar", "test") + path.parent.mkdir(parents=True) + path.write_text("{") + with patch("pipette_scores.api.app._build_prompts", return_value=VLM_SAMPLES) as build: + resp = client.get("/evals/mmstar/datasets/test/samples/digest") + assert resp.status_code == 500 + build.assert_not_called() + # --------------------------------------------------------------------------- # POST /score # --------------------------------------------------------------------------- +class TestScoreBindsToTheSamplesGiven: + def test_scoring_is_refused_when_the_samples_moved(self, client): + with patch("pipette_scores.api.app._build_prompts", return_value=VLM_SAMPLES): + resp = client.post( + "/score", + json={ + "eval_id": "mmstar", + "dataset_name": "test", + "completions": [], + "samples_digest": "sha256:" + "0" * 64, + }, + ) + assert resp.status_code == 409 + assert "not to these prompts" in resp.json()["detail"] + + def test_scoring_goes_on_when_the_samples_are_the_ones_given(self, client): + with ( + patch("pipette_scores.api.app._build_prompts", return_value=VLM_SAMPLES), + patch("pipette_scores.api.app._load_dataset_or_error", return_value=[]), + patch("pipette_scores.api.app._load_prompts_cached", return_value=VLM_SAMPLES), + patch("pipette_scores.api.app.scoring.score", return_value=([], {})), + ): + resp = client.post( + "/score", + json={ + "eval_id": "mmstar", + "dataset_name": "test", + "completions": [], + "samples_digest": samples_digest(VLM_SAMPLES), + }, + ) + assert resp.status_code == 200, resp.text + + class TestScoreEndpoint: def test_unknown_eval_returns_404(self, client): resp = client.post( @@ -144,6 +429,106 @@ def test_score_round_trip(self, client): assert sample["completion"] == '{"name": "Alice"}' assert sample["messages"][0]["content"] == "Q?" + def test_scored_samples_carry_the_profiles_prompts(self, client): + prompts = [EvalSample(id="1", messages=[ChatMessage(role="user", content="Q?")])] + with ( + patch("pipette_scores.api.app._load_dataset_cached", return_value=[_ifstruct_sample()]), + patch("pipette_scores.api.app.scoring.score", return_value=([], {})), + patch("pipette_scores.api.app._load_prompts_cached", return_value=prompts) as load, + ): + resp = client.post( + "/score", + json={ + "eval_id": "mmstar", + "dataset_name": "test", + "completions": [{"id": "1", "completion": "A"}], + "prompt_profile": "internvl", + }, + ) + assert resp.status_code == 200 + load.assert_called_once_with("mmstar", "test", "internvl") + + @pytest.mark.parametrize( + "completions, names", + [ + ([{"id": "a1", "completion": "B"}], "unanswered: b2"), + ( + [{"id": "a1", "completion": "B"}, {"id": "b2", "completion": "", "stop_reason": "failure"}], + "failed: b2", + ), + ], + ) + def test_a_vlm_eval_is_scored_only_with_an_answer_to_every_sample(self, client, completions, names): + dataset = [VlmSample(id=id, prompt="", answer="A", raw_doc={}) for id in ("a1", "b2")] + with ( + patch("pipette_scores.api.app._load_dataset_cached", return_value=dataset), + patch("pipette_scores.api.app.scoring.score") as score, + ): + resp = client.post("/score", json={"eval_id": "pope", "dataset_name": "test", "completions": completions}) + assert resp.status_code == 400 + assert names in resp.json()["detail"] + score.assert_not_called() + + def test_scored_samples_keep_their_text_but_not_their_images(self, client): + scores = [SampleScore(id="a1", is_correct=True), SampleScore(id="b2", is_correct=False)] + dataset = [VlmSample(id=id, prompt="", answer="A", raw_doc={}) for id in ("a1", "b2")] + with ( + patch("pipette_scores.api.app._load_dataset_cached", return_value=dataset), + patch("pipette_scores.api.app.scoring.score", return_value=(scores, {})), + patch("pipette_scores.api.app._load_prompts_cached", return_value=VLM_SAMPLES), + ): + resp = client.post( + "/score", + json={ + "eval_id": "mmstar", + "dataset_name": "test", + "completions": [{"id": "a1", "completion": "B"}, {"id": "b2", "completion": "no"}], + }, + ) + assert resp.status_code == 200 + messages = [s["messages"] for s in resp.json()["scored_samples"]] + assert messages == [ + [{"role": "user", "content": [{"type": "text", "text": "Which mood? 心情\nA. Sad\nB. Happy"}]}], + [{"role": "user", "content": "Is there a dog?"}], + ] + # What the samples route serves is unchanged. + assert VLM_SAMPLES[0].messages[0].content[0]["type"] == "image_url" + + @pytest.mark.parametrize("eval_id, profile", [("mmstar", "nope"), ("ifstruct", "internvl")]) + def test_score_refuses_a_prompt_profile_it_cannot_apply(self, client, eval_id, profile): + resp = client.post( + "/score", + json={"eval_id": eval_id, "dataset_name": "test", "completions": [], "prompt_profile": profile}, + ) + assert resp.status_code == 400 + + @pytest.mark.parametrize("enable_thinking, blanked", [(True, 1), (False, 0), (None, 0)]) + def test_score_reads_stop_reason_and_enable_thinking(self, client, enable_thinking, blanked): + samples = [_ifstruct_sample()] + prompts = [EvalSample(id="1", messages=[ChatMessage(role="user", content="Q?")])] + completion = 'Draft:\n```json\n{"name": "Alice"}\n```\nNow check whether the' + body = { + "eval_id": "ifstruct", + "dataset_name": "test", + # Keys the scorer has no use for, such as stop_detail, are ignored. + "completions": [{"id": "1", "completion": completion, "stop_reason": "truncated", "stop_detail": "limit"}], + } + if enable_thinking is not None: + body["enable_thinking"] = enable_thinking + with ( + patch("pipette_scores.api.app._load_dataset_cached", return_value=samples), + patch("pipette_scores.api.app._load_prompts_cached", return_value=prompts), + ): + resp = client.post("/score", json=body) + assert resp.status_code == 200 + result = resp.json() + # The cut draft is graded as written; only the strict view blanks it. + assert result["scored_samples"][0]["is_correct"] is True + # The completion is echoed as received, not as graded. + assert result["scored_samples"][0]["completion"] == completion + assert result["context"]["ifstruct_blanked_truncated"] == blanked + assert result["context"]["ifstruct_pass_at_1_blanked"] == (0.0 if blanked else 1.0) + def test_unknown_completion_id_returns_400(self, client): samples = [_ifstruct_sample()] prompts = [EvalSample(id="1", messages=[ChatMessage(role="user", content="Q?")])] @@ -201,3 +586,28 @@ def test_score_wrong_answer(self, client): body = resp.json() assert body["scored_samples"][0]["is_correct"] is False assert body["scored_samples"][0]["completion"] == "This is not JSON at all." + + +def test_a_worker_keeps_a_bounded_number_of_prompt_sets(): + """Each (eval, dataset, profile) is a set of its own, images included: a + worker keeps the few most recent, outside the catalog's own cache.""" + from pipette_scores.api import app as app_module + from pipette_scores import dataset_catalog + + assert app_module._load_prompts_cached.cache_info().maxsize == app_module._PROMPT_SETS + app_module._load_prompts_cached.cache_clear() + before = dataset_catalog.load_prompt_samples.cache_info().currsize + with patch.object(dataset_catalog.load_prompt_samples, "__wrapped__", return_value=[]) as built: + for profile in ("a", "b", "c", "d", "e", "a"): + app_module._load_prompts_cached("mmstar", "v", profile) + # "a" was evicted by the fifth set and built again. + assert built.call_count == 6 + assert dataset_catalog.load_prompt_samples.cache_info().currsize == before + app_module._load_prompts_cached.cache_clear() + + +def test_openapi_describes_the_samples_and_scored_samples(): + """The wrap serializers that omit unset fields keep the published schemas.""" + schemas = app.openapi()["components"]["schemas"] + assert {"id", "messages", "chat_template_kwargs"} <= set(schemas["EvalSample"]["properties"]) + assert {"id", "completion", "is_correct", "measures"} <= set(schemas["ScoredSample"]["properties"]) diff --git a/packages/pipette-scores/tests/test_asr_evals.py b/packages/pipette-scores/tests/test_asr_evals.py new file mode 100644 index 0000000..5c63766 --- /dev/null +++ b/packages/pipette-scores/tests/test_asr_evals.py @@ -0,0 +1,1154 @@ +"""The ASR evals end to end: builders, integrity, prompt profiles, audio, the samples stream and /score.""" + +from __future__ import annotations + +import base64 +import dataclasses +import hashlib +import io +import json +import pathlib +import re +import shutil +import uuid +from typing import Any +from unittest.mock import patch + +import numpy as np +import pyarrow as pa +import pyarrow.parquet as pq +import pytest +import soundfile as sf +from fastapi.testclient import TestClient + +from pipette_scores import cli, dataset_catalog, scoring +from pipette_scores.api import app as app_module +from pipette_scores.api.app import app +from pipette_scores.dataset_catalog import _asr_audio, _asr_build, _asr_common, asr_datasets, asr_prompt_profiles +from pipette_scores.dataset_catalog.asr_prompt_profiles import AsrClip +from pipette_scores.hashing import samples_digest, short_hash +from pipette_scores.types import ASR_EVAL_IDS, ASR_EVAL_ORDER, ChatMessage, EvalId, EvalSample, SampleCompletion + +RATE = 16000 + + +def _wav(seconds: float, *, subtype: str = "PCM_16", rate: int = RATE, channels: int = 1, fmt: str = "WAV") -> bytes: + n = int(seconds * rate) + t = np.arange(n, dtype=np.float32) / rate + tone = (0.1 * np.sin(2 * np.pi * 440 * t)).astype(np.float32) + data = np.stack([tone] * channels, axis=1) + buffer = io.BytesIO() + sf.write(buffer, data, rate, format=fmt, subtype=subtype) + return buffer.getvalue() + + +def _source_rows(refs: list[str], *, seconds: float = 1.0, session: str | None = None) -> list[dict[str, Any]]: + return [ + { + "audio": _wav(seconds + 0.1 * i), + "audio_format": "wav", + "reference": ref, + "session": session, + "chunk_index": i if session else None, + "raw_doc": {"id": f"utt-{i}", "text": ref, **({"parent_chunk_count": len(refs)} if session else {})}, + } + for i, ref in enumerate(refs) + ] + + +def _build(tmp_path: pathlib.Path, eval_id: EvalId, refs: list[str], **kw: Any) -> tuple[str, pathlib.Path]: + """A fake dataset under tmp_path/datasets, named uniquely (the loaders cache).""" + name = f"test-{uuid.uuid4().hex[:8]}" + _asr_build.build( + asr_datasets.ASR_DATASETS[eval_id], tmp_path / "datasets", name, rows=iter(_source_rows(refs, **kw)) + ) + return name, tmp_path / "datasets" / eval_id.value / name + + +def _rewrite(target: pathlib.Path, change) -> None: + """Rewrite test.parquet's rows through ``change`` (metadata.json untouched).""" + path = target / "test.parquet" + table = pq.read_table(path) + rows = change(table.to_pylist()) + pq.write_table(pa.Table.from_pylist(rows, schema=table.schema), path, row_group_size=_asr_build.ROW_GROUP_SIZE) + + +# --------------------------------------------------------------------------- +# Catalog and builders +# --------------------------------------------------------------------------- + + +def test_the_catalog_is_the_open_asr_leaderboards_public_english_sets_and_the_fleurs_locales(): + assert len(ASR_EVAL_IDS) == 16 + # The leaderboard's 9 public English sets, in its score_results column order + # (AMI-Cleaned ... SPGISpeech, URGENT2024, Voice Arena Monsoon, Voxpopuli-AA-Cleaned). + assert [e.value for e in asr_datasets.OPEN_ASR_EVAL_IDS] == [ + "asr_esb_ami_cleaned", + "asr_esb_earnings22_cleaned_aa_chunked", + "asr_esb_gigaspeech_cleaned", + "asr_esb_librispeech_test_clean", + "asr_esb_librispeech_test_other", + "asr_esb_spgispeech", + "asr_esb_urgent2024", + "asr_esb_monsoon_en_in", + "asr_esb_voxpopuli_cleaned_aa", + ] + # English and the Open ASR Leaderboard's multilingual FLEURS languages but + # Armenian. + assert [e.value.removeprefix("asr_fleurs_") for e in asr_datasets.FLEURS_EVAL_IDS] == [ + "en_us", "es_419", "pt_br", "de_de", "fr_fr", "it_it", "nl_nl", + ] # fmt: skip + # The dropped locales are gone from the ids, not only from the catalog. + for dropped in ( + "ja_jp", + "pl_pl", + "cs_cz", + "da_dk", + "el_gr", + "fi_fi", + "hu_hu", + "ro_ro", + "sv_se", + "cmn_hans_cn", + "ko_kr", + ): + assert f"asr_fleurs_{dropped}" not in {e.value for e in EvalId}, dropped + assert list(asr_datasets.ASR_DATASETS) == list(ASR_EVAL_ORDER) + english = [asr_datasets.ASR_DATASETS[e] for e in asr_datasets.OPEN_ASR_EVAL_IDS] + fleurs = [asr_datasets.ASR_DATASETS[e] for e in asr_datasets.FLEURS_EVAL_IDS] + assert all(d.normalizer.value == "open_asr_english" and d.open_asr_exclusions for d in english) + assert all(d.aligner.value == "kaldialign_compound" for d in english) + urgent = asr_datasets.ASR_DATASETS[EvalId.ASR_ESB_URGENT2024] + assert (urgent.repo, urgent.revision, urgent.subset, urgent.split, urgent.expected_samples) == ( + "hf-audio/open-asr-leaderboard", + "de79e57adf9983993bcc72586d7313460825f5eb", + "urgent2024", + "test", + 1000, + ) + assert "non-commercial" in urgent.license and urgent.expected_source_sha256 == ( + "d34af39602eb850899831527a95a52c139023f3e4f7ee1706245ac4ecff18c34" + ) + # Every one of the 16 sets has its fingerprint pinned, and only they do. + pins = [d.expected_source_sha256 for d in asr_datasets.ASR_DATASETS.values()] + assert len(pins) == 16 and all(re.fullmatch(r"[0-9a-f]{64}", p) for p in pins) + assert set(asr_datasets.EXPECTED_SOURCE_SHA256) == {e.value for e in ASR_EVAL_ORDER} + # FLEURS is scored with the leaderboard's multilingual scoring, every locale + # against FLEURS's own `transcription` references. + assert all( + (d.normalizer.value, d.aligner.value, d.open_asr_exclusions, d.reference_column) + == ("open_asr_multilingual", "kaldialign_compound_pairs", False, "transcription") + for d in fleurs + ) + assert {(d.repo, d.revision) for d in fleurs} == {("google/fleurs", "70bb2e84b976b7e960aa89f1c648e09c59f894dd")} + + +def test_a_build_keeps_order_audio_and_language(tmp_path): + name, target = _build(tmp_path, EvalId.ASR_FLEURS_NL_NL, ["een", "twee", "drie"]) + meta = json.loads((target / "metadata.json").read_text()) + assert meta["num_samples"] == 3 and meta["scoring"]["aligner"] == "kaldialign_compound_pairs" + assert meta["limit"] is None and meta["source"]["reference_column"] == "transcription" + assert meta["audio"]["subtypes"] == {"PCM_16": 3} and meta["audio"]["formats"] == {"WAV": 3} + assert not (target / "test.parquet.partial").exists() + samples = _asr_common.load_eval_samples(target) + assert [s.reference for s in samples] == ["een", "twee", "drie"] + assert [s.raw_doc["language_name"] for s in samples] == ["Dutch"] * 3 + assert samples[0].raw_doc["language_alpha_2"] == "nl" + # The id is the position in the source split. + assert samples[1].id == short_hash("asr_fleurs_nl_nl|1") + + +def test_a_build_streams_its_row_groups(tmp_path): + _, target = _build( + tmp_path, EvalId.ASR_FLEURS_NL_NL, [f"w{i}" for i in range(_asr_build.ROW_GROUP_SIZE + 5)], seconds=0.05 + ) + groups = pq.ParquetFile(target / "test.parquet").metadata + assert [groups.row_group(i).num_rows for i in range(groups.num_row_groups)] == [_asr_build.ROW_GROUP_SIZE, 5] + + +def test_a_clip_off_16_khz_mono_is_stored_as_serving_would_bring_it(tmp_path): + """A 16 kHz mono clip is kept as the source ships it. Any other (MonsoonASR is + 44.1 kHz) is brought to 16 kHz mono at build, as serving would have, so + serving resamples nothing and its samples are the same on every CPU + architecture; the row records the source's rate and channels.""" + clips = [_wav(1.0), _wav(0.5, rate=44100), _wav(0.4, rate=48000, channels=2, subtype="FLOAT")] + rows = [ + {"audio": a, "audio_format": "wav", "reference": f"r{i}", "session": None, "chunk_index": None, "raw_doc": {}} + for i, a in enumerate(clips) + ] + dataset = asr_datasets.ASR_DATASETS[EvalId.ASR_ESB_MONSOON_EN_IN] + _asr_build.build(dataset, tmp_path / "datasets", "test-rates", rows=iter(rows)) + target = tmp_path / "datasets" / dataset.eval_id.value / "test-rates" + stored = pq.read_table(target / "test.parquet").to_pylist() + assert stored[0]["audio"] == clips[0] + assert [r["audio"] for r in stored[1:]] == [_asr_audio.wav(_asr_audio.resample_16k_mono(c)) for c in clips[1:]] + assert [(r["sample_rate"], r["channels"], r["num_frames"]) for r in stored] == [ + (16000, 1, 16000), + (16000, 1, 8000), + (16000, 1, 6400), + ] + sources = [json.loads(r["raw_doc"]) for r in stored] + assert [(d.get("source_sample_rate"), d.get("source_channels")) for d in sources] == [ + (None, None), + (44100, 1), + (48000, 2), + ] + # Served, a stored clip is the samples serving brought the source clip to. + for clip, row in zip(clips, stored, strict=True): + assert np.array_equal(_asr_audio.to_16k_mono(row["audio"]), _asr_audio.to_16k_mono(clip)) + audio = json.loads((target / "metadata.json").read_text())["audio"] + assert audio["sample_rates"] in ({16000: 3}, {"16000": 3}) + assert audio["source_sample_rates"] in ({16000: 1, 44100: 1, 48000: 1}, {"16000": 1, "44100": 1, "48000": 1}) + assert audio["channels"] in ({1: 3}, {"1": 3}) + assert audio["subtypes"] == {"FLOAT": 2, "PCM_16": 1} + + +def test_the_source_fingerprint_ignores_how_the_builder_resampled(tmp_path): + """The catalog pins the source: a resample that rounds otherwise (another CPU + architecture) moves content_sha256, never source_sha256, which equals the + fingerprint of the same clips stored unresampled.""" + clips = [_wav(1.0), _wav(0.5, rate=44100), _wav(0.4, rate=48000, channels=2, subtype="FLOAT")] + rows = [ + {"audio": a, "audio_format": "wav", "reference": f"r{i}", "session": None, "chunk_index": None, "raw_doc": {}} + for i, a in enumerate(clips) + ] + dataset = asr_datasets.ASR_DATASETS[EvalId.ASR_ESB_MONSOON_EN_IN] + meta = _asr_build.build(dataset, tmp_path / "datasets", "test-source", rows=iter(rows)) + stored = pq.read_table(tmp_path / "datasets" / dataset.eval_id.value / "test-source" / "test.parquet").to_pylist() + assert meta["source_sha256"] == _asr_common.source_sha256(stored) != meta["content_sha256"] + other_arch = [{**r, "audio_sha256": "0" * 64} if i else r for i, r in enumerate(stored)] + assert _asr_common.source_sha256(other_arch) == meta["source_sha256"] + assert _asr_common.content_sha256(other_arch) != meta["content_sha256"] + # As the builder before build-time resampling stored them: the source bytes, rate and raw_doc. + unresampled = [ + { + **r, + "audio_sha256": hashlib.sha256(c).hexdigest(), + "sample_rate": sf.info(io.BytesIO(c)).samplerate, + "channels": sf.info(io.BytesIO(c)).channels, + "num_frames": sf.info(io.BytesIO(c)).frames, + "raw_doc": json.dumps( + {k: v for k, v in json.loads(r["raw_doc"]).items() if k not in _asr_common.SOURCE_KEYS}, + ensure_ascii=False, + sort_keys=True, + ), + } + for r, c in zip(stored, clips, strict=True) + ] + assert _asr_common.content_sha256(unresampled) == meta["source_sha256"] + + +def test_a_full_canonical_build_must_match_the_catalog(tmp_path): + dataset = asr_datasets.ASR_DATASETS[EvalId.ASR_FLEURS_NL_NL] + canonical = asr_datasets.DATASET_VERSION + target = tmp_path / "datasets" / dataset.eval_id.value / canonical + # The pinned 364 clips, not 2. + with pytest.raises(SystemExit, match="expected 364"): + _asr_build.build(dataset, tmp_path / "datasets", canonical, rows=iter(_source_rows(["een", "twee"]))) + assert not (target / "test.parquet").exists() + # The right count but another fingerprint. + two = dataclasses.replace(dataset, expected_samples=2) + with pytest.raises(SystemExit, match="source_sha256 .* != expected"): + _asr_build.build(two, tmp_path / "datasets", canonical, rows=iter(_source_rows(["een", "twee"]))) + assert not (target / "test.parquet").exists() and not (target / "test.parquet.partial").exists() + + +def test_a_smoke_build_may_not_take_the_canonical_version_and_is_never_served(tmp_path): + dataset = asr_datasets.ASR_DATASETS[EvalId.ASR_FLEURS_NL_NL] + with pytest.raises(SystemExit, match="canonical version"): + _asr_build.build( + dataset, tmp_path / "datasets", asr_datasets.DATASET_VERSION, limit=1, rows=iter(_source_rows(["a"])) + ) + _asr_build.build(dataset, tmp_path / "datasets", "smoke", limit=1, rows=iter(_source_rows(["a", "b"]))) + with pytest.raises(_asr_common.AsrDatasetError, match="smoke build"): + _asr_common.load_eval_samples(tmp_path / "datasets" / dataset.eval_id.value / "smoke") + + +def test_the_builder_refuses_an_output_inside_the_checkout(tmp_path): + checkout = tmp_path / "repo" + (checkout / ".git").mkdir(parents=True) + with pytest.raises(SystemExit, match="inside the repository checkout"): + _asr_build.refuse_inside_checkout(checkout / "datasets", checkout) + _asr_build.refuse_inside_checkout(tmp_path / "elsewhere", checkout) + # The real checkout's datasets/ (the old documented --out) is refused too. + with pytest.raises(SystemExit, match="inside the repository checkout"): + _asr_build.build(asr_datasets.ASR_DATASETS[EvalId.ASR_FLEURS_NL_NL], _asr_build._CHECKOUT / "datasets", "x") + + +# --------------------------------------------------------------------------- +# Integrity: the fingerprint, the catalog, the order, the served bytes +# --------------------------------------------------------------------------- + + +def test_a_tampered_digest_is_refused_at_load(tmp_path): + _, target = _build(tmp_path, EvalId.ASR_FLEURS_NL_NL, ["een"]) + meta = json.loads((target / "metadata.json").read_text()) + meta["content_sha256"] = "0" * 64 + (target / "metadata.json").write_text(json.dumps(meta)) + with pytest.raises(_asr_common.AsrDatasetError, match="content_sha256"): + _asr_common.load_eval_samples(target) + + +def test_a_dataset_without_a_digest_is_refused(tmp_path): + _, target = _build(tmp_path, EvalId.ASR_FLEURS_NL_NL, ["een"]) + meta = json.loads((target / "metadata.json").read_text()) + del meta["content_sha256"] + (target / "metadata.json").write_text(json.dumps(meta)) + with pytest.raises(_asr_common.AsrDatasetError, match="no content_sha256"): + _asr_common.load_eval_samples(target) + + +def test_a_recipe_that_is_not_the_catalogs_is_refused(tmp_path): + _, target = _build(tmp_path, EvalId.ASR_FLEURS_NL_NL, ["een"]) + meta = json.loads((target / "metadata.json").read_text()) + meta["scoring"]["normalizer"] = "open_asr_english" + (target / "metadata.json").write_text(json.dumps(meta)) + with pytest.raises(_asr_common.AsrDatasetError, match="scoring recipe"): + _asr_common.load_eval_samples(target) + + +def test_the_canonical_version_must_carry_the_catalogs_fingerprint(tmp_path): + _, target = _build(tmp_path, EvalId.ASR_FLEURS_NL_NL, ["een"]) + canonical = target.with_name(asr_datasets.DATASET_VERSION) + target.rename(canonical) + meta = json.loads((canonical / "metadata.json").read_text()) + (canonical / "metadata.json").write_text(json.dumps(meta | {"dataset": asr_datasets.DATASET_VERSION})) + with pytest.raises(_asr_common.AsrDatasetError, match="expected_source_sha256"): + _asr_common.load_eval_samples(canonical) + + +def test_rows_out_of_position_are_refused(tmp_path): + """Swapping two whole rows keeps the (order-free) digest, but not the order.""" + _, target = _build(tmp_path, EvalId.ASR_FLEURS_NL_NL, ["een", "twee", "drie"]) + _rewrite(target, lambda rows: [rows[1], rows[0], rows[2]]) + with pytest.raises(_asr_common.AsrDatasetError, match="must be its position"): + _asr_common.load_eval_samples(target) + + +def test_a_row_not_at_16_khz_mono_is_refused(tmp_path): + """A build from before resampling moved to the builder: serving would resample it.""" + _, target = _build(tmp_path, EvalId.ASR_FLEURS_NL_NL, ["een"]) + _rewrite(target, lambda rows: [{**r, "sample_rate": 44100} for r in rows]) + meta = json.loads((target / "metadata.json").read_text()) + meta["content_sha256"] = _asr_common.content_sha256(pq.read_table(target / "test.parquet").to_pylist()) + (target / "metadata.json").write_text(json.dumps(meta)) + with pytest.raises(_asr_common.AsrDatasetError, match="44100 Hz x 1"): + _asr_common.load_eval_samples(target) + + +def test_swapped_audio_is_refused_when_served(tmp_path): + """The columns (and so the digest) are intact; the served bytes are not.""" + name, target = _build(tmp_path, EvalId.ASR_FLEURS_NL_NL, ["een", "twee"]) + + def swap(rows): + rows[0]["audio"], rows[1]["audio"] = rows[1]["audio"], rows[0]["audio"] + return rows + + _rewrite(target, swap) + _asr_common.load_eval_samples(target) # scoring reads no audio + with pytest.raises(_asr_common.AsrDatasetError, match="audio does not match"): + list(_asr_common.iter_prompt_samples(EvalId.ASR_FLEURS_NL_NL, target, "lfm3")) + + +# --------------------------------------------------------------------------- +# Prompt profiles (each model's public usage guidance; asr_prompt_profiles.py) +# --------------------------------------------------------------------------- + +AUDIO = {"type": "input_audio", "input_audio": {"data": "QUJD", "format": "wav"}} + + +def _clip(eval_id: EvalId, language: str, name: str, alpha_2: str | None) -> AsrClip: + return AsrClip(eval_id=eval_id, language=language, language_name=name, language_alpha_2=alpha_2) + + +ENGLISH_AMI = _clip(EvalId.ASR_ESB_AMI_CLEANED, "eng", "English", "en") +GERMAN = _clip(EvalId.ASR_FLEURS_DE_DE, "deu", "German", "de") +# No FLEURS locale of the catalog lies outside Voxtral's hinted languages; a +# clip that did (here a Polish one under a catalog id) is left to detection. +POLISH = _clip(EvalId.ASR_FLEURS_NL_NL, "pol", "Polish", "pl") + + +def test_gemma4_asks_with_googles_prompt_then_the_clip(): + [user] = asr_prompt_profiles.PROFILES["gemma4"].messages(ENGLISH_AMI, AUDIO) + assert user["role"] == "user" and user["content"][1] == AUDIO + assert user["content"][0]["text"] == ( + "Transcribe the following speech segment in English into English text.\n\n" + "Follow these specific instructions for formatting the answer:\n" + "* Only output the transcription, with no newlines.\n" + "* When transcribing numbers, write the digits, i.e. write 1.7 and not one point seven, " + "and write 3 instead of three." + ) + [user] = asr_prompt_profiles.PROFILES["gemma4"].messages(GERMAN, AUDIO) + assert user["content"][0]["text"].startswith("Transcribe the following speech segment in German into German text.") + + +def test_qwen25_omni_has_its_system_turn_and_the_clip_first(): + system, user = asr_prompt_profiles.PROFILES["qwen25_omni"].messages(GERMAN, AUDIO) + assert system == {"role": "system", "content": "You are a speech recognition model."} + assert user["content"] == [ + AUDIO, + {"type": "text", "text": "Transcribe the German audio into text without any punctuation marks."}, + ] + + +@pytest.mark.parametrize( + "clip, language", + [(ENGLISH_AMI, "en"), (_clip(EvalId.ASR_FLEURS_NL_NL, "nld", "Dutch", "nl"), "nl"), (GERMAN, "de"), (POLISH, "")], +) +def test_voxtral_names_the_language_only_where_mistral_reports_it(clip, language): + profile = asr_prompt_profiles.PROFILES["voxtral"] + assert profile.messages(clip, AUDIO) == [{"role": "user", "content": [AUDIO]}] + assert profile.chat_template_kwargs(clip) == {"transcription": True, "language": language} + + +_GENERAL = "Please listen to the audio carefully and transcribe it with high precision." +_LOWERCASE = ( + "Please listen to the audio carefully and transcribe it with high precision. " + "Please output in lower case without punctuation." +) +# The FLEURS prompts, spelled out here independently of the profile. +_FLEURS_PROMPTS = { + EvalId.ASR_FLEURS_DE_DE: ( + "deu", + "German", + "Bitte hören Sie sich die Audioaufnahme aufmerksam an und transkribieren Sie sie mit hoher Präzision. " + "Transkribieren Sie sie wortgetreu in der Originalsprache (Deutsch); übersetzen Sie sie nicht.", + ), + EvalId.ASR_FLEURS_ES_419: ( + "spa", + "Spanish", + "Por favor, escuche atentamente el audio y transcríbalo con gran precisión. " + "Transcríbalo palabra por palabra en el idioma original (español); no lo traduzca.", + ), + EvalId.ASR_FLEURS_PT_BR: ( + "por", + "Portuguese", + "Por favor, ouça o áudio com atenção e transcreva-o com alta precisão. " + "Transcreva-o palavra por palavra no idioma original (português); não o traduza.", + ), + EvalId.ASR_FLEURS_FR_FR: ( + "fra", + "French", + "Veuillez écouter attentivement l'audio et le transcrire avec précision. " + "Transcrivez-le mot pour mot dans la langue d'origine (français) ; ne le traduisez pas.", + ), + EvalId.ASR_FLEURS_IT_IT: ( + "ita", + "Italian", + "Per favore, ascolta attentamente l'audio e trascrivilo con alta precisione. " + "Trascrivilo parola per parola nella lingua originale (italiano); non tradurlo.", + ), + EvalId.ASR_FLEURS_NL_NL: ( + "nld", + "Dutch", + "Luister aandachtig naar de audio en transcribeer deze met hoge precisie. " + "Transcribeer het woordelijk in de oorspronkelijke taal (Nederlands); vertaal het niet.", + ), +} + + +@pytest.mark.parametrize( + "eval_id, language, name, prompt", + [ + ( + EvalId.ASR_ESB_SPGISPEECH, + "eng", + "English", + "Please listen to the financial domain audio and transcribe it accurately.", + ), + (EvalId.ASR_ESB_GIGASPEECH_CLEANED, "eng", "English", _LOWERCASE), + (EvalId.ASR_ESB_VOXPOPULI_CLEANED_AA, "eng", "English", _LOWERCASE), + (EvalId.ASR_ESB_LIBRISPEECH_TEST_CLEAN, "eng", "English", _LOWERCASE), + (EvalId.ASR_ESB_LIBRISPEECH_TEST_OTHER, "eng", "English", _LOWERCASE), + (EvalId.ASR_ESB_AMI_CLEANED, "eng", "English", _GENERAL), + (EvalId.ASR_ESB_MONSOON_EN_IN, "eng", "English", _GENERAL), + (EvalId.ASR_ESB_EARNINGS22_CLEANED_AA_CHUNKED, "eng", "English", _GENERAL), + (EvalId.ASR_ESB_URGENT2024, "eng", "English", _GENERAL), + # FLEURS English keeps OpenBMB's English prompt, with no added clause. + (EvalId.ASR_FLEURS_EN_US, "eng", "English", _GENERAL), + *((e, language, name, prompt) for e, (language, name, prompt) in _FLEURS_PROMPTS.items()), + ], +) +def test_minicpm_o_takes_openbmbs_prompt_for_the_dataset(eval_id, language, name, prompt): + clip = _clip(eval_id, language, name, None) + [user] = asr_prompt_profiles.PROFILES["minicpm_o"].messages(clip, AUDIO) + # OmniEvalKit's "\n{audio}". + assert user["content"] == [{"type": "text", "text": prompt + "\n"}, AUDIO] + + +def test_minicpm_o_asks_each_fleurs_language_in_that_language(): + """Every non-English FLEURS locale has its own prompt, none of them English: + the English extension it replaced named the language in English, which the + model then answered in.""" + non_english = [e for e in asr_datasets.FLEURS_EVAL_IDS if asr_datasets.ASR_DATASETS[e].language != "eng"] + assert set(asr_prompt_profiles.MINICPM_FLEURS) == set(non_english) == set(_FLEURS_PROMPTS) + for eval_id, (language, name, prompt) in _FLEURS_PROMPTS.items(): + sent = asr_prompt_profiles.minicpm_o_prompt(_clip(eval_id, language, name, None)) + assert sent == prompt and "Please" not in sent and "transcribe the content" not in sent, eval_id + # French opens with OmniEvalKit's commonvoice_fr prompt, verbatim. + assert asr_prompt_profiles.MINICPM_FLEURS[EvalId.ASR_FLEURS_FR_FR].startswith( + "Veuillez écouter attentivement l'audio et le transcrire avec précision. " + ) + # Negative control: a non-English clip with no prompt is refused, never + # sent the English one. + with pytest.raises(ValueError, match="no Polish prompt"): + asr_prompt_profiles.minicpm_o_prompt(_clip(EvalId.ASR_ESB_AMI_CLEANED, "pol", "Polish", "pl")) + + +def test_the_lfm3_profile_names_the_language_in_the_system_turn(): + system, user = asr_prompt_profiles.PROFILES["lfm3"].messages(GERMAN, AUDIO) + assert system == {"role": "system", "content": "Perform ASR in German."} + assert user == {"role": "user", "content": [AUDIO]} + system, _ = asr_prompt_profiles.PROFILES["lfm3"].messages(ENGLISH_AMI, AUDIO) + assert system == {"role": "system", "content": "Perform ASR in English."} + + +@pytest.mark.parametrize("clip", [ENGLISH_AMI, GERMAN]) +def test_lfm25_audio_always_asks_with_its_cards_fixed_prompt(clip): + """The card's one prompt, "Perform ASR.", whatever the clip's language, with + no language named (LFM3, on the same clip, names it: the negative control).""" + system, user = asr_prompt_profiles.PROFILES["lfm25_audio"].messages(clip, AUDIO) + assert system == {"role": "system", "content": "Perform ASR."} + assert user == {"role": "user", "content": [AUDIO]} + lfm3_system, _ = asr_prompt_profiles.PROFILES["lfm3"].messages(clip, AUDIO) + assert lfm3_system != system + + +def test_every_profile_names_its_public_source(): + for name, profile in asr_prompt_profiles.PROFILES.items(): + assert profile.reference, name + assert "@" in profile.reference or name == "lfm3", name + + +@pytest.mark.parametrize( + "profile, raw, cleaned", + [ + ("qwen25_omni", "system\nyou are\nassistant\n hello world ", "hello world"), + # OpenBMB's text between <|tts_bos|> and <|tts_eos|>, stripped. + ("minicpm_o", "<|tts_bos|> Hello there.<|tts_eos|> ignored", "Hello there."), + ("minicpm_o", "Hello there.", "Hello there."), + ("lfm25_audio", "Hello.<|im_end|>", "Hello."), + ("lfm3", "Hello.<|im_end|>", "Hello.<|im_end|>"), + ("lfm3", " Hello. ", " Hello. "), + ("gemma4", " Hello. ", " Hello. "), + ("voxtral", " Hello. ", " Hello. "), + ], +) +def test_each_profile_cleans_the_answer_as_its_source_does(profile, raw, cleaned): + assert asr_prompt_profiles.PROFILES[profile].postprocess(raw) == cleaned + + +# --------------------------------------------------------------------------- +# Audio: one resample to 16 kHz mono for every profile +# --------------------------------------------------------------------------- + + +def _decode(b64: str) -> tuple[np.ndarray, Any]: + wav = base64.b64decode(b64) + data, _ = sf.read(io.BytesIO(wav), dtype="float32") + return data, sf.info(io.BytesIO(wav)) + + +@pytest.mark.parametrize("profile", sorted(asr_prompt_profiles.PROFILES)) +def test_every_profile_sends_the_same_16_khz_mono_clip(profile): + p = asr_prompt_profiles.PROFILES[profile] + cap = 480_000 if profile == "gemma4" else None + assert p.audio_max_samples == cap + # A 16 kHz clip is not resampled: its samples arrive unchanged (Gemma 4's cut at 30 s), + # as float where they are finer than 16 bits ... + clip = _wav(31.0, subtype="FLOAT") + source, _ = sf.read(io.BytesIO(clip), dtype="float32") + sent, info = _decode(_asr_common.encode_for(clip, p)) + assert (info.subtype, info.frames, info.samplerate, info.channels) == ("FLOAT", cap or len(source), RATE, 1) + assert np.array_equal(sent, source[: cap or len(source)]) + # ... and as 16-bit PCM where they are 16-bit values, the same samples. + clip = _wav(31.0, subtype="PCM_16") + source, _ = sf.read(io.BytesIO(clip), dtype="float32") + sent, info = _decode(_asr_common.encode_for(clip, p)) + assert (info.subtype, info.frames, info.samplerate, info.channels) == ("PCM_16", cap or len(source), RATE, 1) + assert np.array_equal(sent, source[: cap or len(source)]) + # A 44.1 kHz stereo clip comes at 16 kHz mono, ceil(n * 16000 / 44100) samples, + # as float: resampling leaves 16-bit values. + sent, info = _decode(_asr_common.encode_for(_wav(2.0, rate=44100, channels=2), p)) + assert (info.samplerate, info.channels, info.subtype, info.frames) == (RATE, 1, "FLOAT", int(np.ceil(2.0 * RATE))) + # 440 Hz survives the resampling. + spectrum = np.abs(np.fft.rfft(sent)) + assert abs(np.argmax(spectrum) * RATE / len(sent) - 440) < 2 + + +def test_a_clip_of_16_bit_values_is_served_as_16_bit_pcm_exactly(): + """Any float array whose every sample is k / 32768 for a 16-bit k, a float + WAV's included (as some sources ship them), goes out as 16-bit PCM and + decodes to the very same float32 samples; one finer sample keeps it float.""" + values = np.array([-32768, -1, 0, 1, 12345, 32767], dtype=np.int16) + exact = values / 32768.0 + assert np.array_equal(_asr_audio.pcm16_values(exact), values) + for source in (exact, exact.astype(np.float32)): + wav = _asr_audio.wav(source) + decoded, info = sf.read(io.BytesIO(wav), dtype="float32"), sf.info(io.BytesIO(wav)) + assert info.subtype == "PCM_16" and np.array_equal(decoded[0], exact.astype(np.float32)) + # The float32 a model receives is k * 2**-15 either way (llama.cpp's loader + # multiplies by 0.000030517578125, soundfile divides by 32768). + assert np.array_equal(values.astype(np.float32) * np.float32(2.0**-15), exact.astype(np.float32)) + for finer in (np.append(exact, 0.5 / 32768), np.append(exact, 1.0), np.append(exact, -1.0 - 1 / 32768)): + assert _asr_audio.pcm16_values(finer) is None + assert sf.info(io.BytesIO(_asr_audio.wav(finer))).subtype == "FLOAT" + assert np.array_equal(_asr_audio.pcm16_values(np.zeros(0)), np.zeros(0, dtype=np.int16)) + + +def test_the_served_pcm_wav_is_libsndfiles_pcm_16_layout(): + values = (np.random.default_rng(1).standard_normal(16001) * 3000).astype(np.int16) + ours = _asr_audio.pcm16_wav(values) + assert _chunks(ours) == [b"fmt ", b"data"] + buffer = io.BytesIO() + sf.write(buffer, values[:, np.newaxis], RATE, format="WAV", subtype="PCM_16") + assert ours == buffer.getvalue() + + +def test_the_resample_is_soxr_hq_with_librosas_length_rule(): + """22,051 frames at 44.1 kHz are 8000.36 at 16 kHz: librosa's fix_length makes + it ceil() = 8001 samples, soxr alone gives 8000.""" + import soxr + + clip = (0.1 * np.sin(2 * np.pi * 440 * np.arange(22051) / 44100)).astype(np.float64) + buffer = io.BytesIO() + sf.write(buffer, clip, 44100, format="WAV", subtype="DOUBLE") + out = _asr_audio.resample_16k_mono(buffer.getvalue()) + plain = soxr.resample(clip, 44100, 16000, quality="soxr_hq") + assert out.dtype == np.float64 and len(out) == 8001 and len(plain) == 8000 + assert np.array_equal(out[:8000], plain) and out[8000] == 0.0 + # Stereo is the mean of the channels, before resampling. + stereo = np.stack([clip, 0.5 * clip], axis=1) + buffer = io.BytesIO() + sf.write(buffer, stereo, 44100, format="WAV", subtype="DOUBLE") + mixed = _asr_audio.resample_16k_mono(buffer.getvalue()) + assert np.array_equal(mixed[:8000], soxr.resample(stereo.T.mean(axis=0), 44100, 16000, quality="soxr_hq")) + + +def _chunks(wav: bytes) -> list[bytes]: + assert wav[:4] == b"RIFF" and wav[8:12] == b"WAVE" and int.from_bytes(wav[4:8], "little") == len(wav) - 8 + ids, i = [], 12 + while i < len(wav): + size = int.from_bytes(wav[i + 4 : i + 8], "little") + ids.append(wav[i : i + 4]) + i += 8 + size + (size & 1) + assert i == len(wav) + return ids + + +def _libsndfile_float_wav(samples: np.ndarray) -> bytes: + buffer = io.BytesIO() + sf.write(buffer, samples[:, np.newaxis], RATE, format="WAV", subtype="FLOAT") + return buffer.getvalue() + + +def test_the_served_wav_is_libsndfiles_float_layout_without_its_peak_chunk(): + samples = np.random.default_rng(0).standard_normal(16001) * 0.3 + ours = _asr_audio.float_wav(samples) + assert _chunks(ours) == [b"fmt ", b"fact", b"data"] + # libsndfile's own FLOAT WAV, its PEAK chunk (and the RIFF size) aside, is the same bytes. + theirs = _libsndfile_float_wav(samples) + assert _chunks(theirs) == [b"fmt ", b"fact", b"PEAK", b"data"] + peak = theirs.index(b"PEAK") + without = theirs[:peak] + theirs[peak + 8 + 16 :] + assert ours == without[:4] + (len(without) - 8).to_bytes(4, "little") + without[8:] + decoded, rate = sf.read(io.BytesIO(ours), dtype="float32") + assert rate == RATE and np.array_equal(decoded, samples.astype(np.float32)) + + +def test_a_clip_is_served_as_the_same_bytes_on_every_call(): + """The served WAV depends on the samples alone. The negative control: + libsndfile's FLOAT WAV stamps its PEAK chunk with the time, so two writes + in different seconds differ.""" + import time + + clip = _wav(0.7, rate=44100, channels=2) + mono = _asr_audio.resample_16k_mono(clip) + first, libsndfile_first = _asr_audio.encode(clip), _libsndfile_float_wav(mono) + second_of_first = int(time.time()) + while int(time.time()) == second_of_first: + time.sleep(0.01) + second, libsndfile_second = _asr_audio.encode(clip), _libsndfile_float_wav(mono) + assert first == second + assert libsndfile_first != libsndfile_second + + +# --------------------------------------------------------------------------- +# API: the samples stream and its digest +# --------------------------------------------------------------------------- + + +@pytest.fixture +def client(tmp_path, monkeypatch): + monkeypatch.setattr(dataset_catalog, "_DEFAULT_ROOT", tmp_path) + monkeypatch.setattr(app_module, "_DIGESTS", {}) + return TestClient(app) + + +def _samples(resp) -> list[EvalSample]: + return [EvalSample.model_validate(s) for s in resp.json()["samples"]] + + +def test_an_asr_set_is_streamed_whole_with_its_digest(client, tmp_path): + name, _ = _build(tmp_path, EvalId.ASR_FLEURS_NL_NL, ["een", "twee", "drie"]) + resp = client.get(f"/evals/asr_fleurs_nl_nl/datasets/{name}/samples?prompt_profile=voxtral") + assert resp.status_code == 200, resp.text + assert resp.headers["content-type"] == "application/json" + body = resp.json() + assert set(body) == {"samples"} + samples = _samples(resp) + assert [s.id for s in samples] == [short_hash(f"asr_fleurs_nl_nl|{i}") for i in range(3)] + sample = body["samples"][1] + assert sample["chat_template_kwargs"] == {"transcription": True, "language": "nl"} + [part] = sample["messages"][0]["content"] + _, info = _decode(part["input_audio"]["data"]) + # The test clips are 16-bit PCM at 16 kHz: served as such, bit for bit. + assert part["input_audio"]["format"] == "wav" and info.subtype == "PCM_16" and info.frames == int(1.1 * RATE) + # The header is the body's digest, and the digest route says the same. + digest = resp.headers["x-samples-digest"] + assert digest == samples_digest(samples) + route = client.get(f"/evals/asr_fleurs_nl_nl/datasets/{name}/samples/digest?prompt_profile=voxtral").json() + assert route == {"digest": digest, "sample_count": 3} + # A profile without template arguments leaves the field out of the wire shape. + resp = client.get(f"/evals/asr_fleurs_nl_nl/datasets/{name}/samples?prompt_profile=lfm3") + assert "chat_template_kwargs" not in resp.json()["samples"][0] + + +def test_a_build_records_where_the_set_may_be_kept_and_the_text_moves_no_pin(client, tmp_path): + """metadata.json's `redistribution` is the builder's generic use policy (a deployment + records where it keeps the sets itself), and no fingerprint reads it: the same set + reworded loads and serves the same samples digest.""" + name, target = _build(tmp_path, EvalId.ASR_FLEURS_NL_NL, ["een", "twee"]) + meta = json.loads((target / "metadata.json").read_text()) + assert meta["redistribution"] == _asr_build.REDISTRIBUTION + assert "never commit it to a public repository" in meta["redistribution"] + assert "docs/dataset-licenses.md" not in meta["redistribution"] + digest = "/evals/asr_fleurs_nl_nl/datasets/{}/samples/digest?prompt_profile=voxtral" + before = client.get(digest.format(name)) + assert before.status_code == 200, before.text + # A copy under another name, since the loaders cache by directory. + reworded = target.parent / f"{name}-reworded" + shutil.copytree(target, reworded) + meta["redistribution"] = "internal use only: reworded" + (reworded / "metadata.json").write_text(json.dumps(meta, indent=2) + "\n") + after = client.get(digest.format(reworded.name)) + assert after.status_code == 200, after.text + assert after.json() == before.json() + + +def test_the_body_is_the_canonical_form_its_digest_hashes(client, tmp_path): + """A client can hash the stream's samples without re-encoding them.""" + import hashlib + + name, _ = _build(tmp_path, EvalId.ASR_FLEURS_NL_NL, ["een", "twee"]) + resp = client.get(f"/evals/asr_fleurs_nl_nl/datasets/{name}/samples?prompt_profile=gemma4") + raw = resp.content + assert raw.startswith(b'{"samples":[{') and raw.endswith(b"}]}") + lines = raw[len(b'{"samples":[') : -len(b"]}")].split(b',{"id":') + lines = [lines[0]] + [b'{"id":' + rest for rest in lines[1:]] + assert len(lines) == 2 + assert ( + resp.headers["x-samples-digest"] == "sha256:" + hashlib.sha256(b"".join(x + b"\n" for x in lines)).hexdigest() + ) + + +def test_one_pass_gives_every_profiles_digest(tmp_path): + """prompt_digests encodes each clip once per cut and hashes it into every + profile; each must be the digest of that profile's own stream, Gemma 4's 30 s + cut included (a 31 s clip).""" + rows = _source_rows(["een", "twee"], seconds=1.0) + rows.append({**rows[0], "audio": _wav(31.0), "raw_doc": {"id": "long"}}) + name = f"test-{uuid.uuid4().hex[:8]}" + _asr_build.build(asr_datasets.ASR_DATASETS[EvalId.ASR_FLEURS_NL_NL], tmp_path / "datasets", name, rows=iter(rows)) + target = tmp_path / "datasets" / "asr_fleurs_nl_nl" / name + digests = _asr_common.prompt_digests(EvalId.ASR_FLEURS_NL_NL, target) + assert set(digests) == set(asr_prompt_profiles.PROFILES) + for profile, (digest, count) in digests.items(): + streamed = list(_asr_common.iter_prompt_samples(EvalId.ASR_FLEURS_NL_NL, target, profile)) + assert (digest, count) == (samples_digest(streamed), 3), profile + # Each profile's samples are its own: six profiles, six digests. + assert len({d for d, _ in digests.values()}) == len(digests) + + +def test_a_cold_digest_learns_every_profile_at_once(client, tmp_path, monkeypatch): + name, _ = _build(tmp_path, EvalId.ASR_FLEURS_NL_NL, ["een"]) + calls = [] + real = dataset_catalog.asr_prompt_digests + + def spy(*args, **kw): + calls.append(args) + return real(*args, **kw) + + monkeypatch.setattr(dataset_catalog, "asr_prompt_digests", spy) + for profile in asr_prompt_profiles.PROFILES: + resp = client.get(f"/evals/asr_fleurs_nl_nl/datasets/{name}/samples/digest?prompt_profile={profile}") + assert resp.status_code == 200, (profile, resp.text) + assert len(calls) == 1 + + +@pytest.mark.parametrize("route", ["samples", "samples/digest"]) +@pytest.mark.parametrize( + "query, status", + [ + ("", 400), + ("prompt_profile=nope", 400), + ("prompt_profile=lfm3&offset=0&limit=1", 400), + ], +) +def test_an_asr_samples_request_needs_a_profile_and_is_not_paged(client, tmp_path, route, query, status): + name, _ = _build(tmp_path, EvalId.ASR_FLEURS_NL_NL, ["een"]) + resp = client.get(f"/evals/asr_fleurs_nl_nl/datasets/{name}/{route}?{query}") + assert resp.status_code == status, resp.text + + +def test_a_missing_dataset_is_a_404(client, tmp_path): + _build(tmp_path, EvalId.ASR_FLEURS_NL_NL, ["een"]) + for route in ("samples", "samples/digest"): + assert client.get(f"/evals/asr_fleurs_nl_nl/datasets/nope/{route}?prompt_profile=lfm3").status_code == 404 + + +def test_a_dataset_that_fails_its_checks_is_a_500_before_any_byte(client, tmp_path): + name, target = _build(tmp_path, EvalId.ASR_FLEURS_NL_NL, ["een", "twee"]) + _rewrite(target, lambda rows: [rows[1], rows[0]]) + resp = client.get(f"/evals/asr_fleurs_nl_nl/datasets/{name}/samples?prompt_profile=lfm3") + assert resp.status_code == 500 and "position" in resp.json()["detail"] + + +def test_a_stream_that_strays_from_its_digest_is_cut_off(client, tmp_path, monkeypatch): + """The header went out first, so a body that would not hash to it never + ends: the client is left with JSON it cannot parse.""" + name, _ = _build(tmp_path, EvalId.ASR_FLEURS_NL_NL, ["een", "twee"]) + key = ("asr_fleurs_nl_nl", name, "lfm3") + app_module._DIGESTS[key] = ("sha256:" + "0" * 64, 2) + with pytest.raises(RuntimeError, match="other samples than its digest"): + client.get(f"/evals/asr_fleurs_nl_nl/datasets/{name}/samples?prompt_profile=lfm3") + + +def test_a_44_1_khz_or_stereo_clip_is_served_at_16_khz_mono_to_every_profile(client, tmp_path): + rows = [ + {"audio": a, "audio_format": "wav", "reference": "r", "session": None, "chunk_index": None, "raw_doc": {}} + for a in (_wav(0.5, rate=44100), _wav(0.5, rate=16000, channels=2)) + ] + _asr_build.build( + asr_datasets.ASR_DATASETS[EvalId.ASR_ESB_MONSOON_EN_IN], tmp_path / "datasets", "rates", rows=iter(rows) + ) + url = "/evals/asr_esb_monsoon_en_in/datasets/rates/samples?prompt_profile={}" + body = client.get(url.format("voxtral")).json() + [part] = body["samples"][0]["messages"][0]["content"] + _, info = _decode(part["input_audio"]["data"]) + assert (info.samplerate, info.channels, info.frames, info.subtype) == (RATE, 1, 8000, "FLOAT") + for profile in asr_prompt_profiles.PROFILES: + resp = client.get(url.format(profile)) + assert resp.status_code == 200, (profile, resp.text) + parts = [ + p for m in resp.json()["samples"][1]["messages"] if isinstance(m["content"], list) for p in m["content"] + ] + [audio] = [p for p in parts if p["type"] == "input_audio"] + _, info = _decode(audio["input_audio"]["data"]) + # Two identical 16-bit channels average to 16-bit values: PCM again. + assert (info.samplerate, info.channels, info.subtype) == (RATE, 1, "PCM_16") + + +def test_a_text_eval_keeps_its_unpaged_shape(client, tmp_path): + prompts = [EvalSample(id=str(i), messages=[ChatMessage(role="user", content="Q?")]) for i in range(3)] + with patch("pipette_scores.api.app._load_prompts_cached", return_value=prompts): + whole = client.get("/evals/ifstruct/datasets/test/samples").json() + assert set(whole) == {"samples"} and len(whole["samples"]) == 3 + assert "chat_template_kwargs" not in whole["samples"][0] + + +# --------------------------------------------------------------------------- +# API: /score +# --------------------------------------------------------------------------- + + +def _score(client: TestClient, eval_id: EvalId, name: str, completions: list[dict[str, Any]], **extra: Any): + body = {"eval_id": eval_id.value, "dataset_name": name, "completions": completions, **extra} + return client.post("/score", json=body) + + +def test_score_reports_the_corpus_rate_and_each_samples_alignment(client, tmp_path): + name, target = _build(tmp_path, EvalId.ASR_FLEURS_NL_NL, ["een twee drie vier", "vijf zes"]) + ids = [s.id for s in _asr_common.load_eval_samples(target)] + resp = _score( + client, + EvalId.ASR_FLEURS_NL_NL, + name, + [ + # The profile keeps the text after the last "\nassistant\n". + {"id": ids[0], "completion": "system\nx\nassistant\nEen, twee, DRIE!", "stop_reason": "eos"}, + {"id": ids[1], "completion": "vijf zeven", "stop_reason": "truncated"}, + ], + prompt_profile="qwen25_omni", + ) + assert resp.status_code == 200, resp.text + out = resp.json() + ctx = out["context"] + # One deletion (vier) and one substitution (zeven) over six reference words. + assert ctx["error_unit"] == "word" and ctx["units_scored"] == 2 and ctx["samples_total"] == 2 + assert ctx["error_rate"] == pytest.approx(2 / 6) and ctx["result"] == pytest.approx(100 * 2 / 6) + assert ctx["stop_reasons"] == {"eos": 1, "truncated": 1} + assert not {"complete", "samples_missing", "samples_failed", "units_unscored"} & set(ctx) + first, second = out["scored_samples"] + assert first["measures"]["deletions"] == 1 and first["measures"]["hypothesis"] == "een twee drie" + assert second["measures"]["substitutions"] == 1 and not second["is_correct"] + + +def test_scored_samples_keep_their_text_and_an_empty_audio_part_in_its_place(client, tmp_path): + name, target = _build(tmp_path, EvalId.ASR_FLEURS_NL_NL, ["een"]) + [sample] = _asr_common.load_eval_samples(target) + [served] = client.get(f"/evals/asr_fleurs_nl_nl/datasets/{name}/samples?prompt_profile=qwen25_omni").json()[ + "samples" + ] + out = _score( + client, EvalId.ASR_FLEURS_NL_NL, name, [{"id": sample.id, "completion": "een"}], prompt_profile="qwen25_omni" + ).json() + [scored] = out["scored_samples"] + # The turns and parts as served, in order; the audio part keeps its format, not its data. + assert scored["messages"] == [ + {"role": "system", "content": "You are a speech recognition model."}, + { + "role": "user", + "content": [ + {"type": "input_audio", "input_audio": {"data": "", "format": "wav"}}, + {"type": "text", "text": "Transcribe the Dutch audio into text without any punctuation marks."}, + ], + }, + ] + audio = served["messages"][1]["content"][0] + assert audio["type"] == "input_audio" and audio["input_audio"]["data"] # served with its clip + assert [m["role"] for m in served["messages"]] == [m["role"] for m in scored["messages"]] + assert [[p["type"] for p in m["content"]] for m in served["messages"][1:]] == [ + [p["type"] for p in m["content"]] for m in scored["messages"][1:] + ] + + +def test_scored_samples_never_carry_an_audio_payload(client, tmp_path): + """Whatever the prompts hold, the response empties each audio part.""" + name, target = _build(tmp_path, EvalId.ASR_FLEURS_NL_NL, ["een"]) + [sample] = _asr_common.load_eval_samples(target) + prompt = EvalSample( + id=sample.id, + messages=[ChatMessage(role="user", content=[AUDIO, {"type": "text", "text": "Transcribe."}])], + ) + with patch("pipette_scores.api.app._load_prompts_cached", return_value=[prompt]): + resp = _score( + client, EvalId.ASR_FLEURS_NL_NL, name, [{"id": sample.id, "completion": "een"}], prompt_profile="lfm3" + ) + assert resp.status_code == 200, resp.text + assert resp.json()["scored_samples"][0]["messages"][0]["content"] == [ + {"type": "input_audio", "input_audio": {"data": "", "format": "wav"}}, + {"type": "text", "text": "Transcribe."}, + ] + assert prompt.messages[0].content[0] == AUDIO # the cached prompt is untouched + + +def test_score_shows_the_template_arguments_each_sample_was_sent_with(client, tmp_path): + name, target = _build(tmp_path, EvalId.ASR_FLEURS_NL_NL, ["een"]) + [sample] = _asr_common.load_eval_samples(target) + out = _score( + client, EvalId.ASR_FLEURS_NL_NL, name, [{"id": sample.id, "completion": "een"}], prompt_profile="voxtral" + ).json() + assert out["scored_samples"][0]["chat_template_kwargs"] == {"transcription": True, "language": "nl"} + out = _score( + client, EvalId.ASR_FLEURS_NL_NL, name, [{"id": sample.id, "completion": "een"}], prompt_profile="lfm3" + ).json() + assert "chat_template_kwargs" not in out["scored_samples"][0] + + +def test_score_drops_open_asr_references_that_normalize_away(client, tmp_path): + name, target = _build( + tmp_path, EvalId.ASR_ESB_VOXPOPULI_CLEANED_AA, ["hello world", "", "ignore time segment in scoring"] + ) + ids = [s.id for s in _asr_common.load_eval_samples(target)] + out = _score( + client, + EvalId.ASR_ESB_VOXPOPULI_CLEANED_AA, + name, + [{"id": i, "completion": "hello world"} for i in ids], + prompt_profile="lfm25_audio", + ).json() + ctx = out["context"] + assert (ctx["samples_excluded"], ctx["units_scored"], ctx["error_rate"]) == (2, 1, 0.0) + assert [s["measures"] for s in out["scored_samples"][1:]] == [{"excluded": True}, {"excluded": True}] + + +def _earnings22(tmp_path: pathlib.Path) -> tuple[str, list[str]]: + call = "Revenue grew five percent this quarter." + name, target = _build(tmp_path, EvalId.ASR_ESB_EARNINGS22_CLEANED_AA_CHUNKED, [call] * 3, session="call-1") + return name, [s.id for s in _asr_common.load_eval_samples(target)] + + +def test_an_earnings22_call_scores_once_and_its_measures_sum_to_the_totals(client, tmp_path): + name, ids = _earnings22(tmp_path) + out = _score( + client, + EvalId.ASR_ESB_EARNINGS22_CLEANED_AA_CHUNKED, + name, + # Out of order, one chunk blank: joined in chunk order, empties skipped. + [ + {"id": ids[2], "completion": "this quarter. "}, + {"id": ids[0], "completion": " Revenue grew"}, + {"id": ids[1], "completion": " 5% "}, + ], + prompt_profile="gemma4", + ).json() + ctx = out["context"] + assert (ctx["units_scored"], ctx["error_rate"], ctx["reference_units"]) == (1, 0.0, 5) + measures = {s["id"]: s["measures"] for s in out["scored_samples"]} + # The call's counts and texts once, on its first chunk; the others name the call only. + assert measures[ids[0]]["unit"] == "call-1" and measures[ids[0]]["reference_units"] == 5 + assert measures[ids[1]] == measures[ids[2]] == {"unit": "call-1"} + for key in ("hits", "substitutions", "deletions", "insertions", "reference_units"): + assert sum(m.get(key, 0) for m in measures.values()) == ctx[key] + + +@pytest.mark.parametrize( + "answers, names", + [ + # One sample unanswered. + ([("een twee", None), ("drie", None)], ["unanswered: {2}"]), + # One generation failed. + ([("een twee", None), ("", "failure"), ("vier", "eos")], ["failed: {1}"]), + # Both, as the old incomplete result had them. + ([("een twee", None), ("", "failure")], ["unanswered: {2}", "failed: {1}"]), + ], +) +def test_a_run_without_an_answer_to_every_sample_is_refused(client, tmp_path, answers, names): + name, target = _build(tmp_path, EvalId.ASR_FLEURS_NL_NL, ["een twee", "drie", "vier"]) + ids = [s.id for s in _asr_common.load_eval_samples(target)] + completions = [ + {"id": ids[i], "completion": text, **({"stop_reason": reason} if reason else {})} + for i, (text, reason) in enumerate(answers) + ] + with patch("pipette_scores.api.app.scoring.score") as score: + resp = _score(client, EvalId.ASR_FLEURS_NL_NL, name, completions, prompt_profile="lfm3") + assert resp.status_code == 400, resp.text + detail = resp.json()["detail"] + assert detail.startswith("an ASR eval is scored only with an answer to every sample; ") + for label in names: + assert label.format(*ids) in detail + score.assert_not_called() + # Answered in full, the same run scores. + complete = [{**c, "completion": c["completion"] or "drie", "stop_reason": "eos"} for c in completions] + complete += [{"id": i, "completion": "vier"} for i in ids[len(complete) :]] + ok = _score(client, EvalId.ASR_FLEURS_NL_NL, name, complete, prompt_profile="lfm3") + assert ok.status_code == 200, ok.text + assert ok.json()["context"]["error_rate"] == pytest.approx(0.0) + + +def test_the_scorer_itself_refuses_a_failed_or_missing_answer(tmp_path): + """For any direct caller: a result is an answer to every sample, or none.""" + _, target = _build(tmp_path, EvalId.ASR_FLEURS_NL_NL, ["een", "twee"]) + samples = _asr_common.load_eval_samples(target) + answered = [SampleCompletion(id=s.id, completion=s.reference) for s in samples] + failed = [answered[0], SampleCompletion(id=samples[1].id, completion="", stop_reason="failure")] + with pytest.raises(ValueError, match=f"failed: {samples[1].id}"): + scoring.score(EvalId.ASR_FLEURS_NL_NL, failed, samples, prompt_profile="lfm3") + with pytest.raises(ValueError, match=f"unanswered: {samples[1].id}"): + scoring.score(EvalId.ASR_FLEURS_NL_NL, answered[:1], samples, prompt_profile="lfm3") + _, context = scoring.score(EvalId.ASR_FLEURS_NL_NL, answered, samples, prompt_profile="lfm3") + assert context["error_rate"] == 0.0 + + +def test_an_excluded_sample_still_needs_its_answer(client, tmp_path): + name, target = _build(tmp_path, EvalId.ASR_ESB_VOXPOPULI_CLEANED_AA, ["hello world", ""]) + ids = [s.id for s in _asr_common.load_eval_samples(target)] + resp = _score( + client, + EvalId.ASR_ESB_VOXPOPULI_CLEANED_AA, + name, + [{"id": ids[0], "completion": "hello world"}], + prompt_profile="lfm25_audio", + ) + assert resp.status_code == 400 and f"unanswered: {ids[1]}" in resp.json()["detail"] + + +def test_an_earnings22_call_missing_a_chunk_is_refused(client, tmp_path): + name, ids = _earnings22(tmp_path) + resp = _score( + client, + EvalId.ASR_ESB_EARNINGS22_CLEANED_AA_CHUNKED, + name, + [{"id": ids[0], "completion": "Revenue grew"}, {"id": ids[2], "completion": "this quarter."}], + prompt_profile="gemma4", + ) + assert resp.status_code == 400 and f"unanswered: {ids[1]}" in resp.json()["detail"] + + +def test_ids_twice_or_unknown_are_refused_as_before(client, tmp_path): + name, target = _build(tmp_path, EvalId.ASR_FLEURS_NL_NL, ["een"]) + [sample] = _asr_common.load_eval_samples(target) + twice = [{"id": sample.id, "completion": "een"}] * 2 + resp = _score(client, EvalId.ASR_FLEURS_NL_NL, name, twice, prompt_profile="lfm3") + assert resp.status_code == 400 and "duplicate completion ids" in resp.json()["detail"] + unknown = [{"id": sample.id, "completion": "een"}, {"id": "nope", "completion": "x"}] + resp = _score(client, EvalId.ASR_FLEURS_NL_NL, name, unknown, prompt_profile="lfm3") + assert resp.status_code == 400 and "not in dataset: nope" in resp.json()["detail"] + + +def test_score_refuses_thinking_and_a_missing_profile(client, tmp_path): + name, target = _build(tmp_path, EvalId.ASR_FLEURS_NL_NL, ["een"]) + [sample] = _asr_common.load_eval_samples(target) + completions = [{"id": sample.id, "completion": "een"}] + assert _score(client, EvalId.ASR_FLEURS_NL_NL, name, completions).status_code == 400 + resp = _score(client, EvalId.ASR_FLEURS_NL_NL, name, completions, prompt_profile="gemma4", enable_thinking=True) + assert resp.status_code == 400 and "thinking" in resp.json()["detail"] + ok = _score(client, EvalId.ASR_FLEURS_NL_NL, name, completions, prompt_profile="gemma4", enable_thinking=False) + assert ok.status_code == 200 + + +def test_score_checks_the_samples_the_run_was_given(client, tmp_path): + name, target = _build(tmp_path, EvalId.ASR_FLEURS_NL_NL, ["een"]) + [sample] = _asr_common.load_eval_samples(target) + completions = [{"id": sample.id, "completion": "een"}] + served = client.get(f"/evals/asr_fleurs_nl_nl/datasets/{name}/samples/digest?prompt_profile=gemma4").json() + ok = _score( + client, EvalId.ASR_FLEURS_NL_NL, name, completions, prompt_profile="gemma4", samples_digest=served["digest"] + ) + assert ok.status_code == 200, ok.text + # Another profile's samples are other prompts. + other = client.get(f"/evals/asr_fleurs_nl_nl/datasets/{name}/samples/digest?prompt_profile=lfm3").json() + moved = _score( + client, EvalId.ASR_FLEURS_NL_NL, name, completions, prompt_profile="gemma4", samples_digest=other["digest"] + ) + assert moved.status_code == 409 and served["digest"] in moved.json()["detail"] + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + + +def _cli(monkeypatch, capsys, *argv: str) -> str: + monkeypatch.setattr("sys.argv", ["pipette-scores", *argv]) + cli.main() + return capsys.readouterr().out + + +def test_the_cli_scores_an_asr_run_with_its_profile(tmp_path, monkeypatch, capsys): + name, target = _build(tmp_path, EvalId.ASR_FLEURS_NL_NL, ["een twee", "drie"]) + ids = [s.id for s in _asr_common.load_eval_samples(target)] + completions = tmp_path / "completions.jsonl" + completions.write_text("".join(json.dumps({"id": i, "completion": "een twee<|im_end|>"}) + "\n" for i in ids)) + root = ["--data-dir", str(tmp_path)] + out = _cli(monkeypatch, capsys, *root, "score", "--eval", "asr_fleurs_nl_nl", "--dataset", name, + "--prompt-profile", "lfm3", "-f", str(completions), "--no-samples") # fmt: skip + # een twee: right; drie -> "een twee": one substitution, one insertion, over 3 words. + assert "WER" in out and f"{100 * 2 / 3:.4f}%" in out and "ACCURACY" not in out + with pytest.raises(SystemExit, match="--prompt-profile"): + _cli( + monkeypatch, capsys, *root, "score", "--eval", "asr_fleurs_nl_nl", "--dataset", name, "-f", str(completions) + ) + completions.write_text(json.dumps({"id": ids[0], "completion": "een twee"}) + "\n") + with pytest.raises(SystemExit, match=f"answer to every sample; unanswered: {ids[1]}"): + _cli(monkeypatch, capsys, *root, "score", "--eval", "asr_fleurs_nl_nl", "--dataset", name, + "--prompt-profile", "lfm3", "-f", str(completions)) # fmt: skip + completions.write_text(json.dumps({"id": "nope", "completion": "x"}) + "\n") + with pytest.raises(SystemExit, match="not in"): + _cli(monkeypatch, capsys, *root, "score", "--eval", "asr_fleurs_nl_nl", "--dataset", name, + "--prompt-profile", "lfm3", "-f", str(completions)) # fmt: skip + out = _cli(monkeypatch, capsys, *root, "prompts", "--eval", "asr_fleurs_nl_nl", "--dataset", name, + "--prompt-profile", "qwen25_omni") # fmt: skip + first = json.loads(out.splitlines()[0]) + assert first["id"] == ids[0] and first["messages"][0]["content"] == "You are a speech recognition model." diff --git a/packages/pipette-scores/tests/test_asr_parts.py b/packages/pipette-scores/tests/test_asr_parts.py new file mode 100644 index 0000000..7510bfd --- /dev/null +++ b/packages/pipette-scores/tests/test_asr_parts.py @@ -0,0 +1,214 @@ +"""ASR sets split into ordered git-LFS parts: read as one table, the fingerprint unchanged.""" + +from __future__ import annotations + +import importlib.util +import io +import itertools +import json +import pathlib +import uuid +from typing import Any + +import numpy as np +import pyarrow.parquet as pq +import pytest +import soundfile as sf + +from pipette_scores.dataset_catalog import _asr_build, _asr_common, asr_datasets +from pipette_scores.types import EvalId + +RATE = 16000 +EVAL = EvalId.ASR_FLEURS_NL_NL +_SCRIPTS = pathlib.Path(__file__).resolve().parents[1] / "scripts" + + +def _script(name: str) -> Any: + spec = importlib.util.spec_from_file_location(name, _SCRIPTS / f"{name}.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _wav(seconds: float, seed: int) -> bytes: + """Noise, so the parquet's size is its audio (as a real set's is), not its footer.""" + noise = 0.1 * np.random.default_rng(seed).standard_normal(int(seconds * RATE)).astype(np.float32) + buffer = io.BytesIO() + sf.write(buffer, noise, RATE, format="WAV", subtype="PCM_16") + return buffer.getvalue() + + +def _build(tmp_path: pathlib.Path, n: int) -> pathlib.Path: + """A fake built set of ``n`` clips under tmp_path/datasets, uniquely named + (the loaders cache per directory).""" + rows = [ + { + "audio": _wav(0.05 + 0.001 * i, i), + "audio_format": "wav", + "reference": f"w{i}", + "session": None, + "chunk_index": None, + "raw_doc": {"id": f"utt-{i}"}, + } + for i in range(n) + ] + name = f"test-{uuid.uuid4().hex[:8]}" + _asr_build.build(asr_datasets.ASR_DATASETS[EVAL], tmp_path / "datasets", name, rows=iter(rows)) + return tmp_path / "datasets" / EVAL.value / name + + +def _split(tmp_path: pathlib.Path, n: int = 3 * _asr_build.ROW_GROUP_SIZE + 5) -> tuple[pathlib.Path, pathlib.Path]: + """A built set (4 row groups by default) and a copy split into three or more + parts beside it, under another version name, its metadata.json copied as is.""" + single = _build(tmp_path, n) + split = single.with_name(f"{single.name}-split") + bound = (single / "test.parquet").stat().st_size // 2 + files = _asr_common.split_parts(single / "test.parquet", split, bound) + (split / "metadata.json").write_bytes((single / "metadata.json").read_bytes()) + assert len(files) >= 3 and all(f.stat().st_size <= bound for f in files) + return single, split + + +def _clips(target: pathlib.Path, start: int, stop: int) -> list[tuple[str, str]]: + """Rows ``start`` to ``stop`` of the served stream, as (id, clip).""" + served = itertools.islice(_asr_common.iter_prompt_samples(EVAL, target, "lfm3"), start, stop) + return [(s.id, s.messages[-1].content[-1]["input_audio"]["data"]) for s in served] + + +def test_a_split_set_is_the_same_rows_with_the_same_fingerprint(tmp_path): + single, split = _split(tmp_path) + files = _asr_common.parquet_files(split) + count = len(files) + assert [f.name for f in files] == [f"test-{i:05d}-of-{count:05d}.parquet" for i in range(count)] + # Cut between row groups, in order: the parts' groups are the build's. + metas = [pq.ParquetFile(f).metadata for f in files] + assert [m.row_group(i).num_rows for m in metas for i in range(m.num_row_groups)] == [64, 64, 64, 5] + rows = _asr_common.verified_rows(split) # the server's checks, content_sha256 included + assert rows == _asr_common.verified_rows(single) + assert _asr_common.content_sha256(rows) == json.loads((split / "metadata.json").read_text())["content_sha256"] + assert _asr_common.load_eval_samples(split) == _asr_common.load_eval_samples(single) + # A stretch across a cut serves the same clips, and so does the whole set. + first = metas[0].num_rows + assert _clips(split, first - 2, first + 3) == _clips(single, first - 2, first + 3) + assert _clips(split, 0, 256) == _clips(single, 0, 256) + + +def test_a_set_under_the_bound_is_copied_whole(tmp_path): + single = _build(tmp_path, 2) + [file] = _asr_common.split_parts(single / "test.parquet", tmp_path / "copy", 10**9) + assert file.name == "test.parquet" + assert pq.read_table(file).to_pylist() == pq.read_table(single / "test.parquet").to_pylist() + with pytest.raises(ValueError, match="another directory"): + _asr_common.split_parts(single / "test.parquet", single, 10**9) + + +def test_a_row_group_over_the_bound_cannot_be_split(tmp_path): + single = _build(tmp_path, 3) + with pytest.raises(ValueError, match="cannot be cut"): + _asr_common.split_parts(single / "test.parquet", tmp_path / "out", 1000) + assert not (tmp_path / "out").exists() or not any((tmp_path / "out").iterdir()) + + +def test_a_missing_part_is_refused_even_renumbered(tmp_path): + """The negative control: the layout names a missing part, and a set + renumbered to hide one keeps a valid layout but not the fingerprint.""" + _, split = _split(tmp_path) + files = _asr_common.parquet_files(split) + files[-1].unlink() + with pytest.raises(_asr_common.AsrDatasetError, match="for every i from 0"): + _asr_common.parquet_files(split) + for i, f in enumerate(files[:-1]): + f.rename(split / _asr_common.part_name(i, len(files) - 1)) + with pytest.raises(_asr_common.AsrDatasetError, match=r"content_sha256 \w+ != \w+ from metadata.json"): + _asr_common.verified_rows(split) + + +def test_parts_out_of_order_are_refused(tmp_path): + """The rows' digest is order-free, so swapped parts keep it; their ids do not.""" + _, split = _split(tmp_path) + a, b = _asr_common.parquet_files(split)[:2] + a.rename(split / "swap") + b.rename(a) + (split / "swap").rename(b) + with pytest.raises(_asr_common.AsrDatasetError, match="must be its position"): + _asr_common.verified_rows(split) + + +@pytest.mark.parametrize( + "stray, match", + [ + ("test.parquet", "keep one layout"), + ("test-extra.parquet", r"not a test--of-\.parquet part"), + ("test-00000-of-00009.parquet", "for every i from 0"), + ], +) +def test_a_mixed_or_stray_layout_is_refused(tmp_path, stray, match): + single, split = _split(tmp_path) + (split / stray).write_bytes((single / "test.parquet").read_bytes()) + with pytest.raises(_asr_common.AsrDatasetError, match=match): + _asr_common.verified_rows(split) + + +def test_a_dataset_with_no_parquet_is_not_found(tmp_path): + single = _build(tmp_path, 1) + (single / "test.parquet").unlink() + with pytest.raises(FileNotFoundError): + _asr_common.parquet_files(single) + + +def test_verify_checks_a_split_set_and_fails_a_missing_part(tmp_path, monkeypatch, capsys): + _, split = _split(tmp_path) + verify = _script("verify_asr_datasets") + monkeypatch.setattr("sys.argv", ["verify", "--root", str(tmp_path / "datasets"), "--version", split.name]) + assert verify.main() == 0 + out = capsys.readouterr().out + assert f"{EVAL.value}: ok (197 clips in {len(_asr_common.parquet_files(split))} parts, 0 audio mismatches" in out + _asr_common.parquet_files(split)[1].unlink() + assert verify.main() == 1 + assert f"{EVAL.value}: FAILED" in capsys.readouterr().out + + +def test_the_split_script_lays_out_a_tree_and_checks_it(tmp_path, monkeypatch, capsys): + single = _build(tmp_path, 2 * _asr_build.ROW_GROUP_SIZE) + bound = (single / "test.parquet").stat().st_size * 3 // 4 + out = tmp_path / "lfs" + argv = ["split", "--src", str(tmp_path / "datasets"), "--out", str(out), "--version", single.name] + monkeypatch.setattr("sys.argv", [*argv, "--max-part-bytes", str(bound), EVAL.value]) + assert _script("split_asr_dataset").main() == 0 + target = out / EVAL.value / single.name + assert sorted(p.name for p in target.iterdir()) == [ + "metadata.json", + "test-00000-of-00002.parquet", + "test-00001-of-00002.parquet", + ] + assert (target / "metadata.json").read_bytes() == (single / "metadata.json").read_bytes() + assert f"{EVAL.value}: 128 clips in 2 file(s)" in capsys.readouterr().out + with pytest.raises(SystemExit, match="not empty"): + _script("split_asr_dataset").main() + + +def test_neither_scoring_nor_the_stream_holds_a_sets_audio(tmp_path, monkeypatch): + """The integrity load (what scoring uses) reads no audio column, and the + samples stream reads one row group at a time as it goes, so a 9 GB set is + never in memory.""" + _, split = _split(tmp_path) + tables: list[Any] = [] + read_table = pq.read_table + + def spy_table(path, columns=None, **kw): + tables.append(columns) + return read_table(path, columns=columns, **kw) + + monkeypatch.setattr(pq, "read_table", spy_table) + _asr_common.load_eval_samples(split) + assert tables and all(c is not None and "audio" not in c for c in tables) + groups: list[tuple[Any, int]] = [] + read_row_group = pq.ParquetFile.read_row_group + + def spy_group(self, i, columns=None, **kw): + groups.append((columns, i)) + return read_row_group(self, i, columns=columns, **kw) + + monkeypatch.setattr(pq.ParquetFile, "read_row_group", spy_group) + served = list(itertools.islice(_asr_common.iter_prompt_samples(EVAL, split, "lfm3"), 3)) + assert len(served) == 3 and len(groups) == 1 and groups[0][0] == ["audio"] diff --git a/packages/pipette-scores/tests/test_asr_public_reference.py b/packages/pipette-scores/tests/test_asr_public_reference.py new file mode 100644 index 0000000..e7976f3 --- /dev/null +++ b/packages/pipette-scores/tests/test_asr_public_reference.py @@ -0,0 +1,470 @@ +"""ASR scoring and audio against the PUBLIC reference implementations. + +- Open ASR Leaderboard (English sets): the leaderboard's own code + (huggingface/open_asr_leaderboard @ 62a457c5: ``data_utils`` normalizer and + ``prepare_data`` filter, ``eval_utils`` chunk merge and kaldialign alignment) + must give exactly our counts and corpus rate, on every set. +- FLEURS: the same code as the leaderboard's multilingual scoring runs it + (``eval_utils._normalize`` with the language, ``data_utils.ml_normalizer`` + and its num2words step, ``normalize_compound_pairs``, kaldialign) must give + exactly our normalized texts, counts and corpus rate, and our vendored + excerpts must be its source text. +- Audio: ``datasets.Audio(sampling_rate=16000)`` (datasets 2.19.0, librosa, + soundfile) must give exactly our 16 kHz mono samples. + +Two layers. The fixture tests always run: they compare us with the reference +answers recorded on the synthetic corpus (fixtures/asr_public_reference.json; +for the audio, each clip's shape and fixed projections of its samples). The +live tests need the reference environment +(``scripts/setup_asr_reference_env.sh DIR``, then ``PIPETTE_ASR_REFERENCE_ENV=DIR``) +and run the reference code itself on the synthetic corpus plus, with +``PIPETTE_SCORES_DATA_DIR`` set, every reference of each built dataset there; a +root that holds none fails the run. Every answer is paired with its input by +name, and their numbers must agree. Negative controls show the checks can +fail: what a near-miss method (no compound merge on the English sets; on FLEURS +no number words, or no compound-pair step; soxr's very-high quality, no length +rule) gives differs from the reference. +""" + +from __future__ import annotations + +import base64 +import io +import json +import math +import os +from typing import Any + +import numpy as np +import pytest +from asr_public_reference import ( + FIXTURE, + audio_summary, + by_name, + corpus_digest, + data_roots, + digest, + fleurs_transcripts, + real_corpus, + reference_array, + reference_env, + run_reference, + synthetic_calls, + synthetic_clips, + synthetic_english, + synthetic_fixture, + synthetic_fleurs, + synthetic_inputs, +) + +from pipette_scores.dataset_catalog import _asr_audio, asr_datasets +from pipette_scores.scoring.asr.align import Aligner +from pipette_scores.scoring.asr.normalize import OPEN_ASR_LANGUAGE_CODES, NormalizerRecipe, normalize +from pipette_scores.scoring.asr.wer import AsrReference, AsrScore, AsrScoringSpec, score_asr +from pipette_scores.types import EvalId + +OPEN_ASR = AsrScoringSpec(NormalizerRecipe.OPEN_ASR_ENGLISH, Aligner.KALDIALIGN_COMPOUND, open_asr_exclusions=True) + + +def _score_open_asr(manifest: list[dict[str, Any]], spec: AsrScoringSpec = OPEN_ASR) -> AsrScore: + references = [ + AsrReference( + id=str(i), + reference=row["text"], + language="eng", + session=row.get("parent_id"), + chunk_index=row.get("chunk_index"), + ) + for i, row in enumerate(manifest) + ] + return score_asr(spec, references, {str(i): row["pred_text"] for i, row in enumerate(manifest)}) + + +def _fleurs_spec(eval_id: str) -> AsrScoringSpec: + dataset = asr_datasets.ASR_DATASETS[EvalId(eval_id)] + return AsrScoringSpec(dataset.normalizer, dataset.aligner, dataset.open_asr_exclusions) + + +def _score_fleurs(dataset: dict[str, Any]) -> AsrScore: + references = [AsrReference(str(i), r, dataset["language"]) for i, (r, _) in enumerate(dataset["pairs"])] + hypotheses = {str(i): h for i, (_, h) in enumerate(dataset["pairs"])} + return score_asr(_fleurs_spec(dataset["eval_id"]), references, hypotheses) + + +def _assert_open_asr_equal(ours: AsrScore, theirs: dict[str, Any]) -> None: + totals = ours.totals + assert len(ours.units) == theirs["units"] + assert { + "ins": totals.insertions, + "del": totals.deletions, + "sub": totals.substitutions, + "total": totals.errors, + "ref_len": totals.reference_units, + } == {k: theirs["totals"][k] for k in ("ins", "del", "sub", "total", "ref_len")} + rate = ours.corpus_error_rate + assert rate == theirs["totals"]["err_rate"] + assert round(100 * rate, 2) == theirs["wer"] + # Unit by unit, in the leaderboard's order (rows in order; calls sorted by id). + assert [ + [u.counts.insertions, u.counts.deletions, u.counts.substitutions, u.counts.reference_units] for u in ours.units + ] == theirs["pairs"] + + +def _errors_without_merge(manifest: list[dict[str, Any]]) -> int: + """The leaderboard's alignment of the rows (no chunked calls) without + merge_compounds: a near miss.""" + from kaldialign import edit_distance + + errors = 0 + for row in manifest: + reference = normalize(NormalizerRecipe.OPEN_ASR_ENGLISH, row["text"], "eng") + if reference.strip() in ("", "ignore time segment in scoring"): + continue + hypothesis = normalize(NormalizerRecipe.OPEN_ASR_ENGLISH, row["pred_text"], "eng") + errors += edit_distance(reference.split(), hypothesis.split(), merge_compounds=False)["total"] + return errors + + +def _fleurs_totals(ours: AsrScore) -> dict[str, Any]: + totals = ours.totals + return { + "ins": totals.insertions, + "del": totals.deletions, + "sub": totals.substitutions, + "total": totals.errors, + "ref_len": totals.reference_units, + "err_rate": ours.corpus_error_rate, + } + + +def _assert_fleurs_equal(dataset: dict[str, Any], theirs: dict[str, Any]) -> None: + recipe = asr_datasets.ASR_DATASETS[EvalId(dataset["eval_id"])].normalizer + language = dataset["language"] + assert [normalize(recipe, r, language) for r, _ in dataset["pairs"]] == theirs["references"] + assert [normalize(recipe, h, language) for _, h in dataset["pairs"]] == theirs["hypotheses"] + ours = _score_fleurs(dataset) + assert _fleurs_totals(ours) == theirs["totals"] + # Pair by pair, in order. + assert [ + [u.counts.insertions, u.counts.deletions, u.counts.substitutions, u.counts.reference_units] for u in ours.units + ] == theirs["pairs"] + + +def _fleurs_errors(dataset: dict[str, Any], spec: AsrScoringSpec) -> int: + references = [AsrReference(str(i), r, dataset["language"]) for i, (r, _) in enumerate(dataset["pairs"])] + return score_asr(spec, references, {str(i): h for i, (_, h) in enumerate(dataset["pairs"])}).totals.errors + + +# --------------------------------------------------------------------------- +# Always: against the reference answers recorded on the synthetic corpus +# --------------------------------------------------------------------------- + + +@pytest.fixture(scope="module") +def fixture() -> dict[str, Any]: + recorded = json.loads(FIXTURE.read_text(encoding="utf-8")) + assert recorded["corpus_sha256"] == corpus_digest(), "the synthetic corpus changed: regenerate the fixture" + return recorded + + +def _recorded_totals(ours: AsrScore, recorded: dict[str, Any]) -> None: + totals = ours.totals + assert (len(ours.units), totals.insertions, totals.deletions, totals.substitutions, totals.reference_units) == ( + recorded["units"], + recorded["totals"]["ins"], + recorded["totals"]["del"], + recorded["totals"]["sub"], + recorded["totals"]["ref_len"], + ) + assert ours.corpus_error_rate == recorded["totals"]["err_rate"] + assert round(100 * ours.corpus_error_rate, 2) == recorded["wer"] + + +def test_the_english_corpus_scores_as_the_leaderboard_scored_it(fixture): + ours = _score_open_asr([{"text": r, "pred_text": h} for r, h in synthetic_english()]) + _recorded_totals(ours, fixture["open_asr_english"]) + assert len(ours.units) == fixture["open_asr_english"]["kept"] + + +def test_the_chunked_calls_score_as_the_leaderboard_merged_them(fixture): + _recorded_totals(_score_open_asr(synthetic_calls()), fixture["open_asr_calls"]) + + +@pytest.mark.parametrize("dataset", synthetic_fleurs(), ids=lambda d: d["eval_id"]) +def test_each_fleurs_locale_normalizes_and_scores_as_the_leaderboards_multilingual_code_did(dataset, fixture): + recorded = fixture["fleurs"][dataset["eval_id"]] + recipe = asr_datasets.ASR_DATASETS[EvalId(dataset["eval_id"])].normalizer + language = dataset["language"] + assert digest([normalize(recipe, r, language) for r, _ in dataset["pairs"]]) == recorded["references_sha256"] + assert digest([normalize(recipe, h, language) for _, h in dataset["pairs"]]) == recorded["hypotheses_sha256"] + assert _fleurs_totals(_score_fleurs(dataset)) == recorded["totals"] + + +def test_the_negative_controls_miss_the_recorded_answers(fixture, monkeypatch): + """Near-miss methods do not reproduce the leaderboard: on its English sets + without merge_compounds; on FLEURS without the number words (every non-English + locale), or without the compound-pair step (any locale's corpus, whose + hypotheses cut neighbouring words at other points).""" + manifest = [{"text": r, "pred_text": h} for r, h in synthetic_english()] + assert _errors_without_merge(manifest) != fixture["open_asr_english"]["totals"]["total"] + + fleurs = {d["eval_id"]: d for d in synthetic_fleurs()} + no_pairs = AsrScoringSpec(NormalizerRecipe.OPEN_ASR_MULTILINGUAL, Aligner.KALDIALIGN_COMPOUND) + for eval_id, dataset in fleurs.items(): + assert _fleurs_errors(dataset, no_pairs) != fixture["fleurs"][eval_id]["totals"]["total"], eval_id + + from pipette_scores.scoring.asr.open_asr_normalizer.multilingual import MultilingualNormalizer + + monkeypatch.setattr(MultilingualNormalizer, "_normalize_numbers", lambda self, text, lang: text) + for eval_id, dataset in fleurs.items(): + if eval_id == "asr_fleurs_en_us": + continue # the English normalizer has its own number step + assert _fleurs_totals(_score_fleurs(dataset)) != fixture["fleurs"][eval_id]["totals"], eval_id + + +# soxr's "high" quality (librosa's soxr_hq) runs libsoxr's single-precision +# engine, whose SIMD (SSE on x86_64, NEON on arm64) and scalar code round +# differently on each CPU architecture: the same clip, soxr 1.1.0 and numpy give +# float32 samples that differ by up to ~1.2e-7 (4 float32 ulps at these clips' +# peaks) between macOS arm64, Linux x86_64 and Linux aarch64, and the fixture's +# projections by up to ~6e-6. Both are correct; exact equality, on the platform +# the reference runs on, is the live layer's. 1e-4 leaves 16x that, and is +# >200x below the nearest near miss: soxr "very high" moves a projection by +# 2.2e-2 at least, and a wrong length or dtype fails the shape check. +_PROJECTION_TOLERANCE = 1e-4 + + +def _matches_recorded(array: np.ndarray, recorded: dict[str, Any]) -> bool: + ours = audio_summary(array) + return ( + ours["dtype"] == recorded["dtype"] + and ours["shape"] == recorded["shape"] + and all( + math.isclose(a, b, rel_tol=0, abs_tol=_PROJECTION_TOLERANCE) + for a, b in zip(ours["projections"], recorded["projections"], strict=True) + ) + ) + + +def _near_miss_resample(clip: bytes, *, quality: str = "soxr_hq", length_rule: bool = True) -> np.ndarray: + """`resample_16k_mono` with one step changed: soxr's quality, or soxr's own + output length in place of librosa's fix_length to ceil(n * 16000 / sr).""" + import soundfile as sf + import soxr + + array, rate = sf.read(io.BytesIO(clip)) + array = array.T + if array.ndim > 1: + array = np.mean(array, axis=tuple(range(array.ndim - 1))) + if rate == 16000: + return array + out = soxr.resample(array, rate, 16000, quality=quality) + if length_rule: + size = math.ceil(array.shape[-1] * 16000 / rate) + out = out[:size] if len(out) >= size else np.pad(out, (0, size - len(out))) + return out + + +def _rate(clip: bytes) -> int: + import soundfile as sf + + return sf.info(io.BytesIO(clip)).samplerate + + +def test_every_synthetic_clip_resamples_as_datasets_audio_16k_did(fixture): + recorded = fixture["audio"] + clips = synthetic_inputs()["clips"] + assert list(recorded) == list(clips) and len(clips) == len(synthetic_clips()) == 13 + for name, clip in clips.items(): + assert _matches_recorded(_asr_audio.resample_16k_mono(clip), recorded[name]), name + + +def test_the_audio_negative_controls_miss_the_recorded_samples(fixture): + """soxr at "very high" quality misses every resampled clip; soxr's own output + length (no librosa length rule) misses the 22,051-frame one.""" + recorded = fixture["audio"] + clips = synthetic_inputs()["clips"] + resampled = [name for name, clip in clips.items() if _rate(clip) != 16000] + assert len(resampled) == 11 + for name in resampled: + assert not _matches_recorded(_near_miss_resample(clips[name], quality="soxr_vhq"), recorded[name]), name + assert _matches_recorded(_near_miss_resample(clips[name]), recorded[name]), name + missed = [ + n for n in resampled if not _matches_recorded(_near_miss_resample(clips[n], length_rule=False), recorded[n]) + ] + assert missed == ["synthetic:4"] + + +# --------------------------------------------------------------------------- +# Live: the reference code itself (PIPETTE_ASR_REFERENCE_ENV) +# --------------------------------------------------------------------------- + +live = pytest.mark.skipif( + reference_env() is None, + reason="set PIPETTE_ASR_REFERENCE_ENV (scripts/setup_asr_reference_env.sh) to run the public references", +) + + +def test_a_configured_data_dir_must_hold_built_asr_datasets(tmp_path): + """PIPETTE_SCORES_DATA_DIR set to a missing or empty directory, or to one + with a stray asr_* directory, fails; unset checks no real data; a root with + a built dataset (directly, or under datasets/) passes.""" + assert data_roots(None) == [] and data_roots("") == [] + with pytest.raises(ValueError, match="is not a directory"): + data_roots(str(tmp_path / "missing")) + with pytest.raises(ValueError, match="holds no built ASR dataset"): + data_roots(str(tmp_path)) + (tmp_path / "datasets" / "asr_esb_nonexistent" / "2026.10.1").mkdir(parents=True) + with pytest.raises(ValueError, match="no ASR eval of the catalog"): + data_roots(str(tmp_path)) + (tmp_path / "datasets" / "asr_esb_nonexistent" / "2026.10.1").rmdir() + (tmp_path / "datasets" / "asr_esb_nonexistent").rmdir() + built = tmp_path / "datasets" / "asr_fleurs_nl_nl" / "2026.10.1" + built.mkdir(parents=True) + with pytest.raises(ValueError, match="holds no built version"): + data_roots(str(tmp_path)) + (built / "metadata.json").write_text("{}", encoding="utf-8") + assert data_roots(str(tmp_path)) == [tmp_path / "datasets"] + assert data_roots(os.pathsep.join([str(tmp_path / "datasets"), str(tmp_path)])) == [tmp_path / "datasets"] * 2 + with pytest.raises(ValueError, match="is not a directory"): + data_roots(os.pathsep.join([str(tmp_path), str(tmp_path / "missing")])) + + +def test_the_answers_must_be_exactly_the_inputs_by_name(): + answers = [{"name": "a", "x": 1}, {"name": "b", "x": 2}] + assert by_name(["a", "b"], answers, "k") == {"a": answers[0], "b": answers[1]} + with pytest.raises(AssertionError, match="1 answers for 2 inputs"): + by_name(["a", "b"], answers[:1], "k") + with pytest.raises(AssertionError, match="3 answers for 2 inputs"): + by_name(["a", "b"], [*answers, answers[0]], "k") + with pytest.raises(AssertionError, match="answers are for"): + by_name(["a", "b"], answers[::-1], "k") + with pytest.raises(AssertionError, match="answers are for"): + by_name(["a", "c"], answers, "k") + + +@pytest.fixture(scope="module") +def answers() -> dict[str, Any]: + env = reference_env() + assert env is not None + inputs = synthetic_inputs() + open_asr, fleurs, clips = inputs["open_asr"], inputs["fleurs"], inputs["clips"] + fleurs.update(fleurs_transcripts()) + roots = data_roots(os.environ.get("PIPETTE_SCORES_DATA_DIR")) + for root in roots: + real = real_corpus(root) + assert real["open_asr"] or real["fleurs"], root + assert real["clips"], root + for kind, named in (("open_asr", open_asr), ("fleurs", fleurs), ("clips", clips)): + assert not set(real[kind]) & set(named), (root, kind) + named.update(real[kind]) + answer = run_reference(env, open_asr, fleurs, clips) + return {"answer": answer, "open_asr": open_asr, "fleurs": fleurs, "clips": clips, "roots": roots} + + +def _pairs(inputs: dict[str, Any], answers: dict[str, Any]) -> list[tuple[str, Any, Any]]: + assert list(inputs) == list(answers) and inputs + return [(name, inputs[name], answers[name]) for name in inputs] + + +@live +def test_live_the_recorded_fixture_is_the_references_answer(answers): + recorded = json.loads(FIXTURE.read_text(encoding="utf-8")) + expected = synthetic_fixture(answers["answer"]) + assert {k: v for k, v in recorded.items() if k not in ("note", "audio")} == { + k: v for k, v in expected.items() if k not in ("note", "audio") + } + # The audio projections, to rounding (exact equality of the samples is the + # resample test below). + assert list(recorded["audio"]) == list(expected["audio"]) + for name, theirs in expected["audio"].items(): + ours = recorded["audio"][name] + assert (ours["dtype"], ours["shape"]) == (theirs["dtype"], theirs["shape"]), name + assert all( + math.isclose(a, b, rel_tol=0, abs_tol=_PROJECTION_TOLERANCE) + for a, b in zip(ours["projections"], theirs["projections"], strict=True) + ), name + + +@live +def test_live_every_configured_data_dir_contributed_real_data(answers): + for root in answers["roots"]: + names = [n for kind in ("open_asr", "fleurs", "clips") for n in answers[kind] if n.startswith(f"{root}:")] + assert names, root + print("data roots:", answers["roots"]) + + +@live +def test_live_every_english_set_scores_exactly_as_the_leaderboard(answers): + checked = [] + for name, manifest, theirs in _pairs(answers["open_asr"], answers["answer"]["open_asr"]): + ours = _score_open_asr(manifest) + assert len(manifest) - len(ours.excluded_sample_ids) == theirs["kept"], name + _assert_open_asr_equal(ours, theirs) + checked.append((name, len(manifest))) + print("open_asr checked:", checked) + + +@live +def test_live_without_merge_compounds_the_leaderboard_is_missed(answers): + """The negative control on the live corpus: every utterance set with a split + compound differs, so at least the synthetic one must.""" + differs = { + name: _errors_without_merge(manifest) != theirs["totals"]["total"] + for name, manifest, theirs in _pairs(answers["open_asr"], answers["answer"]["open_asr"]) + if not any("parent_id" in row for row in manifest) + } + assert differs["synthetic"] + + +@live +def test_live_every_fleurs_set_normalizes_and_scores_exactly_as_the_leaderboards_multilingual_code(answers): + assert answers["answer"]["versions"]["num2words"] == "0.5.14" + checked = [] + for name, dataset, theirs in _pairs(answers["fleurs"], answers["answer"]["fleurs"]): + # The leaderboard's language code (from its config name) is ours. + assert theirs["language"] == OPEN_ASR_LANGUAGE_CODES[dataset["language"]], name + _assert_fleurs_equal(dataset, theirs) + checked.append((name, len(dataset["pairs"]))) + assert sum(name.startswith("fleurs_tsv:") for name, _ in checked) == len(asr_datasets.FLEURS_LOCALES) + print("fleurs checked:", checked) + + +@live +def test_live_the_vendored_multilingual_excerpts_are_the_leaderboards_source(): + """multilingual.py's FILLER_WORDS / MultilingualNormalizer and + normalize_compound_pairs are the pinned checkout's text, character for + character (only the module's header, imports and stand-in are pipette's).""" + import inspect + + from pipette_scores.scoring.asr.open_asr_normalizer import multilingual + + env = reference_env() + assert env is not None + upstream = env / "open_asr_leaderboard" / "normalizer" + data_utils = (upstream / "data_utils.py").read_text(encoding="utf-8") + eval_utils = (upstream / "eval_utils.py").read_text(encoding="utf-8") + ours = inspect.getsource(multilingual) + filler_block = ours[ours.index("# Language-specific filler words") : ours.index("class MultilingualNormalizer")] + assert filler_block in data_utils + assert inspect.getsource(multilingual.MultilingualNormalizer) in data_utils + assert inspect.getsource(multilingual.normalize_compound_pairs) in eval_utils + + +@live +def test_live_every_clip_resamples_exactly_as_datasets_audio_16k(answers): + import soundfile as sf + + assert answers["answer"]["versions"]["datasets"] == "2.19.0" + checked = 0 + for name, clip, theirs in _pairs(answers["clips"], answers["answer"]["audio"]): + expected = reference_array(theirs) + ours = _asr_audio.resample_16k_mono(clip) + assert theirs["sampling_rate"] == 16000, name + assert ours.dtype == expected.dtype and ours.shape == expected.shape, name + assert np.array_equal(ours, expected), name + # And the served WAV carries those samples as float32. + served, rate = sf.read(io.BytesIO(base64.b64decode(_asr_audio.encode(clip))), dtype="float32") + assert rate == 16000 and np.array_equal(served, expected.astype(np.float32)), name + checked += 1 + print("clips checked:", checked) diff --git a/packages/pipette-scores/tests/test_asr_scoring.py b/packages/pipette-scores/tests/test_asr_scoring.py new file mode 100644 index 0000000..a56e064 --- /dev/null +++ b/packages/pipette-scores/tests/test_asr_scoring.py @@ -0,0 +1,270 @@ +"""ASR scoring: the normalizer recipes, the aligners and the corpus rules.""" + +from __future__ import annotations + +import hashlib +from pathlib import Path + +import pytest + +from pipette_scores.scoring.asr.align import Aligner, Counts, align +from pipette_scores.scoring.asr.normalize import NormalizerRecipe, normalize +from pipette_scores.scoring.asr.wer import AsrReference, AsrScoringSpec, score_asr + +ASR = Path(__file__).parents[1] / "pipette_scores" / "scoring" / "asr" + +OPEN_ASR = AsrScoringSpec(NormalizerRecipe.OPEN_ASR_ENGLISH, Aligner.KALDIALIGN_COMPOUND, open_asr_exclusions=True) +FLEURS_WER = AsrScoringSpec(NormalizerRecipe.OPEN_ASR_MULTILINGUAL, Aligner.KALDIALIGN_COMPOUND_PAIRS) + + +@pytest.mark.parametrize( + ("path", "sha256"), + [ + ("open_asr_normalizer/normalizer.py", "490b56393484ef386b486f3679cc2261264af280b3512154431e0bdcf2778295"), + # Its multilingual normalizer and compound alignment, excerpted unchanged + # (tests/test_asr_public_reference.py checks the excerpts against the source). + ( + "open_asr_normalizer/multilingual.py", + "9562b1d60fd0e691295879d4ea706497ff23e58d53b5f9ef1f33ce2b498678ce", + ), + ( + "open_asr_normalizer/english_abbreviations.py", + "52997cc963e0bd6568d15554ad8d5fb0759f20f361483632a22f90d2d0e07db1", + ), + ], +) +def test_the_vendored_normalizers_are_the_recorded_bytes(path: str, sha256: str) -> None: + """A changed byte changes the numbers; NOTICE.md says how to update.""" + assert hashlib.sha256((ASR / path).read_bytes()).hexdigest() == sha256 + + +@pytest.mark.parametrize( + ("recipe", "language", "raw", "normalized"), + [ + # Open ASR English: contractions, titles, numbers, fillers, casing. + ( + NormalizerRecipe.OPEN_ASR_ENGLISH, + "eng", + "Mr. Smith won't go, um, at 10:00!", + "mister smith will not go at 10", + ), + (NormalizerRecipe.OPEN_ASR_ENGLISH, "eng", "It costs twenty five dollars.", "it costs $25"), + # Open ASR multilingual: English text takes the English normalizer. + (NormalizerRecipe.OPEN_ASR_MULTILINGUAL, "eng", "It costs twenty five dollars.", "it costs $25"), + # Otherwise lowercase, punctuation and symbols to spaces (collapsed), + # diacritics kept, digits spelled out in the language. + (NormalizerRecipe.OPEN_ASR_MULTILINGUAL, "deu", "Grüße, Welt!", "grüße welt"), + (NormalizerRecipe.OPEN_ASR_MULTILINGUAL, "fra", "L'été 2006 à Noël.", "l été deux mille six à noël"), + # Parenthesized and bracketed spans are dropped. + (NormalizerRecipe.OPEN_ASR_MULTILINGUAL, "nld", "Het is (lacht) [ruis] goed.", "het is goed"), + # Space-separated thousands are joined first ("10 000" is one number). + (NormalizerRecipe.OPEN_ASR_MULTILINGUAL, "nld", "10 000 mensen", "tienduizend mensen"), + # The leaderboard's language codes: "pt" is num2words' European Portuguese. + (NormalizerRecipe.OPEN_ASR_MULTILINGUAL, "por", "16 dias", "dezasseis dias"), + # Number words come after the lowercase pass and keep num2words' casing. + (NormalizerRecipe.OPEN_ASR_MULTILINGUAL, "deu", "1000000 Menschen", "eine Million menschen"), + ], +) +def test_each_recipe_normalizes_as_its_public_protocol( + recipe: NormalizerRecipe, language: str, raw: str, normalized: str +) -> None: + assert normalize(recipe, raw, language) == normalized + + +# Each FLEURS language: a reference with digits and a hypothesis that spells the +# number out (and the reverse) score no error. +DIGITS_AND_WORDS = [ + ("eng", "He was 20 years old in 1999.", "he was twenty years old in nineteen ninety nine"), + ("spa", "Tenía 20 años en 1999.", "tenía veinte años en mil novecientos noventa y nueve"), + ("por", "Ela tinha 20 anos em 1999.", "ela tinha vinte anos em mil novecentos e noventa e nove"), + ( + "deu", + "Er war 20 Jahre alt, im Jahr 1999.", + "er war zwanzig jahre alt im jahr eintausendneunhundertneunundneunzig", + ), + ("fra", "Il avait 20 ans en 1900.", "il avait vingt ans en mille neuf cents"), + ("ita", "Aveva 20 anni nel 1999.", "aveva venti anni nel millenovecentonovantanove"), + ("nld", "Hij was 20 jaar oud in 1999.", "hij was twintig jaar oud in duizendnegenhonderdnegenennegentig"), +] + + +@pytest.mark.parametrize(("language", "digits", "words"), DIGITS_AND_WORDS, ids=[c[0] for c in DIGITS_AND_WORDS]) +def test_a_number_written_in_digits_or_in_words_is_no_error(language: str, digits: str, words: str) -> None: + """The leaderboard's multilingual protocol spells digits out on both sides + (num2words in the language; the English normalizer turns words into digits), + so a model is not charged for writing "twenty" for "20", or the reverse.""" + for reference, hypothesis in ((digits, words), (words, digits)): + score = score_asr(FLEURS_WER, [AsrReference("s", reference, language)], {"s": hypothesis}) + assert score.totals.errors == 0, (reference, hypothesis) + assert score.totals.reference_units > 0 + + +@pytest.mark.parametrize( + ("language", "digits", "words"), + [c for c in DIGITS_AND_WORDS if c[0] != "eng"], + ids=[c[0] for c in DIGITS_AND_WORDS if c[0] != "eng"], +) +def test_without_its_number_words_the_protocol_would_charge_them(monkeypatch, language, digits, words) -> None: + """Negative control: the same pairs with the number step switched off + (MultilingualNormalizer._normalize_numbers as the identity) score errors, so + the test above checks the number normalization, not the alignment.""" + from pipette_scores.scoring.asr.open_asr_normalizer.multilingual import MultilingualNormalizer + + monkeypatch.setattr(MultilingualNormalizer, "_normalize_numbers", lambda self, text, lang: text) + score = score_asr(FLEURS_WER, [AsrReference("s", digits, language)], {"s": words}) + assert score.totals.errors > 0 + + +def test_a_hyphenated_number_word_is_still_charged_as_on_the_leaderboard() -> None: + """num2words' hyphens survive (it runs after punctuation becomes spaces), so + "25" reads "vingt-cinq" while a hypothesis "vingt-cinq" splits into two + words: one substitution and one insertion, as the leaderboard counts it.""" + score = score_asr(FLEURS_WER, [AsrReference("s", "25 euros", "fra")], {"s": "vingt-cinq euros"}) + assert (score.units[0].reference, score.units[0].hypothesis) == ("vingt-cinq euros", "vingt cinq euros") + assert score.totals == Counts(1, 1, 0, 1, 2) + + +def test_the_multilingual_recipe_refuses_a_language_it_has_no_code_for() -> None: + """Armenian (the leaderboard's own normalizer, not vendored) and any other + language not evaluated here are refused, never scored without number words.""" + for language in ("hye", "cmn", None): + with pytest.raises(ValueError, match="not set up for language"): + normalize(NormalizerRecipe.OPEN_ASR_MULTILINGUAL, "1 2 3", language) + with pytest.raises(ValueError, match="not set up for language"): + score_asr(FLEURS_WER, [AsrReference("s", "ein Satz", "hye")], {"s": "ein Satz"}) + + +@pytest.mark.parametrize( + ("aligner", "reference", "hypothesis", "counts"), + [ + # Compound-aware: "foot ball" against "football" is one match, not two errors. + (Aligner.KALDIALIGN_COMPOUND, "the football game", "the foot ball game", Counts(3, 0, 0, 0, 3)), + (Aligner.KALDIALIGN_COMPOUND, "a b", "", Counts(0, 0, 2, 0, 2)), + (Aligner.KALDIALIGN_COMPOUND_PAIRS, "a b c d", "a x c", Counts(2, 1, 1, 0, 4)), + (Aligner.KALDIALIGN_COMPOUND_PAIRS, "a b", "a b c", Counts(2, 0, 0, 1, 2)), + (Aligner.KALDIALIGN_COMPOUND_PAIRS, "a b", "", Counts(0, 0, 2, 0, 2)), + # A split compound, either way, as kaldialign's merge alone forgives it. + (Aligner.KALDIALIGN_COMPOUND_PAIRS, "il dopo guerra", "il dopoguerra", Counts(2, 0, 0, 0, 2)), + # normalize_compound_pairs: words cut at another point ("zumbei spiel") + # are the same characters, so both sides take the joined form (one + # reference word fewer); kaldialign's merge alone charges them. + ( + Aligner.KALDIALIGN_COMPOUND_PAIRS, + "das ist zum beispiel gut", + "das ist zumbei spiel gut", + Counts(4, 0, 0, 0, 4), + ), + (Aligner.KALDIALIGN_COMPOUND, "das ist zum beispiel gut", "das ist zumbei spiel gut", Counts(3, 2, 0, 0, 5)), + ], +) +def test_each_aligner_counts_edits(aligner: Aligner, reference: str, hypothesis: str, counts: Counts) -> None: + assert align(aligner, reference, hypothesis) == counts + + +def test_every_aligner_counts_words() -> None: + """The character error rate is retired: no dataset counts characters, and a + recipe naming the character aligner is not one this scorer reads.""" + assert {a.value for a in Aligner} == {"kaldialign_compound", "kaldialign_compound_pairs"} + for retired in ("jiwer_characters", "jiwer_words"): + with pytest.raises(ValueError): + Aligner(retired) + assert {AsrScoringSpec(NormalizerRecipe.OPEN_ASR_MULTILINGUAL, a).error_unit for a in Aligner} == {"word"} + + +def test_the_whisper_recipes_are_retired() -> None: + """FLEURS is scored by the leaderboard's multilingual protocol; a dataset + whose recipe still names Whisper's normalizers is not one this scorer reads.""" + assert {r.value for r in NormalizerRecipe} == {"open_asr_english", "open_asr_multilingual"} + for retired in ("whisper_english", "whisper_multilingual"): + with pytest.raises(ValueError): + NormalizerRecipe(retired) + + +def test_the_corpus_rate_sums_edits_over_reference_units() -> None: + """Not the mean of the per-sample rates: 1 error in 10 words and 1 in 1 is 2/11.""" + refs = [AsrReference("long", " ".join("abcdefghij"), "deu"), AsrReference("short", "z", "deu")] + score = score_asr(FLEURS_WER, refs, {"long": " ".join("abcdefghiX"), "short": "y"}) + assert score.totals == Counts(9, 2, 0, 0, 11) + assert score.corpus_error_rate == pytest.approx(2 / 11) + + +def test_open_asr_drops_references_that_normalize_to_nothing_or_the_ignore_marker() -> None: + refs = [ + AsrReference("kept", "hello world", "eng"), + AsrReference("empty", " [noise]", "eng"), + AsrReference("marker", "IGNORE TIME SEGMENT IN SCORING", "eng"), + ] + score = score_asr(OPEN_ASR, refs, {"kept": "hello world", "empty": "something", "marker": "words"}) + assert [u.unit_id for u in score.units] == ["kept"] + assert score.excluded_sample_ids == ("empty", "marker") + + +def test_fleurs_keeps_every_reference() -> None: + score = score_asr(FLEURS_WER, [AsrReference("p", "...", "deu")], {"p": "ja"}) + assert score.excluded_sample_ids == () + assert score.totals.insertions == 1 + + +def test_a_run_without_an_answer_to_every_sample_is_no_result() -> None: + """Never an empty transcript (all deletions): a missing answer is refused + (pipette's own rule); answered in full, the same run scores.""" + refs = [AsrReference(i, "one two", "deu") for i in ("a", "b", "c")] + with pytest.raises(ValueError, match="an answer to every sample; unanswered: b, c"): + score_asr(FLEURS_WER, refs, {"a": "one two"}) + complete = score_asr(FLEURS_WER, refs, {"a": "one two", "b": "", "c": "one"}) + assert complete.corpus_error_rate == pytest.approx(3 / 6) + + +def test_an_excluded_sample_must_still_have_its_generation() -> None: + refs = [AsrReference("kept", "hello world", "eng"), AsrReference("noise", "", "eng")] + with pytest.raises(ValueError, match="unanswered: noise"): + score_asr(OPEN_ASR, refs, {"kept": "hello world"}) + score = score_asr(OPEN_ASR, refs, {"kept": "hello world", "noise": "words"}) + assert score.excluded_sample_ids == ("noise",) and score.corpus_error_rate == 0.0 + + +def test_a_dataset_with_no_reference_unit_has_no_rate() -> None: + """All references excluded, or only insertions: counts but no rate (the + leaderboard would report 0.0, or infinity).""" + excluded = score_asr(OPEN_ASR, [AsrReference("a", "", "eng")], {"a": "words"}) + assert excluded.units == () and excluded.corpus_error_rate is None + empty = score_asr(FLEURS_WER, [AsrReference("p", "...", "deu")], {"p": "ja nee"}) + assert empty.totals == Counts(0, 0, 0, 2, 0) and empty.corpus_error_rate is None + + +def test_chunks_score_once_per_session_joined_in_order() -> None: + """Stripped, empties skipped, joined with one space, then normalized whole.""" + session_ref = "Revenue grew five percent this quarter." + refs = [ + AsrReference("c2", session_ref, "eng", session="call", chunk_index=2), + AsrReference("c0", session_ref, "eng", session="call", chunk_index=0), + AsrReference("c1", session_ref, "eng", session="call", chunk_index=1), + ] + score = score_asr(OPEN_ASR, refs, {"c0": " Revenue grew ", "c1": " ", "c2": "5% this quarter. "}) + [unit] = score.units + assert unit.unit_id == "call" + assert unit.sample_ids == ("c0", "c1", "c2") + assert unit.hypothesis == unit.reference + assert unit.counts.errors == 0 + + +@pytest.mark.parametrize( + ("chunks", "size", "message"), + [ + ([("c0", "same", 0), ("c2", "same", 2)], None, "chunk indexes"), + ([("c0", "one", 0), ("c1", "other", 1)], None, "different references"), + # The dataset holds two chunks of a call that has three. + ([("c0", "same", 0), ("c1", "same", 1)], 3, "declares"), + ], +) +def test_a_broken_session_is_refused(chunks: list[tuple[str, str, int]], size: int | None, message: str) -> None: + refs = [AsrReference(i, ref, "eng", session="call", chunk_index=k, session_size=size) for i, ref, k in chunks] + with pytest.raises(ValueError, match=message): + score_asr(OPEN_ASR, refs, {i: "x" for i, _, _ in chunks}) + + +def test_a_session_missing_a_chunk_is_refused() -> None: + """pipette's own rule (the leaderboard warns and scores the call without it).""" + refs = [AsrReference(f"c{k}", "a b c", "eng", session="call", chunk_index=k, session_size=3) for k in range(3)] + with pytest.raises(ValueError, match="unanswered: c1"): + score_asr(OPEN_ASR, refs, {"c0": "a", "c2": "c"}) diff --git a/packages/pipette-scores/tests/test_asr_suite.py b/packages/pipette-scores/tests/test_asr_suite.py new file mode 100644 index 0000000..5a6abb3 --- /dev/null +++ b/packages/pipette-scores/tests/test_asr_suite.py @@ -0,0 +1,123 @@ +"""The ASR suite headlines (scoring/asr/suite.py): the Open ASR Leaderboard's macro mean +and FLEURS' macro mean WER.""" + +from __future__ import annotations + +from typing import Any + +import pytest + +from pipette_scores.dataset_catalog.asr_datasets import OPEN_ASR_EVAL_IDS, FLEURS_EVAL_IDS +from pipette_scores.scoring.asr.suite import fleurs_mean, open_asr_mean +from pipette_scores.types import EvalId + + +def _context(eval_id: EvalId, errors: int, reference_units: int) -> dict[str, Any]: + """A /score context with the given corpus counts.""" + return { + "error_unit": "word", + "error_rate": errors / reference_units, + "result": 100 * errors / reference_units, + "substitutions": errors, + "deletions": 0, + "insertions": 0, + "reference_units": reference_units, + } + + +def _open_asr() -> list[tuple[str, dict[str, Any]]]: + # Unequal sizes: dataset i has rate (i + 1)% over 10**(i % 4 + 2) words. + return [ + (e.value, _context(e, (i + 1) * 10 ** (i % 4), 100 * 10 ** (i % 4))) for i, e in enumerate(OPEN_ASR_EVAL_IDS) + ] + + +def test_the_open_asr_mean_is_the_equal_weight_mean_of_its_nine_public_sets() -> None: + mean = open_asr_mean(_open_asr()) + # (1 + 2 + ... + 9) / 9 % = 5 %, whatever each dataset's size. + assert mean.error_unit == "word" and mean.eval_ids == OPEN_ASR_EVAL_IDS and len(mean.eval_ids) == 9 + assert mean.error_rate == pytest.approx(0.05) and mean.result == pytest.approx(5.0) + assert mean.leaderboard_result == 5.0 + # Not the micro-average (summed edits over summed words), which sizes weight. + pairs = _open_asr() + micro = sum(c["substitutions"] for _, c in pairs) / sum(c["reference_units"] for _, c in pairs) + assert micro != pytest.approx(mean.error_rate) + + +def test_the_leaderboard_figure_rounds_each_set_then_the_mean() -> None: + """The console figure of score_results: each set's round(100 * wer, 2), + averaged, the mean shown to two decimals.""" + pairs = [(e.value, _context(e, 1, 3 if i == 0 else 100)) for i, e in enumerate(OPEN_ASR_EVAL_IDS)] + mean = open_asr_mean(pairs) + # 33.333...% and eight 1%: unrounded 4.592592..., rounded sets (33.33 + 8) / 9 = 4.5922... + assert mean.result == pytest.approx((100 / 3 + 8) / 9) + assert mean.leaderboard_result == round((33.33 + 8 * 1.0) / 9, 2) == 4.59 + + +def test_the_leaderboard_figure_is_within_0_01_of_the_unrounded_mean_not_0_005() -> None: + """Both roundings can push the same way: eight sets at 1.0049 % round down to + 1.00 and one at 1.0449 % to 1.04, so the console shows round(9.04 / 9, 2) = + 1.00 for an unrounded mean of 1.00934 %: off by more than 0.005, within 0.01.""" + rates = [0.010049] * 8 + [0.010449] + pairs = [ + (e.value, {**_context(e, 1, 100), "error_rate": rate, "result": 100 * rate}) + for e, rate in zip(OPEN_ASR_EVAL_IDS, rates, strict=True) + ] + mean = open_asr_mean(pairs) + assert mean.result == pytest.approx((8 * 1.0049 + 1.0449) / 9) + assert mean.leaderboard_result == 1.0 + assert 0.005 < mean.result - mean.leaderboard_result <= 0.01 + # The CSV `avg` (the mean of the rounded sets, unrounded) is within 0.005. + assert abs(mean.result - (8 * 1.0 + 1.04) / 9) <= 0.005 + + +def _fleurs() -> list[tuple[str, dict[str, Any]]]: + # Locale i at (i + 1) % over 100 * 10**(i % 3) words. + return [(e.value, _context(e, (i + 1) * 10 ** (i % 3), 100 * 10 ** (i % 3))) for i, e in enumerate(FLEURS_EVAL_IDS)] + + +def test_fleurs_is_the_equal_weight_mean_wer_of_its_seven_locales() -> None: + mean = fleurs_mean(_fleurs()) + assert [e.value.removeprefix("asr_fleurs_") for e in mean.eval_ids] == [ + "en_us", "es_419", "pt_br", "de_de", "fr_fr", "it_it", "nl_nl", + ] # fmt: skip + # (1 + 2 + ... + 7) / 7 % = 4 %, whatever each locale's size. + assert mean.error_unit == "word" and mean.result == pytest.approx(4.0) and mean.leaderboard_result == 4.0 + pairs = _fleurs() + micro = sum(c["substitutions"] for _, c in pairs) / sum(c["reference_units"] for _, c in pairs) + assert micro != pytest.approx(mean.error_rate) + + +@pytest.mark.parametrize( + "change, message", + [ + (lambda pairs: pairs[:-1], "missing"), + (lambda pairs: [*pairs, pairs[0]], "more than once"), + (lambda pairs: [*pairs, ("asr_fleurs_en_us", _context(EvalId.ASR_FLEURS_EN_US, 1, 10))], "not in the suite"), + ( + lambda pairs: [(pairs[0][0], {**pairs[0][1], "error_rate": None}), *pairs[1:]], + "no corpus rate", + ), + (lambda pairs: [(pairs[0][0], {**pairs[0][1], "error_unit": "character"}), *pairs[1:]], "scored in"), + ], +) +def test_a_suite_headline_needs_every_dataset_once_with_a_rate(change, message) -> None: + with pytest.raises(ValueError, match=message): + open_asr_mean(change(_open_asr())) + + +@pytest.mark.parametrize( + "change, message", + [ + (lambda pairs: pairs[:-1], "missing: asr_fleurs_nl_nl"), + # A dropped locale is no eval id any more. + ( + lambda pairs: [*pairs, ("asr_fleurs_ja_jp", _context(EvalId.ASR_FLEURS_EN_US, 1, 10))], + "'asr_fleurs_ja_jp' is not a valid EvalId", + ), + (lambda pairs: [(pairs[0][0], {**pairs[0][1], "error_unit": "character"}), *pairs[1:]], "scored in"), + ], +) +def test_fleurs_needs_its_seven_locales_in_words(change, message) -> None: + with pytest.raises(ValueError, match=message): + fleurs_mean(change(_fleurs())) diff --git a/packages/pipette-scores/tests/test_build_math_500_dataset.py b/packages/pipette-scores/tests/test_build_math_500_dataset.py new file mode 100644 index 0000000..33b039c --- /dev/null +++ b/packages/pipette-scores/tests/test_build_math_500_dataset.py @@ -0,0 +1,102 @@ +"""scripts/build_math_500_dataset.py: the per-version prompt, repeats and grader.""" + +import dataclasses +import importlib.util +import json +import pathlib +import sys + +import pytest + +from pipette_scores.dataset_catalog import load_eval_samples, load_prompt_samples +from pipette_scores.dataset_catalog.math_500 import content_sha256 +from pipette_scores.types import EvalId, Math500Grader + +_SCRIPT = pathlib.Path(__file__).resolve().parents[1] / "scripts" / "build_math_500_dataset.py" +_spec = importlib.util.spec_from_file_location("build_math_500_dataset", _SCRIPT) +build = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(build) + +_DOCS = [ + {"problem": "What is $2+2$?", "solution": "$2+2=\\boxed{4}$.", "answer": "4", "unique_id": "test/a/1.json"}, + { + "problem": "Simplify $\\frac{2}{4}$.", + "solution": "It is $\\boxed{\\frac{1}{2}}$.", + "answer": "\\frac{1}{2}", + "unique_id": "test/a/2.json", + }, +] + + +def test_liquid_evals_prompt_is_the_math500_task_prompt(): + # liquid_evals tasks/math500.py prepare_prompt, line 49. + assert build.VERSIONS["2026.10.1"].prompt("What is $2+2$?") == ( + "What is $2+2$?\n\nPlease reason step by step, and put your final answer within \\boxed{}." + ) + + +def test_std_prompt_is_unchanged(): + assert build.VERSIONS["2026.06.1"].prompt("What is $2+2$?") == ( + "Solve the following math problem step by step. Put your answer inside \\boxed{}.\n\n" + "What is $2+2$?\n\n" + "Remember to put your answer inside \\boxed{}." + ) + + +def test_versions(): + std, le = build.VERSIONS["2026.06.1"], build.VERSIONS["2026.10.1"] + assert (std.repeats, std.grader, std.check_gold_extraction) == (5, None, False) + assert std.content_sha256 == "03d8a41616e2b3f5e0c317e8de72d3a88b930186a0ef7d00686e90051004f588" + assert (le.repeats, le.grader, le.check_gold_extraction) == (5, Math500Grader.LIQUID_EVALS_MATH500, True) + assert le.content_sha256 == "5a0da41f1322ab852cfe7ea2e51fca9277f9142cc64a011f07dd597bb6d5eaa2" + assert "tasks/math500.py" in le.notes and "_std" not in le.notes + + +def _run(monkeypatch, tmp_path, version, docs, *, pin_hash=True): + """Run the builder on ``docs`` in place of the HF dataset; return the dataset root.""" + monkeypatch.setattr(build, "load_dataset", lambda *args, **kwargs: docs) + monkeypatch.setattr(build, "EXPECTED_SAMPLES", len(docs)) + if pin_hash: + spec = build.VERSIONS[version] + rows = [build.format_row(doc, spec) for doc in docs] + digest = content_sha256([(r["prompt"], r["answer"]) for r in rows]) + monkeypatch.setitem(build.VERSIONS, version, dataclasses.replace(spec, content_sha256=digest)) + out = tmp_path / "datasets" / "math_500" + monkeypatch.setattr(sys, "argv", ["build", "--version", version, "--out-dir", str(out)]) + build.main() + return tmp_path + + +@pytest.mark.parametrize( + "version, repeats, grader", + [("2026.06.1", 5, Math500Grader.MATH_GENERIC), ("2026.10.1", 5, Math500Grader.LIQUID_EVALS_MATH500)], +) +def test_build_writes_a_dataset_the_catalog_loads(monkeypatch, tmp_path, version, repeats, grader): + root = _run(monkeypatch, tmp_path, version, _DOCS) + meta = json.loads((root / "datasets" / "math_500" / version / "metadata.json").read_text()) + assert meta["dataset"] == version + assert meta["repeats"] == repeats + assert meta.get("grader") == (None if grader is Math500Grader.MATH_GENERIC else grader.value) + + samples = load_eval_samples(EvalId.MATH_500, version, root_dir=root) + prompts = load_prompt_samples(EvalId.MATH_500, version, root_dir=root) + assert len(samples) == len(prompts) == len(_DOCS) * repeats + # Each problem is served as the attempt ids #0 .. #(repeats-1). + assert [s.id for s in samples] == [p.id for p in prompts] + assert [p.id.rsplit("#", 1)[1] for p in prompts[:repeats]] == [str(k) for k in range(repeats)] + assert {s.grader for s in samples} == {grader} + assert [s.answer for s in samples[::repeats]] == [d["answer"] for d in _DOCS] + first = prompts[0].messages + assert len(first) == 1 and first[0].role == "user" + assert first[0].content == build.VERSIONS[version].prompt(_DOCS[0]["problem"]) + + +def test_build_rejects_gold_that_liquid_evals_would_extract_differently(monkeypatch, tmp_path): + docs = [{**_DOCS[0], "answer": "5"}] + with pytest.raises(SystemExit, match="differs from extract_math_answer"): + _run(monkeypatch, tmp_path, "2026.10.1", docs) + + +def test_build_rejects_unexpected_content(monkeypatch, tmp_path): + with pytest.raises(SystemExit, match="content hash"): + _run(monkeypatch, tmp_path, "2026.10.1", _DOCS, pin_hash=False) diff --git a/packages/pipette-scores/tests/test_dataset_catalog.py b/packages/pipette-scores/tests/test_dataset_catalog.py index b08300a..f144b32 100644 --- a/packages/pipette-scores/tests/test_dataset_catalog.py +++ b/packages/pipette-scores/tests/test_dataset_catalog.py @@ -4,7 +4,7 @@ from pipette_scores.dataset_catalog import list_dataset_names, load_eval_samples, load_prompt_samples from pipette_scores.repeats import logical_id -from pipette_scores.types import EvalId, EvalSample, IFBenchSample +from pipette_scores.types import EvalId, EvalSample, IFBenchSample, Math500Grader def test_list_dataset_names_empty(tmp_path): @@ -98,6 +98,31 @@ def test_math_500_integrity_mismatch_raises(math_500_dataset_dir): load_eval_samples(EvalId.MATH_500, "test_ds", root_dir=root) +def test_math_500_grader_defaults_to_math_generic(math_500_dataset_dir): + # 2026.06.1's metadata.json predates the grader key. + root = math_500_dataset_dir.parent.parent.parent + samples = load_eval_samples(EvalId.MATH_500, "test_ds", root_dir=root) + assert {s.grader for s in samples} == {Math500Grader.MATH_GENERIC} + + +def test_math_500_grader_from_metadata(math_500_dataset_dir): + meta = json.loads((math_500_dataset_dir / "metadata.json").read_text()) + meta["grader"] = "liquid_evals_math500" + (math_500_dataset_dir / "metadata.json").write_text(json.dumps(meta)) + root = math_500_dataset_dir.parent.parent.parent + samples = load_eval_samples(EvalId.MATH_500, "test_ds", root_dir=root) + assert {s.grader for s in samples} == {Math500Grader.LIQUID_EVALS_MATH500} + + +def test_math_500_unknown_grader_raises(math_500_dataset_dir): + meta = json.loads((math_500_dataset_dir / "metadata.json").read_text()) + meta["grader"] = "liquid_evals" + (math_500_dataset_dir / "metadata.json").write_text(json.dumps(meta)) + root = math_500_dataset_dir.parent.parent.parent + with pytest.raises(ValueError, match="metadata.grader must be one of"): + load_eval_samples(EvalId.MATH_500, "test_ds", root_dir=root) + + def test_ifbench_loads_and_verifies_integrity(ifbench_dataset_dir): from pipette_scores.dataset_catalog.ifbench import content_sha256 from pipette_scores.parquet_io import read_parquet diff --git a/packages/pipette-scores/tests/test_gpqa_diamond.py b/packages/pipette-scores/tests/test_gpqa_diamond.py index e1af0aa..ebfd818 100644 --- a/packages/pipette-scores/tests/test_gpqa_diamond.py +++ b/packages/pipette-scores/tests/test_gpqa_diamond.py @@ -27,6 +27,9 @@ def test_answer_out_of_domain_rejected(bad): ("Answer: A", "C", False), # reasoning is stripped before extraction, so the in-think "C" is ignored ("maybe C Answer: D", "D", True), + # same for a Gemma 4 thought channel; one cut off is all reasoning + ("<|channel>thought\nmaybe CAnswer: D", "D", True), + ("<|channel>thought\nAnswer: D? Check the units first", "D", False), # extractor only accepts A-D, so an out-of-range / unparseable answer fails ("I am not sure", "A", False), ], @@ -71,6 +74,8 @@ def test_context_schema_stable_regardless_of_availability(): assert isinstance(empty_ctx["gpqa_diamond_pass_at_1"], float) assert isinstance(empty_ctx["gpqa_diamond_pass_at_1_stderr"], float) assert isinstance(empty_ctx["gpqa_diamond_unparsed_rate"], float) + assert empty_ctx["gpqa_diamond_blanked_truncated"] == 0 + assert isinstance(empty_ctx["gpqa_diamond_pass_at_1_blanked"], float) assert isinstance(empty_ctx["gpqa_diamond_pass_at_1_per_sample"], dict) # flat, fixed choice keys — always present (count 0 when a letter is unused) for choice in ["A", "B", "C", "D", "none"]: @@ -92,3 +97,26 @@ def test_context_extraction_diagnostics(): assert ctx["gpqa_diamond_choice/C"] == 0 assert ctx["gpqa_diamond_choice/D"] == 0 assert ctx["gpqa_diamond_choice/none"] == 1 + + +@pytest.mark.parametrize("stop_reason", ["truncated", "doom_loop"]) +@pytest.mark.parametrize("enable_thinking, blanked", [(True, 2), (False, 0), (None, 0)]) +def test_cut_reasoning_is_graded_raw_and_counted(stop_reason, enable_thinking, blanked): + samples = [_sample("q1", "B"), _sample("q2", "B"), _sample("q3", "B"), _sample("q4", "B")] + completions = [ + # The prompt opened the reasoning and generation was cut: graded whole, + # as liquid_evals grades it. + SampleCompletion(id="q1", completion="Option B fits. Answer: B. Double-check option", stop_reason=stop_reason), + # The completion opened itself: nothing is left to grade. + SampleCompletion(id="q2", completion="Option B fits. Answer: B. Double-check", stop_reason=stop_reason), + # The reasoning closed. + SampleCompletion(id="q3", completion="Option B fits.\n\nAnswer: B", stop_reason=stop_reason), + SampleCompletion(id="q4", completion="Option B fits.\n\nAnswer: B", stop_reason="eos"), + ] + scores, ctx = score(completions, samples, enable_thinking=enable_thinking) + assert [s.is_correct for s in scores] == [True, False, True, True] + assert ctx["gpqa_diamond_pass_at_1"] == 0.75 + assert ctx["gpqa_diamond_choice/none"] == 1 + # The strict view counts q1 and q2 wrong; only q1 was correct. + assert ctx["gpqa_diamond_blanked_truncated"] == blanked + assert ctx["gpqa_diamond_pass_at_1_blanked"] == (0.5 if blanked else 0.75) diff --git a/packages/pipette-scores/tests/test_hashing.py b/packages/pipette-scores/tests/test_hashing.py new file mode 100644 index 0000000..02dbe11 --- /dev/null +++ b/packages/pipette-scores/tests/test_hashing.py @@ -0,0 +1,98 @@ +"""samples_digest: the content digest a client pins a copy of /samples with.""" + +import hashlib + +from pipette_scores.hashing import samples_digest +from pipette_scores.types import ChatMessage, EvalSample + +SAMPLES = [ + EvalSample( + id="a1", + messages=[ + ChatMessage(role="system", content="Think first."), + ChatMessage( + role="user", + content=[ + {"type": "image_url", "image_url": {"url": "data:image/png;base64,iVBORw0KGgo="}}, + {"type": "text", "text": "Which mood? 心情\nA. Sad\nB. Happy"}, + ], + ), + ], + ), + EvalSample(id="b2", messages=[ChatMessage(role="user", content='Is there a dog? "Yes" or no.')]), +] + +# The canonical form written out by hand: keys sorted, no whitespace, UTF-8, one +# line per sample. A client may recompute the digest from a /samples body this way. +CANONICAL = ( + '{"id":"a1","messages":[{"content":"Think first.","role":"system"},{"content":[' + '{"image_url":{"url":"data:image/png;base64,iVBORw0KGgo="},"type":"image_url"},' + '{"text":"Which mood? 心情\\nA. Sad\\nB. Happy","type":"text"}],"role":"user"}]}\n' + '{"id":"b2","messages":[{"content":"Is there a dog? \\"Yes\\" or no.","role":"user"}]}\n' +) + + +def test_the_digest_is_the_sha256_of_the_canonical_form(): + assert samples_digest(SAMPLES) == f"sha256:{hashlib.sha256(CANONICAL.encode()).hexdigest()}" + + +def test_the_digest_is_pinned(): + # Digests are pinned downstream (a corpus records the one it was built from), + # so a change to the canonical form must be deliberate: it moves every pin. + # mgmt's Rust digest pins this same value. + assert samples_digest(SAMPLES) == "sha256:5654ef883c378c4325cfe70594545d4a19c094defdf21b32487f9de94a6b4849" + + +def test_awkward_text_is_pinned(): + # Escapes another language could write differently: control characters, + # DEL, quotes, backslashes, an astral character, U+2028. mgmt recomputes + # this digest in Rust (pipette-plan-types `vlm_eval_corpus`) and pins the + # same value, so a corpus it builds is checked against the scorer's. + sample = EvalSample( + id="c3", + messages=[ChatMessage(role="user", content='tab\there\u0001\u001f\u007f \\ "q" / é 😀 \u2028\r\n\u0008\u000c')], + ) + assert samples_digest([sample]) == "sha256:398cc6ebdc6f48b76c7bc670f047e4f82df20b1443fa0a30b2926808c29238f6" + + +def test_the_digest_follows_what_is_served(): + reworded = [SAMPLES[0], EvalSample(id="b2", messages=[ChatMessage(role="user", content="Is there a cat?")])] + assert len({samples_digest(SAMPLES), samples_digest(reworded), samples_digest(SAMPLES[::-1])}) == 3 + + +def test_an_empty_set_has_a_digest(): + assert samples_digest([]) == f"sha256:{hashlib.sha256(b'').hexdigest()}" + + +_AUDIO = {"type": "input_audio", "input_audio": {"data": "UklGRiQAAABXQVZF", "format": "wav"}} +# An ASR sample's shape: an input_audio part (base64 WAV) and the optional +# chat_template_kwargs, present only when a profile sets them. +ASR_SAMPLES = [ + EvalSample( + id="d4", + messages=[ChatMessage(role="user", content=[_AUDIO])], + chat_template_kwargs={"transcription": True, "language": "nl"}, + ), + EvalSample( + id="e5", + messages=[ + ChatMessage(role="system", content="Perform ASR."), + ChatMessage(role="user", content=[{"type": "text", "text": "Transcribe."}, _AUDIO]), + ], + ), +] + + +def test_an_asr_sample_is_pinned(): + # mgmt's Rust digest pins this value too, so an ASR corpus it packs is + # checked against the scorer's. + canonical = ( + '{"chat_template_kwargs":{"language":"nl","transcription":true},"id":"d4","messages":[{"content":[' + '{"input_audio":{"data":"UklGRiQAAABXQVZF","format":"wav"},"type":"input_audio"}],"role":"user"}]}\n' + '{"id":"e5","messages":[{"content":"Perform ASR.","role":"system"},{"content":[{"text":"Transcribe.",' + '"type":"text"},{"input_audio":{"data":"UklGRiQAAABXQVZF","format":"wav"},"type":"input_audio"}],' + '"role":"user"}]}\n' + ) + digest = samples_digest(ASR_SAMPLES) + assert digest == f"sha256:{hashlib.sha256(canonical.encode()).hexdigest()}" + assert digest == "sha256:b49fae68dfd0d735e59dc3df621d8b6189ec7977fc5830b54b368bd10f396ef7" diff --git a/packages/pipette-scores/tests/test_ifbench.py b/packages/pipette-scores/tests/test_ifbench.py index d4e3a53..b567ffe 100644 --- a/packages/pipette-scores/tests/test_ifbench.py +++ b/packages/pipette-scores/tests/test_ifbench.py @@ -1,4 +1,5 @@ import math +import sys import pytest @@ -155,6 +156,8 @@ def test_score_context_keys(): "ifbench_pass_at_1", "ifbench_pass_at_1_stderr", "ifbench_pass_at_1_per_sample", + "ifbench_blanked_truncated", + "ifbench_pass_at_1_blanked", "strict/count", # tier0: category prefix "loose/count", "strict/count:keywords_multiple", # tier1: full instruction id @@ -246,3 +249,131 @@ def test_pass_at_1_repeats_one_transparent(): _, context = score(completions, samples) assert context["ifbench_pass_at_1"] == 1.0 assert context["ifbench_pass_at_1_per_sample"] == {"1": 1.0} + + +# --- behaviour of the checkers at the liquid_evals revision (vendor/ifbench-checkers) --- + + +def test_curly_quotes_normalized_like_liquid_evals(): + # liquid_evals' checker maps curly quotes to '"' before looking for two + # quoted phrases in a row; the newer allenai checker lost that step. + sample = _sample("q", ["format:quote_unquote"], "Explain every quote.") + completions = [SampleCompletion(id="q", completion="He said “hello”“world” and left.")] + scores, _ = score(completions, [sample]) + assert scores[0].is_correct is False + + +def test_repeat_span_n_start_zero_is_deterministic(): + # Upstream treats n_start=0 as unset and picks a random start on every call; + # the vendored copy uses 0 as given, so the verdict never changes. + words = " ".join(f"w{i}" for i in range(20)) + sample = _sample( + "r", ["repeat:repeat_span"], "Repeat a span.", kwargs=[{"prompt_to_repeat": words, "n_start": 0, "n_end": 3}] + ) + completions = [SampleCompletion(id="r", completion="w0 w1 w2")] + verdicts = {score(completions, [sample])[0][0].is_correct for _ in range(20)} + assert verdicts == {True} + + +@pytest.mark.parametrize( + "instruction_id, kwargs, completion", + [ + ("ratio:stop_words", {"percentage": 20}, "..."), # ZeroDivisionError upstream + ("ratio:stop_words", {"percentage": 20}, "😀😀"), + ("words:last_first", {}, "Wow!! Wow is great."), # IndexError upstream + # ZeroDivisionError under 3 characters; liquid_evals' copy returns False. + ("ratio:overlap", {"reference_text": "abcdefgh", "percentage": 50}, "ab"), + ], +) +def test_checker_exception_scores_as_not_followed(instruction_id, kwargs, completion): + sample = _sample("x", [instruction_id], "Follow the constraint.", kwargs=[kwargs]) + scores, context = score([SampleCompletion(id="x", completion=completion)], [sample]) + assert scores[0].is_correct is False + assert context["strict_instruction_accuracy"] == 0.0 + + +@pytest.mark.parametrize( + "instruction_id, kwargs, completion", + [ + # The full text raises; dropping the first line leaves a passing variant. + ("words:last_first", {}, "Wow!!\nWow is great. Great job."), + ("ratio:overlap", {"reference_text": "the quick brown fox", "percentage": 100}, "the quick brown fox\nab"), + ], +) +def test_loose_tries_other_variants_after_a_check_raises(instruction_id, kwargs, completion): + sample = _sample("x", [instruction_id], "Follow the constraint.", kwargs=[kwargs]) + scores, context = score([SampleCompletion(id="x", completion=completion)], [sample]) + assert scores[0].is_correct is True + assert context["strict_prompt_accuracy"] == 0.0 + + +def _upstream_module(name): + return sys.modules[f"pipette_scores.scoring.ifbench._upstream.{name}"] + + +def test_missing_nltk_data_raises_instead_of_failing_the_check(monkeypatch): + # nltk reports missing data as a bare LookupError; scoring it as "not + # followed" would silently lower the score, so it must propagate. + checker = _upstream_module("instructions_registry").INSTRUCTION_DICT["ratio:stop_words"] + + def missing(self, value): + raise LookupError("Resource stopwords not found.") + + monkeypatch.setattr(checker, "check_following", missing) + sample = _sample("x", ["ratio:stop_words"], "Few stop words.", kwargs=[{"percentage": 50}]) + with pytest.raises(LookupError): + score([SampleCompletion(id="x", completion="Cats chase mice.")], [sample]) + + +def test_nltk_resource_that_cannot_be_downloaded_fails_import(monkeypatch): + util = _upstream_module("instructions_util") + + def not_found(path): + raise LookupError(path) + + monkeypatch.setattr(util.nltk.data, "find", not_found) + monkeypatch.setattr(util.nltk, "download", lambda *a, **k: False) + with pytest.raises(RuntimeError, match="could not be downloaded"): + util.download_nltk_resources() + + +def test_nltk_tokenizer_matches_pinned_version(): + # Title case goes through nltk.word_tokenize, which splits dashes into their + # own tokens from nltk 3.9.4 (the version that the internal liquid_evals + # environments lock). Before 3.9.4, "Hello—world" is one title-case token. + sample = _sample("t", ["format:title_case"], "Write in title case.") + passing, _ = score([SampleCompletion(id="t", completion="Hello World")], [sample]) + dashed, _ = score([SampleCompletion(id="t", completion="Hello—world")], [sample]) + assert passing[0].is_correct is True + assert dashed[0].is_correct is False + + +@pytest.mark.parametrize( + "completion", + [ + # Cohere North: the reasoning names the keywords too, which would break the counts. + f"One banana, two apple, three cherry.<|END_THINKING|><|START_TEXT|>{_PASS}<|END_TEXT|>", + # Gemma 4 on llama.cpp. + f"<|channel>thought\nOne banana, two apple.{_PASS}", + ], +) +def test_reasoning_markup_stripped(completion): + samples = [_sample("1", ["count:keywords_multiple"], _PROMPT, kwargs=[_KW])] + scores, _ = score([SampleCompletion(id="1", completion=completion)], samples) + assert scores[0].is_correct is True + + +@pytest.mark.parametrize("stop_reason", ["truncated", "doom_loop"]) +@pytest.mark.parametrize("enable_thinking, blanked", [(True, 1), (False, 0), (None, 0)]) +def test_cut_reasoning_is_graded_raw_and_counted(stop_reason, enable_thinking, blanked): + # The prompt opened the reasoning and generation was cut after a draft: + # graded whole, as liquid_evals grades it. + completions = [SampleCompletion(id="1", completion=f"Draft: {_PASS}", stop_reason=stop_reason)] + samples = [_sample("1", ["count:keywords_multiple"], _PROMPT, kwargs=[_KW])] + scores, ctx = score(completions, samples, enable_thinking=enable_thinking) + assert scores[0].is_correct is True + assert ctx["ifbench_pass_at_1"] == 1.0 + assert ctx["loose_prompt_accuracy"] == 1.0 + # The strict view counts it wrong. + assert ctx["ifbench_blanked_truncated"] == blanked + assert ctx["ifbench_pass_at_1_blanked"] == (0.0 if blanked else 1.0) diff --git a/packages/pipette-scores/tests/test_ifstruct.py b/packages/pipette-scores/tests/test_ifstruct.py index ec40987..f7feae4 100644 --- a/packages/pipette-scores/tests/test_ifstruct.py +++ b/packages/pipette-scores/tests/test_ifstruct.py @@ -1,3 +1,5 @@ +import pytest + from pipette_scores.scoring.ifstruct import score from pipette_scores.types import IFStructSample, SampleCompletion @@ -114,5 +116,72 @@ def test_score_context_shape_matches_build_summary(): "by_top_level_structure", "by_entity_type", "common_errors", + "ifstruct_blanked_truncated", + "ifstruct_pass_at_1_blanked", } assert expected_keys <= set(context), context + + +@pytest.mark.parametrize( + "completion", + [ + # Cohere North on vLLM, thinking off: its text section wraps the JSON. + '<|START_TEXT|>{"name": "Alice"}<|END_TEXT|>', + # North, thinking on. + 'Only a name is needed.<|END_THINKING|><|START_TEXT|>{"name": "Alice"}<|END_TEXT|>', + # ERNIE-4.5-Thinking: the prompt opened , the answer is wrapped. + 'Only a name is needed.\n\n\n\n{"name": "Alice"}\n\n', + # ERNIE behind vLLM's ernie45 parser: the reasoning is gone, the wrapper stays. + '\n\n\n```json\n{"name": "Alice"}\n```\n\n', + # Gemma 4 on llama.cpp. + '<|channel>thought\nOnly a name is needed.{"name": "Alice"}', + ], +) +def test_reasoning_markup_is_not_commentary(completion): + samples = [_sample("1", require_no_commentary=True)] + scores, _ = score([SampleCompletion(id="1", completion=completion)], samples) + assert scores[0].is_correct is True + + +def test_reasoning_is_stripped_once(): + # The validator strips thinking tags itself. Stripping before it as well + # would cut again at the inside the JSON string. + completions = [SampleCompletion(id="1", completion='Plan.{"name": "a b"}')] + scores, _ = score(completions, [_sample("1")]) + assert scores[0].is_correct is True + + +@pytest.mark.parametrize("stop_reason", ["truncated", "doom_loop"]) +@pytest.mark.parametrize("enable_thinking, blanked", [(True, 1), (False, 0), (None, 0)]) +def test_cut_reasoning_is_graded_raw_and_counted(stop_reason, enable_thinking, blanked): + # The prompt opened the reasoning and generation was cut after a draft: + # graded whole, as liquid_evals grades it. + completion = 'Draft:\n```json\n{"name": "Alice"}\n```\nNow check whether the' + completions = [SampleCompletion(id="1", completion=completion, stop_reason=stop_reason)] + scores, context = score(completions, [_sample("1")], enable_thinking=enable_thinking) + assert scores[0].is_correct is True + assert context["pass_rate"] == 1.0 + # The strict view counts it wrong. + assert context["ifstruct_blanked_truncated"] == blanked + assert context["ifstruct_pass_at_1_blanked"] == (0.0 if blanked else 1.0) + + +def test_cut_reasoning_opened_in_text_fails(): + # The completion opened itself: everything from the tag on goes. + completion = 'Draft:\n```json\n{"name": "Alice"}\n```\nNow check whether the' + completions = [SampleCompletion(id="1", completion=completion, stop_reason="truncated")] + scores, context = score(completions, [_sample("1")], enable_thinking=True) + assert scores[0].is_correct is False + assert context["ifstruct_blanked_truncated"] == 1 + assert context["ifstruct_pass_at_1_blanked"] == 0.0 + + +def test_runaway_nesting_fails_only_its_own_completion(): + samples = [_sample("1"), _sample("2")] + completions = [ + SampleCompletion(id="1", completion='{"name": "Alice"}'), + SampleCompletion(id="2", completion='{"name": [' + '{"name": [' * 20000), + ] + scores, context = score(completions, samples) + assert {s.id: s.is_correct for s in scores} == {"1": True, "2": False} + assert context["common_errors"].get("nesting too deep") == 1, context diff --git a/packages/pipette-scores/tests/test_math_500.py b/packages/pipette-scores/tests/test_math_500.py index d4ddc58..abe5761 100644 --- a/packages/pipette-scores/tests/test_math_500.py +++ b/packages/pipette-scores/tests/test_math_500.py @@ -1,7 +1,7 @@ import pytest from pipette_scores.scoring.math_500 import score -from pipette_scores.types import Math500Sample, SampleCompletion +from pipette_scores.types import Math500Grader, Math500Sample, SampleCompletion def _sample(id, answer): @@ -15,6 +15,9 @@ def _sample(id, answer): ("\\boxed{7}", "42", False), # reasoning is stripped before grading, so the in-think boxed value is ignored ("\\boxed{7} So the answer is \\boxed{42}", "42", True), + # same for Cohere North's markers; a Gemma 4 thought cut off is all reasoning + ("\\boxed{7}?<|END_THINKING|><|START_TEXT|>\\boxed{42}<|END_TEXT|>", "42", True), + ("<|channel>thought\nSo \\boxed{42}, but check the sign", "42", False), ("no boxed answer at all", "42", False), ], ) @@ -56,6 +59,8 @@ def test_context_schema_stable_regardless_of_availability(): assert isinstance(empty_ctx["math_500_pass_at_1"], float) assert isinstance(empty_ctx["math_500_pass_at_1_stderr"], float) assert isinstance(empty_ctx["math_500_unparsed_rate"], float) + assert empty_ctx["math_500_blanked_truncated"] == 0 + assert isinstance(empty_ctx["math_500_pass_at_1_blanked"], float) assert isinstance(empty_ctx["math_500_pass_at_1_per_sample"], dict) for method in ["prm800k", "robust_boxed", "no_match", "failed"]: assert empty_ctx[f"math_500_method/{method}"] == 0 @@ -71,3 +76,135 @@ def test_context_method_diagnostics(): # one of two had no extractable answer -> distinguishes parse-miss from wrong answer assert ctx["math_500_unparsed_rate"] == pytest.approx(0.5) assert ctx["math_500_method/prm800k"] == 1 + + +@pytest.mark.parametrize("stop_reason", ["truncated", "doom_loop"]) +@pytest.mark.parametrize("enable_thinking, blanked", [(True, 1), (False, 0), (None, 0)]) +def test_cut_reasoning_is_graded_raw_and_counted(stop_reason, enable_thinking, blanked): + samples = [_sample("m1", "42"), _sample("m2", "42")] + completions = [ + # The prompt opened the reasoning and generation was cut: graded whole, + # as liquid_evals grades it. + SampleCompletion(id="m1", completion="So \\boxed{42}? Check the sign of", stop_reason=stop_reason), + # The reasoning closed before the cut. + SampleCompletion(id="m2", completion="So 42.The answer is \\boxed{42}. Also", stop_reason=stop_reason), + ] + scores, ctx = score(completions, samples, enable_thinking=enable_thinking) + assert [s.is_correct for s in scores] == [True, True] + assert ctx["math_500_pass_at_1"] == 1.0 + assert ctx["math_500_unparsed_rate"] == 0.0 + # The strict view counts m1 wrong. + assert ctx["math_500_blanked_truncated"] == blanked + assert ctx["math_500_pass_at_1_blanked"] == (0.5 if blanked else 1.0) + + +# --- liquid_evals_math500 grader (dataset 2026.10.1) --- + + +def _le_sample(id, answer): + return Math500Sample(id=id, prompt=f"Problem {id}", answer=answer, grader=Math500Grader.LIQUID_EVALS_MATH500) + + +def _filler(n): + return "\n".join(f"step {i} of the work" for i in range(n)) + + +@pytest.mark.parametrize( + "completion, answer, expected, extraction", + [ + ("So \\boxed{42}.", "42", True, "boxed"), + ("So \\boxed{41}.", "42", False, "boxed"), + # nested braces and nested boxes: the scan keeps the outer box's content + ("\\boxed{\\frac{\\sqrt{3}}{2}}", "\\frac{\\sqrt{3}}{2}", True, "boxed"), + ("\\boxed{\\boxed{7}}", "7", True, "boxed"), + # \frac spellings and decimals are equivalent; an unreduced fraction is not + ("\\boxed{\\dfrac{1}{2}}", "\\frac12", True, "boxed"), + ("\\boxed{0.5}", "\\frac{1}{2}", True, "boxed"), + ("\\boxed{2/4}", "\\frac{1}{2}", False, "boxed"), + # units, degrees and dollars are dropped + ("\\boxed{5\\text{ cm}}", "5", True, "boxed"), + ("\\boxed{90^\\circ}", "90", True, "boxed"), + ("\\boxed{\\$18.90}", "\\$18.90", True, "boxed"), + # no box: the other patterns, then the last number + ("The answer is 7", "7", True, "answer_is"), + ("Final answer: 5", "5", True, "answer_colon"), + ("It is 7 apples", "7", True, "numeric"), + ("no answer here", "7", False, "none"), + # nothing extracted, but a response line matches the text answer + ("Hmm.\nMonday", "\\text{Monday}", True, "none"), + # a wrong box is rescued by a bare line holding the answer ... + ("\\boxed{3}\n4", "4", True, "boxed"), + # ... within the last 150 lines, and not beyond + ("4\n" + _filler(148) + "\n\\boxed{3}", "4", True, "boxed"), + ("4\n" + _filler(149) + "\n\\boxed{3}", "4", False, "boxed"), + # reasoning is stripped first: a box inside it does not count + ("\\boxed{42}\nSo \\boxed{7}", "42", False, "boxed"), + ("\\boxed{7}\nSo \\boxed{42}", "42", True, "boxed"), + ], +) +def test_liquid_evals_single_sample(completion, answer, expected, extraction): + scores, ctx = score([SampleCompletion(id="m1", completion=completion)], [_le_sample("m1", answer)]) + assert scores[0].is_correct is expected + assert ctx["math_500_pass_at_1"] == (1.0 if expected else 0.0) + assert ctx[f"math_500_extraction/{extraction}"] == 1 + assert ctx["math_500_unparsed_rate"] == (1.0 if extraction == "none" else 0.0) + + +def test_liquid_evals_context_schema_stable(): + samples = [_le_sample("m1", "1"), _le_sample("m2", "2")] + empty_ctx = score([], samples)[1] + populated_ctx = score([SampleCompletion(id="m1", completion="\\boxed{1}")], samples)[1] + assert empty_ctx.keys() == populated_ctx.keys() + for key, value in empty_ctx.items(): + assert type(value) is type(populated_ctx[key]), key + # the headline keys match math_generic's; only the per-grader counts differ + generic_ctx = score([], [])[1] + shared = {k for k in generic_ctx if not k.startswith("math_500_method/")} + assert shared == {k for k in empty_ctx if not k.startswith("math_500_extraction/")} + for bucket in ["boxed", "hash", "answer_colon", "answer_is", "other", "numeric", "none"]: + assert empty_ctx[f"math_500_extraction/{bucket}"] == 0 + assert not any(k.startswith("math_500_method/") for k in empty_ctx) + assert populated_ctx["math_500_pass_at_1"] == 0.5 + + +def test_liquid_evals_cut_reasoning_is_graded_raw_and_counted(): + samples = [_le_sample("m1", "42"), _le_sample("m2", "42")] + completions = [ + SampleCompletion(id="m1", completion="So \\boxed{42}? Check the sign of", stop_reason="truncated"), + SampleCompletion(id="m2", completion="So 42.The answer is \\boxed{42}. Also", stop_reason="truncated"), + ] + scores, ctx = score(completions, samples, enable_thinking=True) + assert [s.is_correct for s in scores] == [True, True] + assert ctx["math_500_blanked_truncated"] == 1 + assert ctx["math_500_pass_at_1_blanked"] == 0.5 + + +def test_liquid_evals_pass_at_1_over_repeats_is_the_mean_of_the_passes(): + # 2026.10.1 serves 5 repeats. The model-card number is the mean of 5 math500 + # runs of one pass each; pass@1 over every attempt equals that mean. + gold = {"a": "1", "b": "2", "c": "3"} + right = {("a", 0), ("a", 1), ("a", 2), ("a", 3), ("a", 4), ("b", 0), ("b", 3), ("c", 2)} + samples = [_le_sample(f"{lid}#{k}", answer) for lid, answer in gold.items() for k in range(5)] + completions = [ + SampleCompletion(id=f"{lid}#{k}", completion=f"\\boxed{{{answer if (lid, k) in right else 9}}}") + for lid, answer in gold.items() + for k in range(5) + ] + _, ctx = score(completions, samples) + per_pass = [sum((lid, k) in right for lid in gold) / len(gold) for k in range(5)] + assert ctx["math_500_pass_at_1"] == pytest.approx(sum(per_pass) / 5) + assert ctx["math_500_pass_at_1_per_sample"] == {"a": 1.0, "b": 0.4, "c": 0.2} + + +def test_samples_of_two_graders_are_rejected(): + samples = [_sample("m1", "1"), _le_sample("m2", "1")] + with pytest.raises(ValueError, match="more than one grader"): + score([], samples) + + +def test_default_grader_is_math_generic(): + # A sample without a grader (2026.06.1's metadata has none) keeps score_math_generic. + assert _sample("m1", "1").grader is Math500Grader.MATH_GENERIC + _, ctx = score([SampleCompletion(id="m1", completion="\\boxed{1}")], [_sample("m1", "1")]) + assert ctx["math_500_method/prm800k"] == 1 + assert not any(k.startswith("math_500_extraction/") for k in ctx) diff --git a/packages/pipette-scores/tests/test_math_500_liquid_evals_parity.py b/packages/pipette-scores/tests/test_math_500_liquid_evals_parity.py new file mode 100644 index 0000000..2126f13 --- /dev/null +++ b/packages/pipette-scores/tests/test_math_500_liquid_evals_parity.py @@ -0,0 +1,66 @@ +"""The vendored liquid_evals MATH-500 grader gives the original's outputs. + +The expectations in fixtures/math_500_liquid_evals_parity.json were produced by +running the ORIGINAL liquid_evals/parser.py (the commit is recorded in the +fixture) with scripts/gen_math_500_liquid_evals_parity.py. Regenerate them with +that script after updating the vendored copy. +""" + +import json +import pathlib + +import pytest + +from pipette_scores.scoring.completion_text import remove_thinking_tags +from pipette_scores.scoring.math_500 import liquid_evals_parser as le +from pipette_scores.scoring.math_500 import score +from pipette_scores.types import Math500Grader, Math500Sample, SampleCompletion + +_FIXTURE = json.loads((pathlib.Path(__file__).parent / "fixtures" / "math_500_liquid_evals_parity.json").read_text()) + + +def _short(case: dict, key: str) -> str: + return repr(case[key][:60]) + + +def test_fixture_is_from_the_vendored_commit(): + assert _FIXTURE["source"]["commit"] in le.__doc__ + assert _FIXTURE["source"]["blob"] in le.__doc__ + + +@pytest.mark.parametrize("case", _FIXTURE["extract"], ids=lambda c: _short(c, "text")) +def test_extract_math_answer(case): + assert le.extract_math_answer(case["text"]) == case["expected"] + + +@pytest.mark.parametrize("case", _FIXTURE["check"], ids=lambda c: f"{_short(c, 'given')} vs {_short(c, 'truth')}") +def test_check_math_result(case): + given, truth = case["given"], case["truth"] + assert le.mathd_normalize_answer(given) == case["mathd_normalize_given"] + assert le._normalize(given) == case["normalize_given"] + assert le.grade_answer_mathd(given, truth) is case["mathd"] + assert le.grade_answer_sympy(given, truth) is case["sympy"] + assert le.check_math_result(given, truth) is case["check"] + + +@pytest.mark.parametrize("case", _FIXTURE["verify"], ids=lambda c: f"{_short(c, 'raw')} vs {_short(c, 'gold')}") +def test_math500_task_pipeline(case): + # tasks/math500.py: remove_thinking_tags, extract_math_answer, verify_math_answer. + response = remove_thinking_tags(case["raw"]) + predicted = le.extract_math_answer(response) + assert predicted == case["predicted"] + assert le.verify_math_answer(case["gold"], predicted, response) is case["correct"] + + +def test_scorer_matches_the_task_verdicts(): + # The verify cases carry no markup that normalize_reasoning_markup rewrites, + # so the scorer must give the task's verdict on every one. + cases = _FIXTURE["verify"] + samples = [ + Math500Sample(id=f"m{i}", prompt=f"P{i}", answer=c["gold"], grader=Math500Grader.LIQUID_EVALS_MATH500) + for i, c in enumerate(cases) + ] + completions = [SampleCompletion(id=f"m{i}", completion=c["raw"]) for i, c in enumerate(cases)] + scores, ctx = score(completions, samples) + assert [s.is_correct for s in scores] == [c["correct"] for c in cases] + assert ctx["math_500_pass_at_1"] == pytest.approx(sum(c["correct"] for c in cases) / len(cases)) diff --git a/packages/pipette-scores/tests/test_packages.py b/packages/pipette-scores/tests/test_packages.py new file mode 100644 index 0000000..9263b85 --- /dev/null +++ b/packages/pipette-scores/tests/test_packages.py @@ -0,0 +1,303 @@ +"""The eval packages: mgmt's tar bytes, the refusals, the command and the store.""" + +import hashlib +import io +import json +import pathlib +import tarfile +from unittest.mock import MagicMock + +import boto3 +import numpy as np +import pyarrow as pa +import pyarrow.parquet as pq +import pytest +import soundfile as sf +from botocore.exceptions import ClientError + +from pipette_scores import cli, dataset_catalog, packages +from pipette_scores.dataset_catalog import _asr_build, asr_datasets, asr_prompt_profiles, prompt_profiles +from pipette_scores.hashing import samples_digest +from pipette_scores.types import ChatMessage, EvalId, EvalSample +from test_hashing import ASR_SAMPLES, SAMPLES + +_PNG = b"\x89PNG\r\n\x1a\n" # the golden samples' image, iVBORw0KGgo= + + +def _package(tmp_path: pathlib.Path, samples: list[EvalSample], eval_id: str, profile: str | None = None): + package = packages.Package(eval_id, "2026.10.1", profile, tmp_path) + for sample in samples: + package.add(sample) + return package + + +@pytest.mark.parametrize( + "samples, eval_id, profile, identity, digest", + [ + ( + SAMPLES, + "mmstar", + None, + "5654ef883c378c4325cfe70594545d4a19c094defdf21b32487f9de94a6b4849", + "cecf36b2e29ae61be4fe20df91f8f0e849a3b718e340e831b469b3fd7db90f73", + ), + ( + ASR_SAMPLES, + "asr_fleurs_nl_nl", + "voxtral", + "b49fae68dfd0d735e59dc3df621d8b6189ec7977fc5830b54b368bd10f396ef7", + "b36109aa457771c0273adefb6c467dab0a2c104b376818539da5eb9a3199f550", + ), + ], +) +def test_the_golden_samples_pack_to_the_tar_mgmt_pins(tmp_path, samples, eval_id, profile, identity, digest): + # mgmt's plan-types and format::write_corpus pack the same samples to these bytes. + package = _package(tmp_path, samples, eval_id, profile) + tar = b"".join(package.tar()) + assert package.identity == identity + assert (hashlib.sha256(tar).hexdigest(), len(tar)) == (digest, 4096) + + +def test_a_package_reads_back_as_a_tar(tmp_path): + jpeg = {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,/9j/4AAQ"}} + png = {"type": "image_url", "image_url": {"url": "data:image/png;base64,iVBORw0KGgo="}} + samples = [*SAMPLES, EvalSample(id="c3", messages=[ChatMessage(role="user", content=[jpeg, png])])] + package = _package(tmp_path, samples, "mmstar") + with tarfile.open(fileobj=io.BytesIO(b"".join(package.tar()))) as tar: + members = tar.getmembers() + files = {m.name: tar.extractfile(m).read() for m in members} + names = [m.name for m in members] + assert names[0] == "manifest.json" and names[-1] == "samples.jsonl" and names == sorted(names) + assert {(m.mode, m.uid, m.gid, m.mtime) for m in members} == {(0o644, 0, 0, 0)} + assert json.loads(files["manifest.json"]) == { + "format": "vlm-eval-corpus/1", + "adapter": "evals-server@1", + "upstream": { + "source": "evals_server", + "eval_id": "mmstar", + "dataset_version": "2026.10.1", + "identity": package.identity, + }, + "samples": 3, + "media": 2, + } + # The PNG two samples share is one file; each file is named by its sha256. + media = {name: body for name, body in files.items() if name.startswith("media/")} + assert sorted(media) == sorted( + f"media/{hashlib.sha256(body).hexdigest()}.{ext}" + for body, ext in [(_PNG, "png"), (b"\xff\xd8\xff\xe0\x00\x10", "jpg")] + ) + lines = [json.loads(line) for line in files["samples.jsonl"].splitlines()] + assert [line["id"] for line in lines] == ["a1", "b2", "c3"] + images = [p for line in lines for m in line["messages"] if isinstance(m["content"], list) for p in m["content"]] + images = [p for p in images if p["type"] == "image"] + assert [p["bytes"] for p in images] == [len(media[p["path"]]) for p in images] == [8, 6, 8] + + +@pytest.mark.parametrize( + "eval_id, part, refusal", + [ + ("mmstar", {"type": "image_url", "image_url": {"url": "data:image/png;base64,iVBORw0KGgp="}}, "canonical"), + ("mmstar", {"type": "image_url", "image_url": {"url": "data:image/png;base64,iVBO Rw0KGgo="}}, "canonical"), + ("mmstar", {"type": "image_url", "image_url": {"url": "https://example.test/a.png"}}, "is not data:image"), + ("mmstar", {"type": "image_url", "image_url": {"url": "data:image/svg+xml;base64,PHN2Zz4="}}, "is not data:"), + ("mmstar", {"type": "image_url", "image_url": {"url": "data:image/png;base64,", "detail": "low"}}, "no place"), + ("mmstar", {"type": "text", "text": "hi", "cache_control": {"type": "ephemeral"}}, "no place"), + ("mmstar", {"type": "video_url", "video_url": {"url": "data:video/mp4;base64,"}}, "no place"), + ("mmstar", {"type": "input_audio", "input_audio": {"data": "UklGRiQAAABXQVZF", "format": "wav"}}, "no place"), + ( + "asr_fleurs_nl_nl", + {"type": "input_audio", "input_audio": {"data": "ZkxhQw==", "format": "flac"}}, + "wav only", + ), + ( + "asr_fleurs_nl_nl", + {"type": "input_audio", "input_audio": {"data": "UklGRiQAAABXQVZ", "format": "wav"}}, + "canonical", + ), + ], +) +def test_a_part_that_would_not_rebuild_is_refused(tmp_path, eval_id, part, refusal): + sample = EvalSample(id="x", messages=[ChatMessage(role="user", content=[part])]) + with pytest.raises(ValueError, match=refusal): + _package(tmp_path, [sample], eval_id) + + +def test_a_set_with_an_id_twice_or_no_samples_is_refused(tmp_path): + with pytest.raises(ValueError, match="b2 is served twice"): + _package(tmp_path, [SAMPLES[1], SAMPLES[1]], "mmstar") + with pytest.raises(ValueError, match="no samples"): + _package(tmp_path, [], "mmstar").save(packages.LocalStore(tmp_path / "out")) + assert not (tmp_path / "out").exists() + + +def test_a_path_ustar_cannot_hold_is_refused(): + assert len(packages._header("a" * 100, 8**11 - 1)) == 512 + for path, size in [("a" * 101, 0), ("média", 0), ("a", 8**11)]: + with pytest.raises(ValueError, match="does not fit a ustar header"): + packages._header(path, size) + + +# --------------------------------------------------------------------------- +# The command, into a local directory +# --------------------------------------------------------------------------- + + +def _vlm_dataset(root: pathlib.Path, eval_id: str, name: str) -> None: + rows = [ + { + "sample_id": f"s{i}", + "prompt": question, + "answer": json.dumps("yes"), + "raw_doc": json.dumps({"question": question}), + "num_images": 1, + "image_0": _PNG, + } + for i, question in enumerate(["Is there a dog?", "Is there a cat?"]) + ] + (root / "datasets" / eval_id / name).mkdir(parents=True) + pq.write_table(pa.Table.from_pylist(rows), root / "datasets" / eval_id / name / "test.parquet") + + +def _asr_dataset(root: pathlib.Path, name: str) -> None: + def wav(seconds: float) -> bytes: + buffer = io.BytesIO() + sf.write(buffer, np.full(int(seconds * 16000), 0.25, dtype=np.float32), 16000, format="WAV", subtype="PCM_16") + return buffer.getvalue() + + rows = [ + { + "audio": wav(1 + i / 10), + "audio_format": "wav", + "reference": reference, + "session": None, + "chunk_index": None, + "raw_doc": {"id": f"u{i}"}, + } + for i, reference in enumerate(["een", "twee"]) + ] + _asr_build.build(asr_datasets.ASR_DATASETS[EvalId.ASR_FLEURS_NL_NL], root / "datasets", name, rows=iter(rows)) + + +def _cli(monkeypatch, *argv: str) -> None: + monkeypatch.setattr("sys.argv", ["pipette-scores", *argv]) + cli.main() + + +def test_the_command_packs_each_dataset_once(tmp_path, monkeypatch): + data, out = tmp_path / "data", tmp_path / "out" + _vlm_dataset(data, "pope", "2026.10.1") + _asr_dataset(data, "test-1") + (data / "datasets" / "mmstar" / "broken").mkdir(parents=True) + (data / "datasets" / "mmstar" / "broken" / "test.parquet").write_bytes(b"not parquet") + built, build = [], packages.build + + def spy(eval_id, name, *rest): + built.append((eval_id, name)) + build(eval_id, name, *rest) + + monkeypatch.setattr(packages, "build", spy) + monkeypatch.delenv("PIPETTE_SCORES_VERSION", raising=False) + argv = ["packages", "--datasets", str(data), "--packages", str(out)] + for version in [["--version", "dev"], []]: + with pytest.raises(SystemExit, match="not dev"): + _cli(monkeypatch, *argv, *version) + + # A dataset that fails fails the run, once the others are packed. + with pytest.raises(SystemExit, match="1 datasets failed: mmstar/broken"): + _cli(monkeypatch, *argv, "--version", "v1") + assert built == [("mmstar", "broken"), ("pope", "2026.10.1"), ("asr_fleurs_nl_nl", "test-1")] + pope = json.loads((out / "index/v1/pope/2026.10.1.json").read_text()) + asr = json.loads((out / "index/v1/asr_fleurs_nl_nl/test-1.json").read_text()) + assert [e["prompt_profile"] for e in pope["profiles"]] == [None, *prompt_profiles.PROFILES] + assert [e["prompt_profile"] for e in asr["profiles"]] == list(asr_prompt_profiles.PROFILES) + # Each digest is the one /samples serves; the samples' one image is stored once. + for entry in pope["profiles"]: + served = dataset_catalog.load_prompt_samples.__wrapped__("pope", "2026.10.1", data, entry["prompt_profile"]) + assert (entry["samples_digest"], entry["sample_count"], entry["package"]["media"]) == ( + samples_digest(served), + 2, + 1, + ) + for entry in asr["profiles"]: + streamed = dataset_catalog.iter_asr_prompt_samples( + EvalId.ASR_FLEURS_NL_NL, "test-1", entry["prompt_profile"], root_dir=data + ) + assert (entry["samples_digest"], entry["sample_count"]) == (samples_digest(streamed), 2) + packed = {e["package"]["digest"]: e["package"]["bytes"] for e in pope["profiles"] + asr["profiles"]} + tars = {tar.name: tar.read_bytes() for tar in (out / "tars").iterdir()} + assert {name: (hashlib.sha256(tar).hexdigest(), len(tar)) for name, tar in tars.items()} == { + f"{digest}.tar": (digest, size) for digest, size in packed.items() + } + + # An indexed dataset is skipped; one packed again finds its tars there and leaves them be. + (data / "datasets" / "mmstar" / "broken" / "test.parquet").unlink() + (data / "datasets" / "mmstar" / "broken").rmdir() + (out / "index/v1/asr_fleurs_nl_nl/test-1.json").unlink() + stamps = {tar: (tar.stat().st_ino, tar.stat().st_mtime_ns) for tar in (out / "tars").iterdir()} + built.clear() + _cli(monkeypatch, *argv, "--version", "v1") + assert built == [("asr_fleurs_nl_nl", "test-1")] + assert {tar: (tar.stat().st_ino, tar.stat().st_mtime_ns) for tar in (out / "tars").iterdir()} == stamps + assert json.loads((out / "index/v1/asr_fleurs_nl_nl/test-1.json").read_text()) == asr + + +# --------------------------------------------------------------------------- +# The S3 store +# --------------------------------------------------------------------------- + + +def _s3(monkeypatch): + client = MagicMock() + client.exceptions.ClientError = ClientError + client.head_object.side_effect = ClientError({"Error": {"Code": "404"}}, "HeadObject") + client.create_multipart_upload.return_value = {"UploadId": "u"} + client.upload_part.side_effect = lambda **kw: {"ETag": f"e{kw['PartNumber']}", "ChecksumCRC32": "c"} + monkeypatch.setattr(boto3, "client", lambda service: client) + return packages.S3Store("s3://bucket/scores-packages"), client + + +def _refused(operation: str) -> ClientError: + return ClientError({"Error": {"Code": "PreconditionFailed"}}, operation) + + +def test_a_tar_goes_up_in_parts_completed_only_if_absent(monkeypatch): + monkeypatch.setattr(packages, "_PART_BYTES", 4) + store, client = _s3(monkeypatch) + ids = {"Bucket": "bucket", "Key": "scores-packages/tars/t.tar", "UploadId": "u"} + assert store.create("tars/t.tar", iter([b"abc", b"defgh", b"ij"])) + assert [c.kwargs["Body"] for c in client.upload_part.call_args_list] == [b"abcd", b"efgh", b"ij"] + parts = [{"PartNumber": n, "ETag": f"e{n}", "ChecksumCRC32": "c"} for n in (1, 2, 3)] + client.complete_multipart_upload.assert_called_once_with(**ids, MultipartUpload={"Parts": parts}, IfNoneMatch="*") + client.abort_multipart_upload.assert_not_called() + + # Written by another run meanwhile (412): there already, and the parts are given up. + client.complete_multipart_upload.side_effect = _refused("CompleteMultipartUpload") + assert not store.create("tars/t.tar", iter([b"abc"])) + client.abort_multipart_upload.assert_called_once_with(**ids) + + def broken(): + yield b"abcd" + raise OSError("read failed") + + client.abort_multipart_upload.reset_mock() + with pytest.raises(OSError, match="read failed"): + store.create("tars/t.tar", broken()) + client.abort_multipart_upload.assert_called_once_with(**ids) + + # There at the check: nothing is sent. + client.head_object.side_effect = None + client.create_multipart_upload.reset_mock() + assert not store.create("tars/t.tar", iter([b"abc"])) + client.create_multipart_upload.assert_not_called() + + +def test_an_index_is_put_once(monkeypatch): + store, client = _s3(monkeypatch) + assert store.create("index/v1/mme/2026.10.1.json", b"{}\n") + client.put_object.assert_called_once_with( + Bucket="bucket", Key="scores-packages/index/v1/mme/2026.10.1.json", Body=b"{}\n", IfNoneMatch="*" + ) + client.put_object.side_effect = _refused("PutObject") + assert not store.create("index/v1/mme/2026.10.1.json", b"{}\n") + client.create_multipart_upload.assert_not_called() diff --git a/packages/pipette-scores/tests/test_profile_answers.py b/packages/pipette-scores/tests/test_profile_answers.py new file mode 100644 index 0000000..e15578c --- /dev/null +++ b/packages/pipette-scores/tests/test_profile_answers.py @@ -0,0 +1,269 @@ +"""A profile's VLMEvalKit wrapper extraction (scoring/profile_answers.py).""" + +import json +from unittest.mock import patch + +import pytest +from fastapi.testclient import TestClient + +from pipette_scores import cli +from pipette_scores.api.app import app +from pipette_scores.scoring import profile_answers +from pipette_scores.scoring.completion_text import split_reasoning +from pipette_scores.types import ChatMessage, EvalId, EvalSample, SampleCompletion, VlmSample + + +@pytest.mark.parametrize( + ("benchmark", "reply", "answer"), + [ + # A reply that ends inside its reasoning still gives its stated answer. + (EvalId.REALWORLDQA, "\nThe sign says 35, so B.\nAnswer: B", "B"), + # After a closed reasoning block only the reply counts. + (EvalId.MMSTAR, "Answer: A, no wait\n\nAnswer: C", "C"), + # The last of several, any case, and upstream's other spellings. + (EvalId.MMBENCH_EN_DEV, "Answer: A ... so the Answer is d", "d"), + (EvalId.MMMU_VAL, "答案是B", "B"), + (EvalId.MMSTAR, "C", "C"), + # Yes/no keeps everything after the first Answer:. + (EvalId.POPE, "Is there a dog? No.\nAnswer: No", "No"), + (EvalId.MME, "\nlooking\nAnswer: Yes", "Yes"), + # Nothing to cut: the reply as upstream hands it on. + (EvalId.REALWORLDQA, "\nhmm\nIt is B", "It is B"), + (EvalId.MMSTAR, "\nno answer stated", "\nno answer stated"), + ], +) +def test_minicpm_v_cot_reads_the_answer_as_upstream_does(benchmark, reply, answer): + assert profile_answers.minicpm_v_cot(benchmark, reply) == answer + + +def _completions(text: str) -> list[SampleCompletion]: + return [SampleCompletion(id="1", completion=text, stop_reason="eos")] + + +@pytest.mark.parametrize( + ("profile", "benchmark"), + [ + # Another profile's wrapper keeps the reply whole, as does no profile. + ("lfm2_5_vl_3b", "realworldqa"), + (None, "realworldqa"), + # OCRBench is whole on every profile, as upstream leaves it. + ("minicpm_v_cot", "ocrbench"), + # A text eval is not a VLM benchmark. + ("minicpm_v_cot", "ifbench"), + ], +) +def test_other_replies_are_left_whole(profile, benchmark): + completions = _completions("\nAnswer: B") + assert profile_answers.apply(profile, benchmark, completions) == completions + + +def _rwqa(answer: str) -> VlmSample: + return VlmSample( + id="1", + prompt="Q?", + answer=answer, + num_images=1, + raw_doc={"A": "25", "B": "35", "C": "45", "answer": answer}, + ) + + +@pytest.mark.parametrize(("profile", "correct"), [("minicpm_v_cot", True), (None, False)]) +def test_score_grades_what_the_profiles_wrapper_hands_on(profile, correct): + """A MiniCPM-V reply that states its answer and ends inside its reasoning is + right under the profile, as under VLMEvalKit; without it the reasoning + rules drop the reply. The row keeps the reply as sent.""" + reply = "\nThe sign says 35, so B.\nAnswer: B" + body = { + "eval_id": "realworldqa", + "dataset_name": "test", + "completions": [{"id": "1", "completion": reply, "stop_reason": "eos"}], + } + if profile: + body["prompt_profile"] = profile + prompts = [EvalSample(id="1", messages=[ChatMessage(role="user", content="Q?")])] + with ( + patch("pipette_scores.api.app._load_dataset_cached", return_value=[_rwqa("B")]), + patch("pipette_scores.api.app._load_prompts_cached", return_value=prompts), + ): + resp = TestClient(app).post("/score", json=body) + assert resp.status_code == 200, resp.text + row = resp.json()["scored_samples"][0] + assert row["is_correct"] is correct + assert row["completion"] == reply + + +@pytest.mark.parametrize(("profile", "accuracy"), [("minicpm_v_cot", "100.00%"), (None, "0.00%")]) +def test_the_cli_grades_what_the_profiles_wrapper_hands_on(profile, accuracy, tmp_path, monkeypatch, capsys): + """The CLI grades as /score does.""" + completions = tmp_path / "completions.jsonl" + completions.write_text(json.dumps({"id": "1", "completion": "\nSo B.\nAnswer: B", "stop_reason": "eos"})) + argv = ["pipette-scores", "score", "--eval", "realworldqa", "--dataset", "test", "-f", str(completions)] + monkeypatch.setattr("sys.argv", [*argv, "--no-samples", *(["--prompt-profile", profile] if profile else [])]) + with patch("pipette_scores.dataset_catalog.load_eval_samples", return_value=[_rwqa("B")]): + cli.main() + assert accuracy in capsys.readouterr().out + + +# Replies from the b11744 smoke runs (Kimi-VL-A3B-Thinking-2506, Q4_K_M) to its MCQ and OCR questions, and the +# 2075 reply, which the server delivered starting mid-thought. +KIMI_MCQ = "\u25c1think\u25b7Got it, let's look at the image. The circle is red. So the answer is C.\u25c1/think\u25b7C" +KIMI_OCR = ( + "\u25c1think\u25b7So, let's look at the image. The text in the image is \"PIPELINE 4\". Let's check each part. " + "The letters are P, I, R, E, P, L, E, then 4. So combining them: PIPELINE 4.\u25c1/think\u25b7PIPELINE 4" +) +KIMI_NO_OPENER = '5"? Wait, no, no, wait. Wait, no, the number is 2075.\u25c1/think\u25b72075' + + +@pytest.mark.parametrize( + ("benchmark", "reply", "answer"), + [ + # The summary after the thought, on every benchmark. + (EvalId.REALWORLDQA, KIMI_MCQ, "C"), + (EvalId.OCRBENCH, KIMI_OCR, "PIPELINE 4"), + (EvalId.MME, "\u25c1think\u25b7Is it? Yes.\u25c1/think\u25b7 Yes\n", "Yes"), + (EvalId.POPE, KIMI_NO_OPENER, "2075"), + # A close after the answer began is a stray one: the text before it is reasoning, as liquid_evals reads it. + (EvalId.MMBENCH_EN_DEV, "\u25c1think\u25b7a\u25c1/think\u25b7B\u25c1/think\u25b7C", "C"), + # An unclosed thought is no answer, as upstream's extract_summary has it, however far the reply got. + (EvalId.MMSTAR, "\u25c1think\u25b7So the answer is C, but let me check", ""), + # Unless the model stopped with a complete ... in it: split_reasoning grades that. + (EvalId.MMSTAR, "\u25c1think\u25b7So C, but let me check", "C"), + (EvalId.REALWORLDQA, "C\u25c1think\u25b7wait", "C"), # the text before the opener stays, as for every family + # Unless the thought closes: the summary is only what follows the close, as extract_summary reads it. + (EvalId.MMSTAR, "A\u25c1think\u25b7Maybe A. No, it is B.\u25c1/think\u25b7Answer: $\\boxed{B}$", "B"), + (EvalId.REALWORLDQA, "A\u25c1think\u25b7Maybe A. No, B.\u25c1/think\u25b7B", "B"), + # No tags: the reply as it is. + (EvalId.REALWORLDQA, "It is B", "It is B"), + # The boxed content of the last \\boxed{}, on MMMU and MMStar only. + (EvalId.MMSTAR, "\u25c1think\u25b7Red.\u25c1/think\u25b7Answer: $\\boxed{C}$", "C"), + (EvalId.MMMU_VAL, "\u25c1think\u25b7x\u25c1/think\u25b7First \\boxed{A}, no: Answer: $\\boxed{B}$", "B"), + (EvalId.MMSTAR, "\u25c1think\u25b7x\u25c1/think\u25b7$\\boxed{\\frac{1}{2}}$", "\\frac{1}{2}"), + (EvalId.MMSTAR, "\u25c1think\u25b7x\u25c1/think\u25b7$\\boxed{\\text{C}}$", "\\text{C}"), + (EvalId.MMSTAR, "\u25c1think\u25b7x\u25c1/think\u25b7I think \\boxed{C", "I think \\boxed{C"), + (EvalId.REALWORLDQA, "\u25c1think\u25b7x\u25c1/think\u25b7Answer: $\\boxed{C}$", "Answer: $\\boxed{C}$"), + # A boxed answer inside the thought is not the answer. + (EvalId.MMSTAR, "\u25c1think\u25b7\\boxed{A}\u25c1/think\u25b7C", "C"), + ], +) +def test_kimi_vl_reads_the_answer_as_its_wrapper_does(benchmark, reply, answer): + # The extraction keeps the thought in the reply; the reasoning rules then cut it, as they do for every family. + extracted = profile_answers.kimi_vl(benchmark, reply) + graded = split_reasoning(SampleCompletion(id="1", completion=extracted, stop_reason="eos"), opens_reasoning=True) + assert graded.answer == answer + + +def test_apply_leaves_ocrbench_whole_on_every_profile(): + completions = _completions("\u25c1think\u25b7PIPETTE 42, wait") + for profile in ("kimi_vl", "minicpm_v_cot", "phi4_rv_think"): + assert profile_answers.apply(profile, "ocrbench", completions) == completions + + +def test_a_profile_without_an_extraction_leaves_the_new_profiles_siblings_whole(): + completions = _completions("\u25c1think\u25b7x\u25c1/think\u25b7C") + for profile in ("glm_v", "nemotron_think", "mimo_vl", "phi4_rv_think"): + assert profile_answers.apply(profile, "mmstar", completions) == completions + + +def _mmstar(answer: str) -> VlmSample: + return VlmSample( + id="1", + prompt="Q?", + answer=answer, + raw_doc={"A": "blue", "B": "green", "C": "red", "D": "black", "answer": answer, "l2_category": "x"}, + ) + + +def _post_score(profile, reply, *, stop_reason="eos", benchmark="mmstar", sample=None): + body = { + "eval_id": benchmark, + "dataset_name": "test", + "completions": [{"id": "1", "completion": reply, "stop_reason": stop_reason}], + } + if profile: + body["prompt_profile"] = profile + prompts = [EvalSample(id="1", messages=[ChatMessage(role="user", content="Q?")])] + with ( + patch("pipette_scores.api.app._load_dataset_cached", return_value=[sample or _mmstar("C")]), + patch("pipette_scores.api.app._load_prompts_cached", return_value=prompts), + ): + resp = TestClient(app).post("/score", json=body) + assert resp.status_code == 200, resp.text + _post_score.context = resp.json()["context"] + return resp.json()["scored_samples"][0] + + +@pytest.mark.parametrize( + ("profile", "reply", "correct"), + [ + # Kimi's real reply is graded on what follows its thought, with or without its profile. + ("kimi_vl", KIMI_MCQ, True), + (None, KIMI_MCQ, True), + # A reply with a final answer after the thought, whatever the thought says on the way. + ("kimi_vl", "\u25c1think\u25b7Maybe A? No, B? No.\u25c1/think\u25b7Answer: $\\boxed{C}$", True), + (None, "\u25c1think\u25b7Maybe A? No, B? No.\u25c1/think\u25b7Answer: $\\boxed{C}$", True), + # The thought is cut: the letter it mentions is not an answer. + ("kimi_vl", "\u25c1think\u25b7The answer is C, but wait", False), + (None, "\u25c1think\u25b7The answer is C, but wait", False), + # The wrong answer after a right thought is wrong. + ("kimi_vl", "\u25c1think\u25b7The answer is C.\u25c1/think\u25b7Answer: $\\boxed{B}$", False), + ], +) +def test_score_grades_kimi_replies_on_the_summary_after_their_thought(profile, reply, correct): + row = _post_score(profile, reply) + assert row["is_correct"] is correct + assert row["completion"] == reply + + +def test_score_grades_kimi_ocrbench_on_the_summary(): + sample = VlmSample( + id="1", + prompt="Q?", + answer=json.dumps(["PIPELINE 4"]), + raw_doc={"question": "What is written?", "answer": ["PIPELINE 4"], "category": "Regular Text Recognition"}, + ) + assert _post_score("kimi_vl", KIMI_OCR, benchmark="ocrbench", sample=sample)["is_correct"] is True + # A cut thought that quotes the text is not the answer. + cut = _post_score("kimi_vl", "\u25c1think\u25b7The text is PIPELINE 4, but", benchmark="ocrbench", sample=sample) + assert cut["is_correct"] is False + + +@pytest.mark.parametrize( + ("reply", "stop_reason", "correct", "blanked"), + [ + # What follows the prefill: the rest of the thought, the close, the solution. + ("The circle is red, so C.\nC", "eos", True, 0), + ("The circle is red, so C.\n\nAnswer: C", "eos", True, 0), + ("Not A, not B.\nA", "eos", False, 0), + # The solution cut short after the close is graded as it stands. + ("Red.\nAnswer: C. The circle", "truncated", True, 0), + # The model answered and stopped without closing the thought its prefill opened: graded as written. + ("C", "eos", True, 0), + # Cut inside the thought: graded whole, as liquid_evals grades a prompt-opened thought, and + # counted by the strict view, which grades it as empty. + ("The circle is red, so the answer is C. Let me double", "truncated", True, 1), + ("The circle is red, so the answer is B. Let me double", "doom_loop", False, 1), + ], +) +def test_score_grades_phi4_replies_that_follow_the_think_prefill(reply, stop_reason, correct, blanked): + row = _post_score("phi4_rv_think", reply, stop_reason=stop_reason) + assert row["is_correct"] is correct + assert row["completion"] == reply + assert _post_score.context["blanked_truncated"] == blanked + + +def test_without_the_profile_a_cut_thought_with_no_tag_is_not_known_to_be_reasoning(): + """The profile is what says a tagless reply was reasoning.""" + _post_score(None, "The circle is red, so the answer is C. Let me double", stop_reason="truncated") + assert _post_score.context["blanked_truncated"] == 0 + + +@pytest.mark.parametrize("profile", ["kimi_vl", "phi4_rv_think"]) +def test_the_cli_applies_the_new_extractions(profile, tmp_path, monkeypatch, capsys): + reply = {"kimi_vl": KIMI_MCQ, "phi4_rv_think": "Red.\nC"}[profile] + completions = tmp_path / "completions.jsonl" + completions.write_text(json.dumps({"id": "1", "completion": reply, "stop_reason": "eos"})) + argv = ["pipette-scores", "score", "--eval", "mmstar", "--dataset", "test", "-f", str(completions)] + monkeypatch.setattr("sys.argv", [*argv, "--no-samples", "--prompt-profile", profile]) + with patch("pipette_scores.dataset_catalog.load_eval_samples", return_value=[_mmstar("C")]): + cli.main() + assert "100.00%" in capsys.readouterr().out diff --git a/packages/pipette-scores/tests/test_prompt_profiles.py b/packages/pipette-scores/tests/test_prompt_profiles.py new file mode 100644 index 0000000..caaac14 --- /dev/null +++ b/packages/pipette-scores/tests/test_prompt_profiles.py @@ -0,0 +1,406 @@ +"""Each profile's text, spelled out, against VLMEvalKit 8eca8e5's wrappers.""" + +import functools +import hashlib +import json + +import pyarrow as pa +import pyarrow.parquet as pq +import pytest + +from pipette_scores import dataset_catalog +from pipette_scores.dataset_catalog import _vlm_common, prompt_profiles +from pipette_scores.dataset_catalog.prompt_profiles import BENCHMARKS, PROFILES +from pipette_scores.types import EvalId + +NAN = float("nan") +CHOICE = {"question": "What is shown?", "hint": "Look closely.", "A": "a cat", "B": "a dog", "C": NAN, "D": None} +CHOICE_DEFAULT = ( + "Hint: Look closely.\nQuestion: What is shown?\nOptions:\nA. a cat\nB. a dog\n" + "Please select the correct answer from the options above. \n" +) +OPEN = {"question": "How many legs?", "hint": NAN, "A": NAN} # an MMMU question without options +OPEN_DEFAULT = "Question: How many legs?\n" +YES_NO = {"question": "Is there a cat? Please answer yes or no.", "category": "existence"} +OCR = {"question": "what is written in the image?", "category": "Regular Text Recognition"} + +LETTER = "Answer with the option's letter from the given choices directly." +SHORT = "Answer the question using a single word or phrase." +LFM_DIRECT = "Please answer directly with only the final answer, do not give any explanation." +MINICPM_CHOICE_COT = ( + "Carefully read the following multichoice question, solve it step by step and finally pick the option " + 'associated with the correct answer in the format of "Answer: selected option\n\n' +) +MINICPM_SHORT_COT = ( + "Read the following question carefully, solve it step by step, and then output the final answer in the " + 'format of "Answer: single number or single word or phrase".\n\n' +) + + +def text(profile: str, benchmark: EvalId, doc: dict, default: str = "DEFAULT") -> str: + return PROFILES[profile](benchmark, doc, default) + + +class TestInternVL: + def test_multiple_choice_puts_hint_question_options_then_the_letter_instruction(self): + assert text("internvl", EvalId.MMSTAR, CHOICE) == ( + f"Look closely.\nWhat is shown?\nA. a cat\nB. a dog\n{LETTER}" + ) + + def test_without_options_it_asks_for_a_direct_answer(self): + assert text("internvl", EvalId.MMMU_VAL, OPEN) == "How many legs?\nAnswer the question directly." + + def test_chinese_text_gets_the_chinese_instruction(self): + doc = {"question": "图中是什么?", "A": "猫", "B": "狗"} + assert text("internvl", EvalId.MMBENCH_EN_DEV, doc) == "图中是什么?\nA. 猫\nB. 狗\n请直接回答选项字母。" + + def test_yes_no_and_ocr(self): + assert text("internvl", EvalId.MME, YES_NO) == f"Is there a cat? Please answer yes or no. {SHORT}" + assert text("internvl", EvalId.POPE, YES_NO) == YES_NO["question"] + assert text("internvl", EvalId.OCRBENCH, OCR) == f"what is written in the image?\n{SHORT}" + + +class TestGraniteVision: + def test_multiple_choice_matches_internvl(self): + for benchmark in (EvalId.MMSTAR, EvalId.REALWORLDQA, EvalId.MMBENCH_EN_DEV, EvalId.MMMU_VAL): + assert text("granite_vision", benchmark, CHOICE) == text("internvl", benchmark, CHOICE) + + def test_ocr_suffix_and_dataset_prompt_elsewhere(self): + assert text("granite_vision", EvalId.OCRBENCH, OCR) == ( + "what is written in the image?\nReply with only one word or a short phrase or a full address." + ) + assert text("granite_vision", EvalId.MME, YES_NO, "D") == "D" + assert text("granite_vision", EvalId.POPE, YES_NO, "D") == "D" + + +class TestMiniCPMV: + def test_direct_multiple_choice(self): + assert text("minicpm_v", EvalId.REALWORLDQA, CHOICE) == ( + f"Hint: Look closely.\nQuestion: What is shown?\nOptions:\nA. a cat\nB. a dog\n\n{LETTER}" + ) + + def test_direct_without_options(self): + assert text("minicpm_v", EvalId.MMMU_VAL, OPEN) == ( + "Carefully read the following question. Answer the question directly.\nQuestion: How many legs?\n" + ) + + def test_direct_yes_no_and_ocr(self): + assert text("minicpm_v", EvalId.MME, YES_NO) == YES_NO["question"] + assert text("minicpm_v", EvalId.POPE, YES_NO) == YES_NO["question"] + assert text("minicpm_v", EvalId.OCRBENCH, OCR) == f"{SHORT}\nwhat is written in the image?" + + def test_step_by_step_multiple_choice(self): + assert text("minicpm_v_cot", EvalId.MMSTAR, CHOICE) == ( + MINICPM_CHOICE_COT + "Hint: Look closely.\nWhat is shown?\nA. a cat\nB. a dog\n" + ) + + def test_step_by_step_without_options_yes_no_and_ocr(self): + assert text("minicpm_v_cot", EvalId.MMMU_VAL, OPEN) == MINICPM_SHORT_COT + "How many legs?\n" + assert text("minicpm_v_cot", EvalId.MME, YES_NO) == MINICPM_SHORT_COT + YES_NO["question"] + assert text("minicpm_v_cot", EvalId.OCRBENCH, OCR) == ( + "Carefully observe the image and answer the OCR-related questions below. \n\nwhat is written in the image?" + ) + + +class TestLFM: + def test_lfm_vl_appends_the_direct_answer_instruction_everywhere(self): + for benchmark in BENCHMARKS: + assert text("lfm_vl", benchmark, CHOICE, CHOICE_DEFAULT) == f"{CHOICE_DEFAULT}\n{LFM_DIRECT}" + + def test_lfm2_5_vl_3b(self): + assert text("lfm2_5_vl_3b", EvalId.MMBENCH_EN_DEV, CHOICE, CHOICE_DEFAULT) == f"{CHOICE_DEFAULT}\n{LFM_DIRECT}" + assert text("lfm2_5_vl_3b", EvalId.MMMU_VAL, OPEN, OPEN_DEFAULT) == OPEN_DEFAULT + assert text("lfm2_5_vl_3b", EvalId.POPE, YES_NO, "D") == "D" + assert text("lfm2_5_vl_3b", EvalId.OCRBENCH, OCR, OCR["question"]) == ( + "what is written in the image?\nPlease answer concisely with short words or phrases when possible." + ) + + def test_lfm2_5_vl_3b_mme_prompt_follows_the_category(self): + assert text("lfm2_5_vl_3b", EvalId.MME, YES_NO) == YES_NO["question"] + artwork = {**YES_NO, "category": "artwork"} + assert text("lfm2_5_vl_3b", EvalId.MME, artwork) == ( + "Is there a cat? Please answer yes or no.\nAnswer with a single word: Yes or No." + ) + celebrity = {**YES_NO, "category": "celebrity"} + assert text("lfm2_5_vl_3b", EvalId.MME, celebrity) == ( + "This is a face recognition question about the person shown. Decide whether the name in the question " + "matches the image. Is there a cat? Please answer yes or no.\nAnswer with a single word: Yes or No." + ) + + def test_mme_categories_are_upstreams_fourteen(self): + assert set(prompt_profiles.LFM_MME_BY_CATEGORY) == { + "ocr", "artwork", "celebrity", "color", "count", "existence", "landmark", "position", "posters", + "scene", "code_reasoning", "commonsense_reasoning", "numerical_calculation", "text_translation", + } # fmt: skip + + +class TestQwen3_5: + QWEN_CHOICE = 'Please show your choice in the `answer` field with only the choice letter, e.g., `"answer": "C"`.' + + def test_multiple_choice_adds_the_model_cards_answer_field_instruction(self): + for benchmark in (EvalId.MMSTAR, EvalId.REALWORLDQA, EvalId.MMBENCH_EN_DEV, EvalId.MMMU_VAL): + assert text("qwen3_5", benchmark, CHOICE, CHOICE_DEFAULT) == ( + "Hint: Look closely.\nQuestion: What is shown?\nOptions:\nA. a cat\nB. a dog\n" + f"Please select the correct answer from the options above.\n{self.QWEN_CHOICE}" + ) + + def test_open_questions_yes_no_and_ocr_keep_the_dataset_prompt(self): + assert text("qwen3_5", EvalId.MMMU_VAL, OPEN, OPEN_DEFAULT) == OPEN_DEFAULT + assert text("qwen3_5", EvalId.MME, YES_NO, "D") == "D" + assert text("qwen3_5", EvalId.POPE, YES_NO, "D") == "D" + assert text("qwen3_5", EvalId.OCRBENCH, OCR, "D") == "D" + + +GLM_YES_NO = "Please answer yes or no." +KIMI_COT = ( + "Answer the preceding question. The last line of your response should follow this format: " + "'Answer: $\\boxed{LETTER}$' (without quotes), where LETTER is one of the options. " + "Think step by step logically, considering all relevant information before answering." +) +SELECT = "Please select the correct answer from the options above. \n" + + +class TestGlmV: + def test_pope_gets_vlmevalkits_yes_no_line_on_its_own_line(self): + assert text("glm_v", EvalId.POPE, YES_NO, "Is there a cat in the image?") == ( + f"Is there a cat in the image?\n{GLM_YES_NO}" + ) + + def test_every_other_benchmark_keeps_the_dataset_prompt(self): + for benchmark in BENCHMARKS - {EvalId.POPE}: + assert text("glm_v", benchmark, CHOICE, CHOICE_DEFAULT) == CHOICE_DEFAULT + + +class TestKimiVL: + def test_mmstar_replaces_the_closing_line_with_the_boxed_instruction(self): + assert text("kimi_vl", EvalId.MMSTAR, CHOICE, CHOICE_DEFAULT) == ( + "Hint: Look closely.\nQuestion: What is shown?\nOptions:\nA. a cat\nB. a dog\n" + KIMI_COT + ) + + def test_mmmu_question_without_options_gets_the_instruction_too(self): + # Upstream's replace is a no-op there and its append is not conditional. + assert text("kimi_vl", EvalId.MMMU_VAL, OPEN, OPEN_DEFAULT) == OPEN_DEFAULT + KIMI_COT + + def test_mmmu_carries_the_instruction_on_every_piece_of_text_around_its_images(self): + # VLMEvalKit's split_MMMU turns this into text, image, text, image, text, and appends the + # instruction to each text item; the loader puts the images back at the placeholders. + default = "Question: and differ.\nOptions:\nA. a\nB. b\n" + SELECT + assert text("kimi_vl", EvalId.MMMU_VAL, CHOICE, default) == ( + f"Question: {KIMI_COT} and {KIMI_COT} differ.\nOptions:\nA. a\nB. b\n{KIMI_COT}" + ) + + def test_mmmu_pieces_around_adjacent_placeholders_are_empty_text_items_with_the_instruction(self): + default = "Which?\n" + SELECT + assert text("kimi_vl", EvalId.MMMU_VAL, CHOICE, default) == ( + f"{KIMI_COT}{KIMI_COT}Which?\n{KIMI_COT}" + ) + + def test_mmmu_with_a_malformed_placeholder_gets_the_instruction_once(self): + default = "See please.\n" + SELECT + assert text("kimi_vl", EvalId.MMMU_VAL, CHOICE, default) == "See please.\n" + KIMI_COT + + def test_the_other_benchmarks_keep_the_dataset_prompts(self): + for benchmark in (EvalId.REALWORLDQA, EvalId.MMBENCH_EN_DEV): + assert text("kimi_vl", benchmark, CHOICE, CHOICE_DEFAULT) == CHOICE_DEFAULT + for benchmark in (EvalId.MME, EvalId.POPE, EvalId.OCRBENCH): + assert text("kimi_vl", benchmark, YES_NO, "D") == "D" + + +class TestNemotronThink: + def test_the_dataset_prompt_starts_with_a_newline(self): + for benchmark in BENCHMARKS: + assert text("nemotron_think", benchmark, CHOICE, CHOICE_DEFAULT) == "\n" + CHOICE_DEFAULT + + def test_the_system_message_is_the_think_flag(self): + assert prompt_profiles.SYSTEM_PROMPTS["nemotron_think"] == "/think" + + +class TestPhi4RvThink: + def test_the_dataset_prompts_stand_and_the_conversation_ends_with_a_think_prefill(self): + for benchmark in BENCHMARKS: + assert text("phi4_rv_think", benchmark, CHOICE, CHOICE_DEFAULT) == CHOICE_DEFAULT + assert prompt_profiles.PREFILLS == {"phi4_rv_think": ""} + assert "phi4_rv_think" not in prompt_profiles.SYSTEM_PROMPTS + + +class TestMimoVl: + def test_the_dataset_prompts_stand_under_a_helpful_assistant_system_message(self): + for benchmark in BENCHMARKS: + assert text("mimo_vl", benchmark, CHOICE, CHOICE_DEFAULT) == CHOICE_DEFAULT + assert prompt_profiles.SYSTEM_PROMPTS["mimo_vl"] == "You are a helpful assistant." + + +# The profiles that existed before glm_v, kimi_vl, nemotron_think, phi4_rv_think and mimo_vl. +_EXISTING = ( + "internvl", + "internvl_r1", + "granite_vision", + "minicpm_v", + "minicpm_v_cot", + "lfm_vl", + "lfm2_5_vl_3b", + "qwen3_5", +) + + +def test_existing_profiles_render_the_same_prompts_as_before_the_new_ones_were_added(): + """Every existing profile on every benchmark, for a multiple-choice row and an open one, + digested; the digest was taken on origin/main (a8afd30) before the new profiles.""" + docs = {"choice": {**CHOICE, "category": "existence"}, "open": {**OPEN, "category": "ocr"}} + rendered = { + f"{name}|{benchmark.value}|{kind}": PROFILES[name](benchmark, doc, "DEFAULT\n") + for name in _EXISTING + for benchmark in sorted(BENCHMARKS, key=lambda b: b.value) + for kind, doc in docs.items() + } + assert len(rendered) == 112 + blob = json.dumps(rendered, sort_keys=True, ensure_ascii=False) + assert ( + hashlib.sha256(blob.encode()).hexdigest() == "bbe18c2f3e8df1edc5f0147d606822e3b8da96a6e7f2ba8d04359ea57d051b47" + ) + + +def test_the_profile_names_are_the_existing_ones_and_the_new_ones(): + assert set(PROFILES) == {*_EXISTING, "glm_v", "kimi_vl", "nemotron_think", "phi4_rv_think", "mimo_vl"} + + +@pytest.mark.parametrize("name", PROFILES) +@pytest.mark.parametrize("benchmark", sorted(BENCHMARKS)) +def test_every_profile_covers_every_benchmark(name, benchmark): + doc = {**CHOICE, "category": "existence"} if benchmark != EvalId.OCRBENCH else OCR + assert isinstance(text(name, benchmark, doc), str) + + +class TestLoading: + def _dataset(self, tmp_path): + row = { + "sample_id": "s1", + "doc_id": "d1", + "prompt": CHOICE_DEFAULT, + "answer": json.dumps("A"), + "raw_doc": json.dumps({k: (None if v != v else v) for k, v in CHOICE.items()}), + "num_images": 0, + } + d = tmp_path / "ds" + d.mkdir() + pq.write_table(pa.Table.from_pylist([row]), d / "test.parquet") + return d + + def test_a_profile_replaces_the_prompt_text(self, tmp_path): + d = self._dataset(tmp_path) + default = _vlm_common.load_prompt_samples(d) + profiled = _vlm_common.load_prompt_samples(d, prompt=functools.partial(PROFILES["internvl"], EvalId.MMSTAR)) + assert default[0].messages[0].content[-1]["text"] == CHOICE_DEFAULT + assert profiled[0].messages[0].content[-1]["text"] == text("internvl", EvalId.MMSTAR, CHOICE) + + def test_internvl_r1_adds_the_r1_system_prompt_before_internvls_prompt(self, tmp_path): + d = self._dataset(tmp_path) + samples = _vlm_common.load_prompt_samples( + d, + prompt=functools.partial(PROFILES["internvl_r1"], EvalId.MMSTAR), + system=prompt_profiles.SYSTEM_PROMPTS["internvl_r1"], + ) + system, user = samples[0].messages + assert system.role == "system" + assert system.content.startswith("You are an AI assistant that rigorously follows this response protocol:") + assert "Enclose this entire thinking process within and tags." in system.content + assert user.content[-1]["text"] == text("internvl", EvalId.MMSTAR, CHOICE) + + def test_only_these_profiles_have_a_system_prompt(self): + assert set(prompt_profiles.SYSTEM_PROMPTS) == {"internvl_r1", "nemotron_think", "mimo_vl"} + + def test_only_phi4_rv_think_has_a_prefill(self): + assert set(prompt_profiles.PREFILLS) == {"phi4_rv_think"} + + def test_existing_profiles_send_the_same_messages_as_before_the_prefill_existed(self, tmp_path): + d = self._dataset(tmp_path) + for name in _EXISTING: + samples = _vlm_common.load_prompt_samples( + d, + prompt=functools.partial(PROFILES[name], EvalId.MMSTAR), + system=prompt_profiles.SYSTEM_PROMPTS.get(name), + prefill=prompt_profiles.PREFILLS.get(name), + ) + roles = [m.role for m in samples[0].messages] + assert roles == (["system", "user"] if name == "internvl_r1" else ["user"]) + + def test_a_system_message_comes_first_and_a_prefill_last(self, tmp_path): + d = self._dataset(tmp_path) + samples = _vlm_common.load_prompt_samples(d, system="S", prefill="") + assert [(m.role, m.content if isinstance(m.content, str) else "blocks") for m in samples[0].messages] == [ + ("system", "S"), + ("user", "blocks"), + ("assistant", ""), + ] + + def _served(self, tmp_path, profile: str, benchmark: EvalId = EvalId.MMSTAR): + """The samples the catalog serves for a profile: what /samples sends.""" + row = { + "sample_id": "s1", + "doc_id": "d1", + "prompt": CHOICE_DEFAULT, + "answer": json.dumps("A"), + "raw_doc": json.dumps({k: (None if v != v else v) for k, v in CHOICE.items()}), + "num_images": 1, + "image_0": b"\x89PNG\r\n\x1a\n0", + } + dataset = tmp_path / "datasets" / benchmark.value / "x" + dataset.mkdir(parents=True) + pq.write_table(pa.Table.from_pylist([row]), dataset / "test.parquet") + return dataset_catalog.load_prompt_samples.__wrapped__( + benchmark, "x", root_dir=tmp_path, prompt_profile=profile + ) + + def test_nemotron_think_serves_a_think_system_message_then_the_image_then_a_newline_led_question(self, tmp_path): + sample = self._served(tmp_path, "nemotron_think", EvalId.MMSTAR)[0] + system, user = sample.messages + assert (system.role, system.content) == ("system", "/think") + assert user.role == "user" + assert [b["type"] for b in user.content] == ["image_url", "text"] + assert user.content[-1]["text"] == "\n" + CHOICE_DEFAULT + + def test_phi4_rv_think_serves_the_user_turn_then_an_assistant_think_prefill_and_no_system_message(self, tmp_path): + sample = self._served(tmp_path, "phi4_rv_think")[0] + assert [m.role for m in sample.messages] == ["user", "assistant"] + assert [b["type"] for b in sample.messages[0].content] == ["image_url", "text"] + assert sample.messages[0].content[-1]["text"] == CHOICE_DEFAULT + assert sample.messages[1].content == "" + assert sample.messages[1].model_dump() == {"role": "assistant", "content": ""} + + def test_mimo_vl_serves_a_helpful_assistant_system_message(self, tmp_path): + sample = self._served(tmp_path, "mimo_vl")[0] + assert [(m.role, m.content if isinstance(m.content, str) else "") for m in sample.messages] == [ + ("system", "You are a helpful assistant."), + ("user", ""), + ] + + def test_kimi_vl_serves_mmmu_images_between_pieces_of_text_that_each_carry_the_instruction(self, tmp_path): + row = { + "sample_id": "s1", + "doc_id": "d1", + "prompt": "Question: vs ?\nOptions:\nA. a\nB. b\n" + SELECT, + "answer": json.dumps("A"), + "raw_doc": json.dumps({"question": "q", "A": "a", "B": "b"}), + "num_images": 2, + "image_0": b"\xff\xd8\xff0", + "image_1": b"\xff\xd8\xff1", + } + dataset = tmp_path / "datasets" / "mmmu_val" / "x" + dataset.mkdir(parents=True) + pq.write_table(pa.Table.from_pylist([row]), dataset / "test.parquet") + sample = dataset_catalog.load_prompt_samples.__wrapped__( + EvalId.MMMU_VAL, "x", root_dir=tmp_path, prompt_profile="kimi_vl" + )[0] + (user,) = sample.messages + assert [b["type"] for b in user.content] == ["text", "image_url", "text", "image_url", "text"] + assert [b["text"] for b in user.content if b["type"] == "text"] == [ + f"Question: {KIMI_COT}", + f" vs {KIMI_COT}", + f"?\nOptions:\nA. a\nB. b\n{KIMI_COT}", + ] + + def test_unknown_profile_and_text_evals_are_refused(self, tmp_path): + with pytest.raises(ValueError, match="unknown prompt profile"): + dataset_catalog.load_prompt_samples(EvalId.MMSTAR, "x", root_dir=tmp_path, prompt_profile="nope") + with pytest.raises(ValueError, match="no prompt profiles"): + dataset_catalog.load_prompt_samples(EvalId.MATH_500, "x", root_dir=tmp_path, prompt_profile="internvl") diff --git a/packages/pipette-scores/tests/test_reasoning.py b/packages/pipette-scores/tests/test_reasoning.py new file mode 100644 index 0000000..5b18504 --- /dev/null +++ b/packages/pipette-scores/tests/test_reasoning.py @@ -0,0 +1,644 @@ +"""split_reasoning: one reading of a completion's reasoning, whatever opened it and however it ended.""" + +import random +import time + +import pytest + +from pipette_scores.dataset_catalog import prompt_profiles +from pipette_scores.scoring import mmstar, ocrbench, pope, realworldqa, score +from pipette_scores.scoring.completion_text import ( + CLOSED, + NO_REASONING, + ORPHAN_CLOSE, + STATUSES, + UNCLOSED_CUT, + UNCLOSED_STOP, + ReasoningTally, + grading_text, + normalize_reasoning_markup, + normalize_vlm_reasoning_markup, + remove_thinking_tags, + split_reasoning, +) +from pipette_scores.types import EvalId, SampleCompletion, VlmSample + + +def comp(text: str, stop: str | None = "eos") -> SampleCompletion: + return SampleCompletion(id="1", completion=text, stop_reason=stop) + + +# --- 1. Nothing that scored before scores differently ------------------------------------------ +# Completions of the families scored before, with what the thinking-tag rules of main (a8afd30) graded +# each as (the completion's grading text, then remove_thinking_tags) and whether the strict view, +# with enable_thinking true and the completion cut, blanked it. +EXISTING = [ + pytest.param("\nThe circle is red.\n\n\nC", "C", False, id="qwen-closed"), + pytest.param( + "\nA? no. B? no.\n\n\nThe answer is $\\boxed{C}$.", + "The answer is $\\boxed{C}$.", + False, + id="qwen-closed-boxed", + ), + pytest.param("The circle is red, so option C.\n\n\nAnswer: C", "Answer: C", False, id="qwen-prompt-opened"), + pytest.param("\nThe circle is red, so the answer is C but", "", True, id="qwen-unclosed"), + pytest.param("\n\n\n\nC", "C", False, id="qwen-no-thinking"), + pytest.param('\nRed.\n\n\n"answer": "C"', '"answer": "C"', False, id="qwen-answer-field"), + pytest.param("<|channel>thought\nThe circle is red.C", "C", False, id="gemma-closed"), + pytest.param("<|channel>thought\nAnswer: C", "Answer: C", False, id="gemma-empty"), + pytest.param("<|channel>thought\nThe circle is red, so C and", "", True, id="gemma-unterminated"), + pytest.param("The circle is red.C", "The circle is red.C", True, id="gemma-stray-close"), + pytest.param( + "The circle is red.<|END_THINKING|><|START_TEXT|>Answer: C<|END_TEXT|>", "Answer: C", False, id="cohere-on" + ), + pytest.param("<|START_TEXT|>C<|END_TEXT|>", "C", True, id="cohere-off"), + pytest.param( + "<|START_THINKING|>Red.<|END_THINKING|><|START_TEXT|>C<|END_TEXT|>", "C", False, id="cohere-thinking-block" + ), + pytest.param("Red.\n\n\nC\n", "C", False, id="ernie"), + pytest.param("We need the colour. It is red.assistantfinalC", "C", False, id="gptoss-bare"), + pytest.param( + "<|channel|>analysis<|message|>Red.<|end|><|start|>assistant<|channel|>final<|message|>C<|return|>", + "C", + False, + id="gptoss-tokens", + ), + pytest.param( + "<|channel|>analysis<|message|>We need to find the colour. Let u =", + "<|channel|>analysis<|message|>We need to find the colour. Let u =", + True, + id="gptoss-cut", + ), + pytest.param("\nThe circle is red.\n\nAnswer: C", "Answer: C", False, id="minicpm-think"), + pytest.param("The circle is red.\n\nAnswer: C", "The circle is red.\n\nAnswer: C", True, id="minicpm-plain"), + pytest.param("\nThe image shows a red circle on the left.\n\n\nC", "C", False, id="internvl-r1"), + pytest.param("\nThe image shows a red circle and", "", True, id="internvl-r1-cut"), + pytest.param("C", "C", True, id="lfm-plain"), + pytest.param("C. red", "C. red", True, id="lfm-letter-text"), + pytest.param( + "The circle is red, which is option C.", "The circle is red, which is option C.", True, id="lfm-sentence" + ), + pytest.param("[THINK]The circle is red.[/THINK]C", "C", False, id="ministral-closed"), + pytest.param("The circle is red.[/THINK]C", "C", False, id="ministral-orphan"), + pytest.param("[THINK]The circle is red, so", "", True, id="ministral-cut"), + pytest.param("\nGot it, the circle is red.\n\n\n\\boxed{C}", "\\boxed{C}", False, id="mimo"), + pytest.param("aCb", "C", False, id="two-blocks"), + pytest.param("abC", "C", False, id="two-closes"), + pytest.param("[THINK]a[/THINK]C[THINK]more", "C", False, id="answer-then-open"), + pytest.param("", "", False, id="empty"), + pytest.param(" \n ", "", False, id="whitespace"), + pytest.param("aC", "C", False, id="think-uppercase"), + # remove_thinking_tags' orphan-close rule cuts through the first , else the first [/THINK]. + pytest.param("x[/THINK]yz", "z", False, id="mixed-orphan-closes"), + # InternVL3.5 under the R1 prompt that wrote no : graded whole, its included. + pytest.param( + "The circle is red, so C it is.", + "The circle is red, so C it is.", + True, + id="internvl-r1-untagged-answer", + ), +] + + +@pytest.mark.parametrize("completion, answer, blanked", EXISTING) +def test_existing_families_grade_as_they_did(completion, answer, blanked): + assert remove_thinking_tags(normalize_reasoning_markup(completion)) == answer + assert split_reasoning(comp(completion)).answer == answer + cut = split_reasoning(comp(completion, "truncated"), True) + assert cut.answer == answer + assert cut.blanked is blanked + assert grading_text(comp(completion, "truncated"), True) == (normalize_reasoning_markup(completion), blanked) + + +_MCQ = {"A": "blue", "B": "green", "C": "red", "D": "black", "answer": "C", "l2_category": "x"} + + +def test_existing_families_score_as_they_did_on_mmstar(): + """The letter each family's reply reaches, through the scorer: every reply above that names C.""" + named_c = [ + c for c, a, _ in (p.values for p in EXISTING) if a.strip() and "C" in a and "\\boxed" not in a and len(a) < 40 + ] + assert len(named_c) >= 12 + samples = [VlmSample(id=str(i), prompt="q", answer=None, raw_doc=_MCQ) for i in range(len(named_c))] + completions = [SampleCompletion(id=str(i), completion=c, stop_reason="eos") for i, c in enumerate(named_c)] + _, ctx = mmstar.score(completions, samples) + assert ctx["accuracy"] == 1.0 + assert ctx["blanked_truncated"] == 0 + + +# --- 2. The matrix: where the opener came from x how the reasoning ended x tag family ----------- +# (open tag, close tag); a hidden opener has only its close in the text. +FAMILIES = { + "think": ("", ""), + "THINK": ("[THINK]", "[/THINK]"), + "kimi": ("\u25c1think\u25b7", "\u25c1/think\u25b7"), + "north": ("<|START_THINKING|>", "<|END_THINKING|>"), + "thinking": ("", ""), + "gemma": ("<|channel>thought\n", ""), +} +# Gemma's with no opener before it is a stray marker and is dropped, so a hidden Gemma opener is +# not one a reply can close. +HIDDEN_OK = [f for f in FAMILIES if f != "gemma"] +R, A = "The circle is red, so the option is C", "C" + +# source -> (opens_reasoning, enable_thinking). visible: the model writes the opener, nothing declared. +# prefill: phi4_rv_think's assistant "". system: a profile's system message has the model reason from +# its first token (nemotron_think's /think). template: the chat template opens it in the generation prompt +# (Step3), declared by enable_thinking. hidden: a control token no text shows (Ministral's [THINK]), nothing declared. +SOURCES = { + "visible": (False, None), + "system": (True, None), + "prefill": (True, None), + "template": (False, True), + "hidden": (False, None), +} + + +def _text(family: str, shape: str) -> str: + o, c = FAMILIES[family] + return { + "opened_closed": f"{o}{R}{c}{A}", + "opened_unclosed": f"{o}{R}", + "hidden_closed": f"{R}{c}{A}", + "hidden_unclosed": R, + "plain": A, + }[shape] + + +# (source, shape, stop) -> (answer, status, blanked). The answer of an unclosed reply is what the +# thinking-tag rules grade: the whole text when no tag shows the reasoning, nothing when its opener does. +MATRIX = { + # The opener is in the text and nothing says so beforehand. + ("visible", "opened_closed", "eos"): (A, CLOSED, False), + ("visible", "opened_closed", "truncated"): (A, CLOSED, False), + ("visible", "opened_unclosed", "truncated"): ("", UNCLOSED_CUT, True), + ("visible", "opened_unclosed", "doom_loop"): ("", UNCLOSED_CUT, True), + ("visible", "opened_unclosed", "eos"): ("", UNCLOSED_STOP, False), + ("visible", "hidden_closed", "eos"): (A, ORPHAN_CLOSE, False), + ("visible", "hidden_closed", "truncated"): (A, ORPHAN_CLOSE, False), + ("visible", "plain", "eos"): (A, NO_REASONING, False), + ("visible", "plain", "truncated"): (A, NO_REASONING, False), + # A hidden opener nothing declared: only a close shows it, and a reply cut before the close is not known as reasoning. + ("hidden", "hidden_closed", "eos"): (A, ORPHAN_CLOSE, False), + ("hidden", "hidden_closed", "truncated"): (A, ORPHAN_CLOSE, False), + ("hidden", "hidden_unclosed", "truncated"): (R, NO_REASONING, False), + ("hidden", "plain", "eos"): (A, NO_REASONING, False), +} +for _source in ("system", "prefill", "template"): + MATRIX.update( + { + # The opener is known; the model may write it again. + (_source, "opened_closed", "eos"): (A, CLOSED, False), + (_source, "opened_unclosed", "truncated"): ("", UNCLOSED_CUT, True), + (_source, "opened_unclosed", "doom_loop"): ("", UNCLOSED_CUT, True), + (_source, "opened_unclosed", "eos"): ("", UNCLOSED_STOP, False), + (_source, "hidden_closed", "eos"): (A, CLOSED, False), + (_source, "hidden_closed", "truncated"): (A, CLOSED, False), + # Cut inside reasoning the prompt opened: graded whole, counted by the strict view. + (_source, "hidden_unclosed", "truncated"): (R, UNCLOSED_CUT, True), + (_source, "hidden_unclosed", "doom_loop"): (R, UNCLOSED_CUT, True), + # The model stopped without ever closing: graded as written, flagged. + (_source, "hidden_unclosed", "eos"): (R, UNCLOSED_STOP, False), + (_source, "plain", "eos"): (A, UNCLOSED_STOP, False), + (_source, "plain", "truncated"): (A, UNCLOSED_CUT, True), + } + ) + + +def _cells(): + for (source, shape, stop), expected in MATRIX.items(): + families = ( + HIDDEN_OK + if shape.startswith("hidden") and source != "visible" or shape == "hidden_closed" + else list(FAMILIES) + ) + if shape == "plain": + families = ["think"] + if shape == "hidden_unclosed": + families = ["think"] + for family in families: + yield pytest.param(source, shape, stop, family, expected, id=f"{source}-{shape}-{stop}-{family}") + + +@pytest.mark.parametrize("source, shape, stop, family, expected", list(_cells())) +def test_the_matrix(source, shape, stop, family, expected): + answer, status, blanked = expected + opens, thinking = SOURCES[source] + text = _text(family, shape) + got = split_reasoning(comp(text, stop), thinking, opens_reasoning=opens) + assert (got.answer, got.status, got.blanked) == (answer, status, blanked) + # A second pass over the answer changes nothing. + again = split_reasoning(comp(got.answer, stop), thinking, opens_reasoning=opens) + assert again.answer == got.answer + + +def test_the_matrix_covers_every_status_from_every_source(): + for source in SOURCES: + statuses = {s for (src, _, _), (_, s, _) in MATRIX.items() if src == source} + assert statuses <= set(STATUSES) + assert {s for (_, _, _), (_, s, _) in MATRIX.items()} == set(STATUSES) + + +@pytest.mark.parametrize( + "text, known, status, answer", + [ + # gpt-oss: the control tokens are gone, the transition shows the analysis channel closed. + ("We need the colour. It is red.assistantfinalC", False, ORPHAN_CLOSE, "C"), + ("We need the colour. It is red.assistantfinalC", True, CLOSED, "C"), + ( + "<|channel|>analysis<|message|>Red.<|end|><|start|>assistant<|channel|>final<|message|>C<|return|>", + False, + ORPHAN_CLOSE, + "C", + ), + ("aassistantfinalC", False, CLOSED, "C"), + ], +) +def test_the_harmony_transition_is_a_close(text, known, status, answer): + got = split_reasoning(comp(text, "truncated"), True if known else None) + assert (got.answer, got.status, got.blanked) == (answer, status, False) + + +def test_harmony_cut_inside_analysis_is_graded_whole_and_blanked_when_declared(): + text = "<|channel|>analysis<|message|>We need to compute. Let u =" + assert split_reasoning(comp(text, "truncated"), True).blanked is True + assert split_reasoning(comp(text, "truncated")).blanked is False + + +# --- 3. Policies for the odd shapes ---------------------------------------------------------- +class TestPolicies: + def test_answer_before_close_then_a_reopened_block_that_gives_no_answer_reads_the_text_before_the_close(self): + # Ministral-3B-Reasoning on POPE: it answers, closes a block it never opened, and starts thinking again. + got = split_reasoning(comp("yes[/THINK][THINK]I need to determine whether", "truncated")) + assert (got.answer, got.status, got.answer_before_close) == ("yes", ORPHAN_CLOSE, True) + + def test_answer_before_close_is_not_read_when_reasoning_precedes_the_close(self): + # Ministral also reasons before [/THINK]; cut after it opens a second block, it gave no answer. + got = split_reasoning(comp("The circle looks red, so B.[/THINK][THINK]Wait, check the options", "truncated")) + assert (got.answer, got.status, got.answer_before_close) == ("", ORPHAN_CLOSE, False) + got = split_reasoning(comp("The circle is red.\nSo it is C.wait", "doom_loop")) + assert (got.answer, got.answer_before_close) == ("", False) + + def test_the_real_ministral_reply_with_a_thinking_block_and_an_answer_wrapper(self): + reply = "yes[/THINK][THINK]I need to determine whether the image has a red circle.\n\nyes\n" + got = split_reasoning(comp(reply)) + assert got.answer == "yes" + assert got.status == ORPHAN_CLOSE + # The thinking-tag rules of main grade this empty. + assert remove_thinking_tags(reply) == "" + + def test_answer_before_close_is_not_read_when_the_opener_was_known(self): + got = split_reasoning(comp("reasoningmore", "truncated"), opens_reasoning=True) + assert got.answer == "" + assert got.answer_before_close is False + + def test_answer_before_close_is_not_read_when_a_visible_answer_follows(self): + assert split_reasoning(comp("yes[/THINK]no")).answer == "no" + + def test_a_later_stray_close_is_text_and_what_precedes_it_is_reasoning(self): + assert split_reasoning(comp("abc")).answer == "bc" + assert split_reasoning(comp("abc")).answer == "c" + + def test_a_second_block_after_the_answer_is_dropped(self): + assert split_reasoning(comp("aansmore")).answer == "ans" + assert split_reasoning(comp("aansmore")).answer == "ans" + + def test_an_opener_inside_a_block_is_part_of_it(self): + assert split_reasoning(comp("abc")).answer == "c" + + def test_the_two_tag_families_close_each_other(self): + assert split_reasoning(comp("[THINK]aC")).answer == "C" + assert split_reasoning(comp("a[/THINK]C")).answer == "C" + + def test_tags_are_case_insensitive(self): + assert split_reasoning(comp("aC")).answer == "C" + assert split_reasoning(comp("a[/think]C")).answer == "C" + + def test_unclosed_stop_grades_the_last_complete_answer_wrapper_when_the_opener_is_known_or_visible(self): + text = "Red, so C. B no wait C" + got = split_reasoning(comp(text), opens_reasoning=True) + assert (got.answer, got.status) == ("C", UNCLOSED_STOP) + got = split_reasoning(comp("Red. C")) + assert (got.answer, got.status) == ("C", UNCLOSED_STOP) + + def test_unclosed_stop_without_a_wrapper_is_graded_as_the_thinking_tag_rules_grade_it(self): + assert split_reasoning(comp("Red, so C."), opens_reasoning=True).answer == "Red, so C." + assert split_reasoning(comp("Red, so C.")).answer == "" + + def test_an_answer_wrapper_is_not_read_without_a_known_or_visible_opener_or_when_the_run_was_cut(self): + text = "Red, so C it is" + assert split_reasoning(comp(text)).answer == text + assert split_reasoning(comp(text), True).answer == text # enable_thinking alone + assert split_reasoning(comp(text, "truncated"), opens_reasoning=True).answer == text + + def test_the_answer_wrapper_is_peeled_after_reasoning_only(self): + assert split_reasoning(comp("r\n\nC\n")).answer == "C" + assert split_reasoning(comp("C")).answer == "C" + + def test_empty_and_whitespace_completions(self): + for text in ("", " \n\t "): + got = split_reasoning(comp(text, "truncated"), True, opens_reasoning=True) + assert (got.answer, got.blanked, got.status) == ("", False, UNCLOSED_CUT) + assert split_reasoning(comp(text)).status == NO_REASONING + + def test_unicode_and_whitespace_around_the_tags(self): + assert split_reasoning(comp("\u00a0r\n\u00a0\nC\u00a0")).answer == "C" + assert split_reasoning(comp("r\r\n\r\nC\r\n")).answer == "C" + assert ( + split_reasoning(comp("\u25c1think\u25b7\u8d64\u2615\u25c1/think\u25b7\u7b54\u3048: C")).answer + == "\u7b54\u3048: C" + ) + assert split_reasoning(comp("\U0001f600\U0001f600C")).answer == "\U0001f600C" + + def test_the_strict_view_needs_text_and_a_cut_and_no_close(self): + assert split_reasoning(comp("", "truncated"), True).blanked is False + assert split_reasoning(comp("x", "eos")).blanked is False + assert split_reasoning(comp("x", "failure")).blanked is False + assert split_reasoning(comp("x", None)).blanked is False + assert split_reasoning(comp("x", "truncated")).blanked is True + assert split_reasoning(comp("x", "truncated")).blanked is False + assert split_reasoning(comp("x", "truncated"), False).blanked is False + + def test_the_strict_view_does_not_need_the_run_to_record_enable_thinking(self): + for thinking in (None, False, True): + assert split_reasoning(comp("x", "truncated"), thinking).blanked is True + + +# --- 4. Cost --------------------------------------------------------------------------------- +_RUNAWAYS = { + "opens": "" * 40_000, + "closes": "" * 40_000, + "pairs": "x" * 20_000, + "mixed": "[THINK]ab[/THINK]" * 12_000, + "unclosed": "" + "word " * 80_000, + "gemma": "<|channel>thought\n" * 15_000, + "answers": "" * 30_000 + "" * 30_000, + "wrappers": "" * 30_000 + "x", + "text-markers": "<|START_TEXT|>" * 30_000, + "harmony": "assistant<|channel|>" * 15_000, + "spaces": " " * 300_000 + "<|end|>", + "kimi": "\u25c1think\u25b7" * 20_000, +} + + +@pytest.mark.parametrize("name", _RUNAWAYS) +def test_a_runaway_completion_is_read_in_linear_time(name): + text = _RUNAWAYS[name] + start = time.perf_counter() + for opens in (False, True): + got = split_reasoning(comp(text, "doom_loop"), opens_reasoning=opens) + assert got.status in STATUSES + assert time.perf_counter() - start < 2.0, f"{name} took {time.perf_counter() - start:.1f}s" + + +# --- VLM markup: what the VLM evals rewrite beyond the text evals' normalize_reasoning_markup ------- +_VLM_MARKUP = [ + # Kimi-VL-A3B-Thinking-2506 on llama.cpp (b11744 smoke): the tags are plain text in the vocabulary, so + # the server prints them. The first two are the model's replies to the smoke's MCQ and OCR questions. + pytest.param( + "◁think▷Got it, let's look at the image. The circle is red. So the answer is C.◁/think▷C", + "C", + id="kimi-closed", + ), + pytest.param( + "◁think▷So, let's look at the image. The text in the image is \"PIPELINE 4\". Let's check each part." + "◁/think▷PIPELINE 4", + "PIPELINE 4", + id="kimi-closed-ocr", + ), + # The opener came before the part the server kept (a smoke reply that starts mid-thought): the + # orphan close still cuts through. + pytest.param( + "5\"? Wait, no, no, wait. Wait, no, the number is 2075. Yes, that's the number.◁/think▷2075", + "2075", + id="kimi-opener-missing", + ), + # Cut inside the thought: nothing, as Kimi's own extract_summary gives. + pytest.param("◁think▷Got it, the circle is red, so the answer is C, but wait", "", id="kimi-unclosed"), + pytest.param("C◁think▷and then more thought", "C", id="kimi-unclosed-after-text"), + pytest.param("◁think▷Red.◁/think▷Answer: $\\boxed{C}$", "Answer: $\\boxed{C}$", id="kimi-boxed-summary"), + pytest.param("The circle is red.C", "C", id="thinking-tags"), +] + + +@pytest.mark.parametrize("raw, expected", _VLM_MARKUP) +def test_the_vlm_markup_rewrite(raw, expected): + once = normalize_vlm_reasoning_markup(raw) + assert remove_thinking_tags(once) == expected + assert split_reasoning(comp(raw)).answer == expected + assert normalize_vlm_reasoning_markup(once) == once + + +@pytest.mark.parametrize( + "raw, blanked", + [ + # Kimi-VL cut inside its thought, like Gemma 4's; once its close is rewritten it is a + # reasoning close, so an answer cut after it is graded as is. + ("◁think▷Compute 17*23: 17*20 = 340", True), + ("◁think▷Red.◁/think▷The answer is", False), + ], +) +def test_a_cut_kimi_reply_is_blanked_only_inside_its_thought(raw, blanked): + assert split_reasoning(comp(raw, "truncated"), True).blanked is blanked + + +# --- 5. Never raises, never returns reasoning that a close ended ---------------------------------- +_TOKENS = [ + "", + "", + "[THINK]", + "[/THINK]", + "\u25c1think\u25b7", + "\u25c1/think\u25b7", + "", + "", + "<|START_THINKING|>", + "<|END_THINKING|>", + "<|channel>thought\n", + "", + "assistantfinal", + "", + "", + "", + "", + "<|START_TEXT|>", + "<|END_TEXT|>", + "<|end|>", + "", + " ", + "\n", + "\u00a0", +] + + +def test_any_completion_is_read_without_raising_and_to_a_status(): + rng = random.Random(7) + for _ in range(6000): + text = "".join(rng.choice(_TOKENS + ["word", "yes", "C"]) for _ in range(rng.randint(0, 12))) + stop = rng.choice(["eos", "truncated", "doom_loop", None, "failure"]) + thinking = rng.choice([None, True, False]) + opens = rng.random() < 0.3 + got = split_reasoning(comp(text, stop), thinking, opens_reasoning=opens) + assert got.status in STATUSES + assert isinstance(got.answer, str) and got.answer == got.answer.strip() + assert got.blanked <= (got.status == UNCLOSED_CUT) + + +def test_reasoning_that_a_close_ended_is_never_returned(): + """A reply made of reasoning, a close, then an answer never returns the reasoning, whoever opened it.""" + rng = random.Random(11) + for _ in range(4000): + family = rng.choice(list(FAMILIES)) + o, c = FAMILIES[family] + reasoning = " ".join(f"REASON{rng.randint(0, 99)}" for _ in range(rng.randint(1, 6))) + answer = " ".join(f"ANS{rng.randint(0, 99)}" for _ in range(rng.randint(1, 3))) + # Gemma's close with no opener is a stray marker, so its opener is always in the text. + visible_opener = family == "gemma" or rng.random() < 0.5 + text = (o if visible_opener else "") + reasoning + c + answer + for thinking, opens in ((None, False), (True, False), (None, True)): + got = split_reasoning(comp(text, rng.choice(["eos", "truncated"])), thinking, opens_reasoning=opens) + assert "REASON" not in got.answer, (text, got) + assert got.answer.startswith("ANS"), (text, got) + assert got.blanked is False + + +# --- 6. Real replies (smoke runs of llama.cpp b11744) ----------------------------------------- +REAL = { + "kimi-mcq": ( + "\u25c1think\u25b7Got it, let's look at the image. The circle is red. So the answer is C.\u25c1/think\u25b7C", + "C", + ), + "kimi-opener-missing": ('5"? Wait, no, no, wait. Wait, no, the number is 2075.\u25c1/think\u25b72075', "2075"), + "mimo-mcq": ( + "\nGot it, let's look at the image. There's a red circle on the left. The options are A to D. So the circle is red, " + "which is option C.\n\n\n\\boxed{C}", + "\\boxed{C}", + ), + "mimo-ocr": ('\nThe text is "PIPETTE 42".\n\n\n\\boxed{PIPETTE 42}', "\\boxed{PIPETTE 42}"), + "ministral-pope": ('So the answer should be "yes".[/THINK]yes', "yes"), + "ministral-describe": ( + "It shows the front page.[/THINK]The image is an old newspaper.", + "The image is an old newspaper.", + ), +} + + +@pytest.mark.parametrize("name", REAL) +def test_real_replies(name): + reply, answer = REAL[name] + assert split_reasoning(comp(reply)).answer == answer + + +@pytest.mark.parametrize( + "name, reply, thinking", + [ + # Reasoning the server already split off: the content is the answer alone, whatever model wrote it. + ("qwen3.5-9b", "C", True), + ("qwen3.6-35b", "C", True), + ("gemma-4-26b", "C", True), + ("nemotron-3-omni", "C", True), + ("nemotron-12b", "C\n", True), + ( + "minicpm-o-4.5", + "To determine the color of the circle, observe the image: the circle on the left is red. Among the options, option C is Red. \n\n\nC", + True, + ), + ("step3", "C", None), + ], +) +def test_replies_the_server_already_split_are_graded_whole_and_flagged_unclosed_stop_when_thinking_was_declared( + name, reply, thinking +): + got = split_reasoning(comp(reply), thinking) + assert got.answer == reply.strip() + assert got.status == (UNCLOSED_STOP if thinking else NO_REASONING) + assert got.blanked is False + + +def test_the_scorers_grade_the_real_replies(): + samples = [VlmSample(id="1", prompt="q", answer=None, raw_doc=_MCQ)] + for name in ("kimi-mcq", "mimo-mcq"): + _, ctx = mmstar.score([comp(REAL[name][0])], samples) + assert ctx["accuracy"] == 1.0, name + assert ctx["reasoning_status"][CLOSED] == 1 + pope_sample = VlmSample(id="1", prompt="q", answer=None, raw_doc={"answer": "Yes", "category": "adversarial"}) + scores, ctx = pope.score([comp(REAL["ministral-pope"][0])], [pope_sample]) + assert scores[0].is_correct is True + assert ctx["reasoning_status"][ORPHAN_CLOSE] == 1 + ocr = VlmSample( + id="1", prompt="q", answer=None, raw_doc={"answer": ["PIPETTE 42"], "category": "Regular Text Recognition"} + ) + scores, _ = ocrbench.score([comp(REAL["mimo-ocr"][0])], [ocr]) + assert scores[0].is_correct is True + + +# --- 7. What the scorers report ---------------------------------------------------------------- +def test_the_tally_counts_statuses_and_answers_read_before_a_close(): + tally = ReasoningTally() + for text, stop in [ + ("aC", "eos"), + ("aC", "eos"), + ("a", "truncated"), + ("C", "eos"), + ("yes[/THINK][THINK]x", "truncated"), + ]: + tally.add(split_reasoning(comp(text, stop))) + assert tally.context() == { + "reasoning_status": {CLOSED: 1, ORPHAN_CLOSE: 2, UNCLOSED_CUT: 1, UNCLOSED_STOP: 0, NO_REASONING: 1}, + "reasoning_answer_before_close": 1, + } + + +def test_a_scorer_reports_unclosed_stops_and_the_strict_count_without_enable_thinking(): + samples = [VlmSample(id=str(i), prompt="q", answer=None, raw_doc=_MCQ) for i in range(3)] + completions = [ + SampleCompletion(id="0", completion="The circle is red, so C", stop_reason="truncated"), + SampleCompletion(id="1", completion="The circle is red, so C", stop_reason="eos"), + SampleCompletion(id="2", completion="rC", stop_reason="eos"), + ] + _, ctx = mmstar.score(completions, samples) + assert ctx["blanked_truncated"] == 1 + assert ctx["reasoning_status"] == {CLOSED: 0, ORPHAN_CLOSE: 1, UNCLOSED_CUT: 1, UNCLOSED_STOP: 1, NO_REASONING: 0} + + +def test_every_registered_opening_profile_is_a_profile_and_the_facade_passes_it_on(): + assert prompt_profiles.REASONING_OPENERS <= set(prompt_profiles.PROFILES) + samples = [VlmSample(id="1", prompt="q", answer=None, raw_doc=_MCQ)] + cut = [comp("The circle is red, so the answer is C. Let me double", "truncated")] + for profile, blanked in ( + ("phi4_rv_think", 1), + ("nemotron_think", 1), + ("kimi_vl", 1), + ("mimo_vl", 1), + # Its R1 system prompt has the model write itself: counted only with enable_thinking, as on main. + ("internvl_r1", 0), + ("minicpm_v", 0), + (None, 0), + ): + _, ctx = score(EvalId.MMSTAR, cut, samples, prompt_profile=profile) + assert ctx["blanked_truncated"] == blanked, profile + + +def test_the_text_evals_do_not_read_the_parser(): + """Only the VLM scorers read a profile's opener: a text eval scores and counts as on main.""" + from pipette_scores.types import GPQADiamondSample + + sample = GPQADiamondSample(id="1", prompt="Q\nA) a\nB) b\nC) c\nD) d", answer="B") + cut = [comp("Option B fits. Answer: B. Double-check option", "truncated")] + _, with_profile = score(EvalId.GPQA_DIAMOND, cut, [sample], prompt_profile="phi4_rv_think") + _, without = score(EvalId.GPQA_DIAMOND, cut, [sample]) + assert with_profile == without + assert with_profile["gpqa_diamond_blanked_truncated"] == 0 + assert "gpqa_diamond_reasoning_status" not in with_profile + # Nor the VLM markup rewrite. + for raw in ("◁think▷Red.◁/think▷C", "Red.C"): + assert normalize_reasoning_markup(raw) == raw + + +def test_realworldqa_and_pope_report_the_tally_too(): + sample = VlmSample(id="1", prompt="q", answer=None, raw_doc={"A": "x", "B": "y", "answer": "B"}) + _, ctx = realworldqa.score([comp("rB")], [sample]) + assert ctx["reasoning_status"][ORPHAN_CLOSE] == 1 + pope_sample = VlmSample(id="1", prompt="q", answer=None, raw_doc={"answer": "Yes", "category": "adversarial"}) + _, ctx = pope.score([comp("ryes")], [pope_sample]) + assert ctx["reasoning_status"][ORPHAN_CLOSE] == 1 diff --git a/packages/pipette-scores/tests/test_scoring.py b/packages/pipette-scores/tests/test_scoring.py index 6dbdde7..7bb2711 100644 --- a/packages/pipette-scores/tests/test_scoring.py +++ b/packages/pipette-scores/tests/test_scoring.py @@ -3,8 +3,10 @@ import pipette_scores.scoring as scoring from pipette_scores.types import ( EvalId, + GPQADiamondSample, IFBenchSample, IFStructSample, + Math500Sample, SampleCompletion, ) @@ -66,6 +68,116 @@ def test_dispatch_ifstruct(): assert scores[0].is_correct is True +def test_dispatch_passes_enable_thinking(): + samples = [ + IFStructSample( + id="1", + seed=1, + prompt="Generate JSON", + json_schema={"type": "object", "properties": {"name": {"type": "string"}}, "required": ["name"]}, + top_level_count=None, + top_level_key=None, + require_wrapper_key=False, + require_code_block=False, + require_no_commentary=False, + output_format="json", + entity_type="json", + ) + ] + completion = 'Draft:\n```json\n{"name": "Alice"}\n```\nNow check whether the' + completions = [SampleCompletion(id="1", completion=completion, stop_reason="truncated")] + scores, context = scoring.score("ifstruct", completions, samples, enable_thinking=True) + assert scores[0].is_correct is True + assert context["ifstruct_blanked_truncated"] == 1 + assert context["ifstruct_pass_at_1_blanked"] == 0.0 + + +_KW = {"keyword1": "banana", "keyword2": "apple", "keyword3": "cherry", "keyword4": "date", "keyword5": "fig"} + +# One sample per eval, the answer that passes it, and reasoning that would fail +# it if it were graded as the answer. +_EVAL_CASES = { + EvalId.GPQA_DIAMOND: ( + GPQADiamondSample(id="1", prompt="Q\nA) a\nB) b\nC) c\nD) d", answer="B"), + "The answer is B", + "Option C looks right. Answer: C", + ), + EvalId.MATH_500: (Math500Sample(id="1", prompt="P", answer="42"), "The answer is 42", "Maybe \\boxed{7}."), + EvalId.IFBENCH: ( + IFBenchSample( + id="1", + key="1", + instruction_id_list=("count:keywords_multiple",), + prompt="Use banana once, apple twice, cherry three times, date five times, fig seven times.", + kwargs=(_KW,), + n_constraints=1, + ), + " ".join(["banana"] * 1 + ["apple"] * 2 + ["cherry"] * 3 + ["date"] * 5 + ["fig"] * 7), + "banana apple cherry date fig, each once to start", + ), + EvalId.IFSTRUCT: ( + IFStructSample( + id="1", + seed=1, + prompt="Generate JSON", + json_schema={"type": "object", "properties": {"name": {"type": "string"}}, "required": ["name"]}, + top_level_count=None, + top_level_key=None, + require_wrapper_key=False, + require_code_block=False, + require_no_commentary=True, + output_format="json", + entity_type="json", + ), + '{"name": "Alice"}', + 'A name only, so {"name": 1} is wrong.', + ), +} + +# Every way the scorer reads a closed reasoning block. +_CLOSED_FORMATS = { + "think": "{r}{a}", + "prompt-opened": "{r}\n\n\n{a}", + "mistral": "[THINK]{r}[/THINK]{a}", + "gemma4": "<|channel>thought\n{r}{a}", + "north": "{r}<|END_THINKING|><|START_TEXT|>{a}<|END_TEXT|>", + "ernie": "{r}\n\n\n\n{a}\n\n", + "gpt-oss": "<|channel|>analysis<|message|>{r}<|end|><|start|>assistant<|channel|>final<|message|>{a}<|return|>", + "gpt-oss-vllm": "analysis{r}assistantfinal{a}", +} + + +@pytest.mark.parametrize("enable_thinking", [True, False, None]) +@pytest.mark.parametrize("stop_reason", ["eos", "truncated", "doom_loop", "unknown", None]) +@pytest.mark.parametrize("fmt", list(_CLOSED_FORMATS.values()), ids=list(_CLOSED_FORMATS)) +@pytest.mark.parametrize("eval_id", list(_EVAL_CASES)) +def test_closed_reasoning_grades_the_same_whatever_the_stop(eval_id, fmt, stop_reason, enable_thinking): + # Neither this scorer nor 552094f blanks a sample whose reasoning closed, so + # its verdict is the one it gets with no stop reason and no thinking flag, + # and the strict view agrees. + sample, answer, reasoning = _EVAL_CASES[eval_id] + # The reasoning fails the sample if it is left in. + unstripped, _ = scoring.score(eval_id, [SampleCompletion(id="1", completion=f"{reasoning} {answer}")], [sample]) + assert unstripped[0].is_correct is False + completion = fmt.format(r=reasoning, a=answer) + baseline, _ = scoring.score(eval_id, [SampleCompletion(id="1", completion=completion)], [sample]) + assert baseline[0].is_correct is True + c = SampleCompletion(id="1", completion=completion, stop_reason=stop_reason) + scores, context = scoring.score(eval_id, [c], [sample], enable_thinking=enable_thinking) + assert scores[0].is_correct is True + assert context[f"{eval_id}_blanked_truncated"] == 0 + assert context[f"{eval_id}_pass_at_1_blanked"] == 1.0 + + +@pytest.mark.parametrize("eval_id", list(_EVAL_CASES)) +def test_empty_completion_is_wrong(eval_id): + # The strict view counts a blanked sample wrong without grading it again; + # this holds only because every eval fails an empty completion. + sample, _, _ = _EVAL_CASES[eval_id] + scores, _ = scoring.score(eval_id, [SampleCompletion(id="1", completion="")], [sample]) + assert scores[0].is_correct is False + + def test_list_eval_ids(): ids = scoring.list_eval_ids() assert set(ids) == set(EvalId) diff --git a/packages/pipette-scores/tests/test_text.py b/packages/pipette-scores/tests/test_text.py index a91359b..3cbeb3e 100644 --- a/packages/pipette-scores/tests/test_text.py +++ b/packages/pipette-scores/tests/test_text.py @@ -1,6 +1,7 @@ import pytest -from pipette_scores.scoring.completion_text import remove_thinking_tags +from pipette_scores.scoring.completion_text import grading_text, normalize_reasoning_markup, remove_thinking_tags +from pipette_scores.types import SampleCompletion @pytest.mark.parametrize( @@ -32,3 +33,255 @@ ) def test_remove_thinking_tags(raw, expected): assert remove_thinking_tags(raw) == expected + + +_J = '{"name": "Ada Lovelace", "born": 1815}' + +# Completions as the runtimes deliver them, and what remove_thinking_tags leaves +# once normalize_reasoning_markup has run. +_MARKUP_CASES = [ + # Gemma 4 on llama.cpp: the thought channel, then the answer. + pytest.param( + "<|channel>thought\nThe user wants a JSON object with name and born, nothing else.\n" + f"Draft: {_J}\nLooks right.{_J}", + _J, + id="gemma4-closed", + ), + pytest.param( + "<|channel>thoughtThe answer is $\\boxed{391}$.", "The answer is $\\boxed{391}$.", id="gemma4-empty" + ), + pytest.param(f"{_J}<|channel>thought\n", _J, id="gemma4-trailing-empty"), + # Cut inside the thought: nothing but reasoning. + pytest.param( + "<|channel>thought\nCompute 17*23: 17*20 = 340, 17*3 = 51, total 391. Now double-check whether", + "", + id="gemma4-unterminated", + ), + pytest.param( + "<|channel><|channel>thought\nOption C fits the spectrum.Answer: C", + "Answer: C", + id="gemma4-double-open", + ), + pytest.param("Answer: C", "Answer: C", id="gemma4-stray-close"), + # An answer, a second thought, then the answer again: the thought goes, as + # the template's strip_thinking reads it. + pytest.param( + "I don't know\n<|channel>thought\nThe reply must be one word per line.I\ndon't\nknow", + "I don't know\nI\ndon't\nknow", + id="gemma4-second-thought", + ), + # vLLM without a reasoning parser strips both channel tokens: no delimiter left. + pytest.param( + f"thought\nThe user wants name and born.{_J}", f"thought\nThe user wants name and born.{_J}", id="gemma4-vllm" + ), + # gpt-oss's <|channel|> is a different token. + pytest.param( + "<|channel|>analysis<|message|>We need the option with the lowest energy.<|end|>" + "<|start|>assistant<|channel|>final<|message|>Answer: B", + "Answer: B", + id="gpt-oss-harmony", + ), + # ERNIE-4.5-Thinking: the prompt opens , the answer is wrapped. + pytest.param( + 'The schema needs title and year; Dune was published in 1965.\n\n\n\n{"title": "Dune", ' + '"year": 1965}\n\n', + '{"title": "Dune", "year": 1965}', + id="ernie", + ), + pytest.param( + 'Need fenced output.\n\n\n\n```json\n{"a": 1}\n```\n\n', + '```json\n{"a": 1}\n```', + id="ernie-fenced", + ), + # vLLM's ernie45 parser moves the reasoning out but leaves the wrapper. + pytest.param( + '\n\n\n```json\n{"a": 1}\n```\n\n', '```json\n{"a": 1}\n```', id="ernie-vllm-parser" + ), + # Cut inside the answer: no closing tag. + pytest.param( + "Reason.\n\n\n\nThe answer is (B) because the", + "The answer is (B) because the", + id="ernie-unclosed", + ), + pytest.param("abc\n\ndef", "def", id="ernie-unwrapped"), + # The tags are answer text anywhere but at the start of a section. + pytest.param( + "Reason.\n\nWrap the reply in a element, then close it with .", + "Wrap the reply in a element, then close it with .", + id="response-tag-mid-answer", + ), + pytest.param( + "Return it as ok", + "Return it as ok", + id="response-tag-mid-text", + ), + # Cohere North on vLLM: its markers are not special tokens, so they reach the text. + pytest.param( + "The user wants JSON only, no prose. Keys: city, country.<|END_THINKING|>" + '<|START_TEXT|>{"city": "Paris", "country": "France"}<|END_TEXT|>', + '{"city": "Paris", "country": "France"}', + id="north-on", + ), + pytest.param( + '<|START_TEXT|>{"city": "Paris", "country": "France"}<|END_TEXT|>', + '{"city": "Paris", "country": "France"}', + id="north-off", + ), + # Off mode can still reason inside the text section, then close it. + pytest.param( + '<|START_TEXT|>The densest is option c.\n\nThus answer: C. The answer format: "Answer: C".<|END_THINKING|>' + "<|START_TEXT|>Answer: C<|END_TEXT|>", + "Answer: C", + id="north-off-reasoning", + ), + pytest.param( + "<|START_TEXT|><|START_THINKING|>Option C keeps the three-step order.<|END_THINKING|>" + "<|START_TEXT|>Answer: C<|END_TEXT|>", + "Answer: C", + id="north-off-thinking", + ), + pytest.param("Plan.<|END_THINKING|><|START_TEXT|>The answer is", "The answer is", id="north-unclosed"), + # North on llama.cpp without preserved_tokens: every marker is hidden. + pytest.param( + f"The user wants JSON only, no prose.{_J}", f"The user wants JSON only, no prose.{_J}", id="north-llamacpp" + ), + pytest.param( + "Its replies sit between <|START_TEXT|> and <|END_TEXT|> markers.", + "Its replies sit between <|START_TEXT|> and <|END_TEXT|> markers.", + id="text-markers-mid-text", + ), + # Peeling one wrapper moves the section starts the next one is looked for at. + pytest.param( + "Option C fits.<|START_TEXT|>Answer: C<|END_TEXT|>", + "Answer: C", + id="response-then-text-wrapper", + ), + pytest.param( + "Plan.The marker <|START_TEXT|> opens North's answer.", + "The marker <|START_TEXT|> opens North's answer.", + id="text-marker-mid-answer-after-peel", + ), + # A wrapper nested in another goes too. + pytest.param( + "Option C fits.<|START_TEXT|>Answer: C<|END_TEXT|>", + "Answer: C", + id="nested-wrappers", + ), + # A section also starts where [/THINK] closes the reasoning. + pytest.param("Plan.[/THINK]Answer: B", "Answer: B", id="response-after-mistral-close"), + pytest.param(" 42 ", "42", id="plain"), + pytest.param(f"r\n\n{_J}", _J, id="think"), +] + + +@pytest.mark.parametrize("raw, expected", _MARKUP_CASES) +def test_normalize_reasoning_markup(raw, expected): + assert remove_thinking_tags(normalize_reasoning_markup(raw)) == expected + + +@pytest.mark.parametrize("raw, expected", _MARKUP_CASES) +def test_normalize_reasoning_markup_is_idempotent(raw, expected): + once = normalize_reasoning_markup(raw) + assert normalize_reasoning_markup(once) == once + + +@pytest.mark.parametrize( + "raw", + [ + "the answer is 42", + "reasoninganswer", + "leftover reasoninganswer", + "answertruncated reasoning", + 'assistantfinal{"x":1}', + "Return it as ok", + ], +) +def test_normalize_reasoning_markup_leaves_other_text_alone(raw): + assert normalize_reasoning_markup(raw) == raw + + +_PROMPT_OPENED = "Let me compare the four options. Option A predicts" + + +@pytest.mark.parametrize( + "raw, stop_reason, enable_thinking, blanked", + [ + # The prompt opened the reasoning and the cap cut it: no tag at all. + (_PROMPT_OPENED, "truncated", True, True), + # A repetition stop (vLLM's repetition_detection) also cut generation. + (_PROMPT_OPENED, "doom_loop", True, True), + # Thinking off or not reported: graded as before. + (_PROMPT_OPENED, "truncated", False, False), + (_PROMPT_OPENED, "truncated", None, False), + # Not cut, or not known to be. + (_PROMPT_OPENED, "eos", True, False), + (_PROMPT_OPENED, "unknown", True, False), + (_PROMPT_OPENED, "failure", True, False), + (_PROMPT_OPENED, None, True, False), + # Nothing to drop: generation produced no text. + ("", "truncated", True, False), + # The reasoning closed and the answer was cut: graded as is. + ("Reason.\n\n\n\nThe answer is (B) because the", "truncated", True, False), + ("<|channel>thought\nOption C fits.The answer is", "truncated", True, False), + ("Plan.<|END_THINKING|><|START_TEXT|>The answer is", "truncated", True, False), + ("[THINK]plan[/THINK]Answer: B", "truncated", True, False), + ( + "<|channel|>analysis<|message|>Done.<|end|><|start|>assistant<|channel|>final<|message|>Answer", + "truncated", + True, + False, + ), + # gpt-oss cut in its analysis channel. + ( + "<|channel|>analysis<|message|>We need to compute the integral. Let u = x^2, then du =", + "truncated", + True, + True, + ), + # Gemma 4 cut inside its thought. + ("<|channel>thought\nCompute 17*23: 17*20 = 340", "truncated", True, True), + ], +) +def test_grading_text(raw, stop_reason, enable_thinking, blanked): + text, cut = grading_text(SampleCompletion(id="1", completion=raw, stop_reason=stop_reason), enable_thinking) + # The flag only marks the samples the strict view blanks; the text graded + # is the rewritten completion either way. + assert cut is blanked + assert text == normalize_reasoning_markup(raw) + + +# What a cut completion is graded on: the text after the markup rewrite and +# remove_thinking_tags. For completions without Gemma 4, North or ERNIE markup +# this is what liquid_evals (d5534cf, parser.remove_thinking_tags, called +# without opener_in_prompt by gpqa_diamond_aa, ifbench_aa, ifstruct and +# math500) grades for the same response. +@pytest.mark.parametrize("enable_thinking", [True, False, None]) +@pytest.mark.parametrize("stop_reason", ["truncated", "doom_loop"]) +@pytest.mark.parametrize( + "raw, graded", + [ + # The prompt opened the reasoning: no tag, graded whole. + pytest.param(_PROMPT_OPENED, _PROMPT_OPENED, id="prompt-opened"), + # The completion opened itself: everything from the tag on goes. + pytest.param("Option B fits. Answer: B. Double-check", "", id="opened-in-text"), + pytest.param("Sure.\nOption B fits. Answer: B", "Sure.", id="opened-after-text"), + pytest.param("[THINK]Option B fits. Answer: B", "", id="mistral-opened"), + # gpt-oss cut in its analysis channel: no final channel, graded whole. + pytest.param( + "<|channel|>analysis<|message|>Let u = x^2, then du =", + "<|channel|>analysis<|message|>Let u = x^2, then du =", + id="gpt-oss-analysis", + ), + # The reasoning closed and the answer was cut: the answer so far. + pytest.param( + "Reason.\n\n\nThe answer is (B) because the", "The answer is (B) because the", id="closed" + ), + # Markup liquid_evals does not read keeps the scorer's reading. + pytest.param("<|channel>thought\nCompute 17*23: 17*20 = 340", "", id="gemma4-thought"), + pytest.param("<|START_THINKING|>Option B fits. Answer: B", "", id="north-thinking"), + pytest.param("Reason.\n\n\n\nThe answer is (B", "The answer is (B", id="ernie-response"), + ], +) +def test_cut_completion_is_graded_like_liquid_evals(raw, graded, stop_reason, enable_thinking): + text, _ = grading_text(SampleCompletion(id="1", completion=raw, stop_reason=stop_reason), enable_thinking) + assert remove_thinking_tags(text) == graded diff --git a/packages/pipette-scores/tests/test_vlm_answer.py b/packages/pipette-scores/tests/test_vlm_answer.py new file mode 100644 index 0000000..d9ea71d --- /dev/null +++ b/packages/pipette-scores/tests/test_vlm_answer.py @@ -0,0 +1,241 @@ +"""The deterministic fallbacks after VLMEvalKit's exact matching (vlm_answer).""" + +import pytest + +import pipette_scores.scoring as scoring +from pipette_scores.scoring.vlm_answer import option, option_hit, yes_no +from pipette_scores.types import SampleCompletion, VlmSample + +ABCD = {"A": "Left side", "B": "Right side", "C": "Center", "D": "Behind"} +ABCDEF = {**ABCD, "E": "E", "F": "F"} + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + ("B", "B"), + ("A. Left side", "A"), + ("The answer is C.", "C"), + ], +) +def test_an_answer_upstream_reads_is_scored_as_upstream_scores_it(text, expected): + assert option(text, ABCD) == (expected, False) + + +@pytest.mark.parametrize( + ("text", "choices", "expected"), + [ + # Verbose answers that mention other letters before committing to one. + ( + "He stands near A, with B and C to his right.\n\nSo, the correct answer is: \n**A. Left side** ✅", + ABCD, + "A", + ), + ( + "Only **D** matches, not A or B.\n\nThus, the correct answer is:\n\n> **D. D**\n\n✅ Final Answer: **D**", + ABCD, + "D", + ), + # The last stated answer wins over an earlier quoted option. + ("Columns C, F, B, D, A grow.\n\n> **E**\n\n---\n\n✅ Correct Answer: **A. E**", ABCDEF, "A"), + ("Not B or D; the houses are small.\n\n### Correct Answer: **C. Center** ✅", ABCD, "C"), + ("A and B are wrong, so the answer is (B), not D.", ABCD, "B"), + ("Between A and D, I pick\n\\boxed{C}", ABCD, "C"), + ("A or B?\n\\boxed{\\text{B}}", ABCD, "B"), + # The last line alone, when nothing is stated. + ("Options A, B and C look close, as does D.\nC.", ABCD, "C"), + ], +) +def test_a_committed_answer_is_read_by_the_fallback(text, choices, expected): + assert option(text, choices) == (expected, True) + + +@pytest.mark.parametrize( + "text", + [ + "The answer could be a dog, which neither A nor B shows.", + "Answer: All of the above, since A and B both apply.", + "Final answer: E, as A and B are wrong.", + "It could be A or B; I am not sure.", + ], +) +def test_no_letter_is_guessed(text): + assert option(text, ABCD) == (None, False) + + +@pytest.mark.parametrize( + "text", + [ + '{"answer": "C"}', + 'The middle one.\n\n```json\n{\n "answer": "C"\n}\n```', + "answer: 'C'", + 'They are students in the same course.\n\n`answer`: "C"', + '`answer` "C"', + 'A Pizza Hut serves pizza.\n\n`answer: "C"`', + ], +) +def test_a_json_answer_field_is_read(text): + # Qwen3.5's model card asks for `"answer": "C"` on multiple-choice questions. + assert option(text, ABCD) == ("C", True) + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + ('"B"', "B"), + ("“A”", "A"), + ("‘D’.", "D"), + ('"a"', "A"), + ('"B', "B"), + ("The lane on the right, past the barrier.\n\n'C'", "C"), + ], +) +def test_the_answer_fields_value_alone_is_read(text, expected): + # Qwen3.5-4B often answers with the field's value and no key: `"B"`. + assert option(text, ABCD) == (expected, True) + + +@pytest.mark.parametrize( + "text", + [ + '"E"', + '"B" or "C"', + "I'm not sure about A or B.", + '"Bus"', + '"A man stands left.', + '"I think so."', + # Review cases: the format described or refused, alternatives, a quoted article. + "The required format is `answer`: A, B, C, or D. Unable to decide.", + "Do not output `answer`: A or B.", + 'Either "A" or\n"B".', + 'The sign on the shop reads\n"a"', + ], +) +def test_a_quoted_word_or_two_letters_is_not_an_answer(text): + assert option(text, ABCD) == (None, False) + + +def test_the_last_commitment_wins_across_statements_and_fields(): + assert option('A and C were considered. Final answer: **A**.\n`answer`: "B"', ABCD) == ("B", True) + assert option('`answer`: "B"\nA and B fall short.\nFinal Answer: **C**', ABCD) == ("C", True) + + +def test_a_backticked_yes_no_field_is_read(): + assert yes_no('Both yes and no were considered.\n`answer`: "no"\nThat is my conclusion.') == ("No", True) + + +def test_a_json_yes_no_answer_field_is_read(): + # Mentions both words, so upstream's rule reads nothing and the field decides. + assert yes_no('Yes, there is a dog, but no cat.\n{"answer": "no"}') == ("No", True) + + +@pytest.mark.parametrize( + "text", + [ + "I cannot answer yes or no, as the image is missing.", + "The view is unclear. Therefore, I cannot answer yes or no with certainty.", + "Without more context I can't answer yes/no.", + ], +) +def test_a_reply_that_cannot_choose_is_not_read(text): + # Found by the VLMEvalKit cross-check: the statement rule read the "yes" of + # "answer yes or no" as the answer. + assert yes_no(text) == ("Unknown", False) + + +def test_a_quoted_option_line_is_read(): + # Upstream reads neither: both options' texts occur in "Not the same". + same = {"A": "The same", "B": "Not the same"} + assert option('The two shapes differ in size.\n\n"B. Not the same"', same) == ("B", True) + # Quoted prose that names an option is not an answer. + assert option('"B. Not the same" is one of the candidates, but no selection is possible.', same) == ( + None, + False, + ) + + +def test_upstreams_own_reading_stands(): + # Upstream's case-insensitive "answer is X" reads an article as option A. + # A quirk of the vendored rule, kept as upstream scores it. + assert option("The answer is a dog, which neither A nor B shows.", ABCD) == ("A", False) + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + # Answer first, explained after (InternVL3.5's shape). + ( + "B. Right side\n\nThe man stands near the right edge of the frame, beside a lamp post, while the left side and the center of the stage are empty apart from a drum kit and some amplifiers, and nothing is behind him.", + "B", + ), + ( + "**D**\n\nThe man stands near the right edge of the frame, beside a lamp post, while the left side and the center of the stage are empty apart from a drum kit and some amplifiers, and nothing is behind him.", + "D", + ), + ( + "(C) Center\n\nThe man stands near the right edge of the frame, beside a lamp post, while the left side and the center of the stage are empty apart from a drum kit and some amplifiers, and nothing is behind him.", + "C", + ), + # A line that is exactly an option, after a lead-in. + ( + "Where he stands:\n\nA. Left side\n\nThe man stands near the right edge of the frame, beside a lamp post, while the left side and the center of the stage are empty apart from a drum kit and some amplifiers, and nothing is behind him.", + "A", + ), + ], +) +def test_an_answer_given_first_is_read(text, expected): + assert option(text, ABCD) == (expected, True) + + +@pytest.mark.parametrize( + "text", + [ + # A walk through several options commits to none of them. + "A. Left side: possible, but B is likelier.\nB. Right side: also possible.\nHard to say.", + # A line naming an option whose text is not that option's. + "The scene:\n\nA. a dog on the left\n\nBut B and C are unclear.", + ], +) +def test_a_walk_through_or_a_mismatched_line_is_not_read(text): + assert option(text, ABCD) == (None, False) + + +def test_a_yes_or_no_given_first_is_read(): + assert yes_no("Yes, there is a dog. There is no cat.") == ("Yes", True) + assert yes_no("**No**. The sign says yes, but no dog is visible.") == ("No", True) + + +def test_option_hit_compares_with_the_ground_truth(): + text = "Not B.\n\nThe correct answer is: **A**" + assert option_hit(text, ABCD, "A") == (True, True) + assert option_hit(text, ABCD, "B") == (False, True) + assert option_hit("No idea, A or B.", ABCD, "A") == (False, False) + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + ("Yes", ("Yes", False)), + ("No, there is none.", ("No", False)), + ("Is there no dog? There is one, so yes.\n\nFinal Answer: **Yes**", ("Yes", True)), + ("Yes, a car; no, not a bus.\n\\boxed{No}", ("No", True)), + ("Is there a car? Yes, but no bus is visible.\nNo.", ("No", True)), + ("Yes and no.", ("Unknown", False)), + ], +) +def test_yes_no(text, expected): + assert yes_no(text) == expected + + +def test_a_reasoning_model_is_read_after_its_reasoning(): + sample = VlmSample( + id="1", prompt="q", answer=None, raw_doc={**ABCD, "answer": "B", "category": "c", "l2_category": "x"} + ) + completion = SampleCompletion( + id="1", + completion="A? C? It is not D.\nLooking again, A and C are wrong.\n\nFinal Answer: **B**", + stop_reason="eos", + ) + scores, ctx = scoring.score("mmstar", [completion], [sample], enable_thinking=True) + assert scores[0].is_correct + assert ctx["fallback_answers"] == 1 diff --git a/packages/pipette-scores/tests/test_vlm_dataset_builders.py b/packages/pipette-scores/tests/test_vlm_dataset_builders.py new file mode 100644 index 0000000..d6ce518 --- /dev/null +++ b/packages/pipette-scores/tests/test_vlm_dataset_builders.py @@ -0,0 +1,461 @@ +"""Unit tests for the VLM dataset builder scripts (synthetic TSV rows, no network). + +Prompt-format expectations follow +pipette_scores/scoring/vlm_upstream/PORTING_NOTES.md (VLMEvalKit +``ImageMCQDataset.build_prompt`` @ 8eca8e5). The per-benchmark specs are +loaded from the one-shot scripts in ``packages/pipette-scores/scripts/``. +""" + +import base64 +import importlib.util +import io +import json +import pathlib + +import pandas as pd +import pytest +from PIL import Image + +from pipette_scores.dataset_catalog import _vlm_build, _vlm_common + +_SCRIPTS_DIR = pathlib.Path(__file__).parents[1] / "scripts" +_EVAL_IDS = ["mme", "mmstar", "ocrbench", "mmmu_val", "realworldqa", "mmbench_en_dev", "pope"] + + +def _load_script(eval_id: str): + path = _SCRIPTS_DIR / f"build_{eval_id}_dataset.py" + spec = importlib.util.spec_from_file_location(f"build_{eval_id}_dataset", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +_SCRIPTS = {eval_id: _load_script(eval_id) for eval_id in _EVAL_IDS} +SPECS = {eval_id: mod.SPEC for eval_id, mod in _SCRIPTS.items()} + + +def _png_b64(size: int = 8, color=(255, 0, 0)) -> str: + buf = io.BytesIO() + Image.new("RGB", (size, size), color).save(buf, format="PNG") + return base64.b64encode(buf.getvalue()).decode("ascii") + + +def _jpeg_bytes(size: int = 128) -> bytes: + buf = io.BytesIO() + Image.new("RGB", (size, size), (0, 128, 255)).save(buf, format="JPEG") + return buf.getvalue() + + +def test_load_tsv_keeps_literal_none_options(tmp_path): + # MMMU validation_Geography_15: option D is the text "None" and the answer. + path = tmp_path / "t.tsv" + path.write_text("index\tquestion\tA\tB\tC\tD\tanswer\n1\tq\tx\ty\t\tNone\tD\n") + row = _vlm_build.load_tsv(path).iloc[0] + assert row["D"] == "None" + assert pd.isna(row["C"]) + + +# --------------------------------------------------------------------------- +# Prompt format (PORTING_NOTES MCQ spec) +# --------------------------------------------------------------------------- + + +class TestMcqPrompt: + def test_full_format_with_hint(self): + line = { + "question": "What is shown?", + "hint": "Look closely.", + "A": "a cat", + "B": "a dog", + "C": "a bird", + "D": "a fish", + } + assert _vlm_build.build_mcq_prompt(line) == ( + "Hint: Look closely.\n" + "Question: What is shown?\n" + "Options:\n" + "A. a cat\n" + "B. a dog\n" + "C. a bird\n" + "D. a fish\n" + "Please select the correct answer from the options above. \n" + ) + + def test_no_hint_and_missing_options(self): + line = {"question": "Pick one.", "hint": float("nan"), "A": "yes", "B": "no", "C": float("nan"), "D": None} + assert _vlm_build.build_mcq_prompt(line) == ( + "Question: Pick one.\nOptions:\nA. yes\nB. no\nPlease select the correct answer from the options above. \n" + ) + + def test_open_question_has_no_options_block(self): + line = {"question": "Describe the image.", "A": float("nan"), "B": float("nan")} + assert _vlm_build.build_mcq_prompt(line) == "Question: Describe the image.\n" + + def test_bare_prompt_is_question_verbatim(self): + q = "Is there a car in the image? Please answer yes or no." + assert _vlm_build.build_bare_prompt({"question": q}) == q + + +# --------------------------------------------------------------------------- +# MMBench: circular rotations + image pointer dedup +# --------------------------------------------------------------------------- + + +class TestMmbench: + def _df(self): + img = _png_b64() + return pd.DataFrame( + [ + # base question + { + "index": 42, + "question": "What color?", + "hint": float("nan"), + "A": "red", + "B": "blue", + "C": "green", + "D": "gray", + "answer": "A", + "category": "color", + "l2-category": "attribute", + "image": img, + }, + # circular rotation row: image cell is a pointer to index 42 + { + "index": 1000042, + "question": "What color?", + "hint": float("nan"), + "A": "blue", + "B": "green", + "C": "gray", + "D": "red", + "answer": "D", + "category": "color", + "l2-category": "attribute", + "image": "42", + }, + ] + ) + + def test_rotation_rows_preserved_with_original_index(self): + df = _vlm_build.resolve_image_pointers(self._df()) + rows, _ = _vlm_build.build_rows(df, SPECS["mmbench_en_dev"], max_image_px=None) + assert len(rows) == 2 + indices = [json.loads(r["raw_doc"])["index"] for r in rows] + assert indices == [42, 1000042] + # circular grouping key: rotations of one base question share index % 1e6 + assert indices[1] % int(1e6) == indices[0] + + def test_image_pointer_resolves_to_same_bytes(self): + df = _vlm_build.resolve_image_pointers(self._df()) + rows, _ = _vlm_build.build_rows(df, SPECS["mmbench_en_dev"], max_image_px=None) + assert rows[0]["_images"] == rows[1]["_images"] + assert rows[1]["num_images"] == 1 + + def test_unresolvable_pointer_raises(self): + df = self._df() + df.loc[1, "image"] = "999" # no such index + with pytest.raises(ValueError): + _vlm_build.resolve_image_pointers(df) + + def test_raw_doc_carries_options_answer_and_hint(self): + df = pd.DataFrame( + [ + { + "index": 7, + "question": "Q?", + "hint": "H.", + "A": "a", + "B": "b", + "answer": "B", + "category": "c", + "l2-category": "l2", + "image": _png_b64(), + } + ] + ) + rows, _ = _vlm_build.build_rows(df, SPECS["mmbench_en_dev"], max_image_px=None) + doc = json.loads(rows[0]["raw_doc"]) + assert doc["A"] == "a" and doc["B"] == "b" + assert doc["answer"] == "B" + assert doc["hint"] == "H." + assert doc["category"] == "c" and doc["l2-category"] == "l2" + assert "image" not in doc + assert rows[0]["prompt"].startswith("Hint: H.\nQuestion: Q?\n") + + +# --------------------------------------------------------------------------- +# MME: pairing key +# --------------------------------------------------------------------------- + + +class TestMme: + def test_raw_doc_carries_pairing_key_shared_by_question_pair(self): + img = _png_b64() + df = pd.DataFrame( + [ + { + "index": 0, + "category": "existence", + "image_path": "existence/0001.jpg", + "question": "Is there a dog? Please answer yes or no.", + "answer": "Yes", + "image": img, + }, + { + "index": 1, + "category": "existence", + "image_path": "existence/0001.jpg", + "question": "Is there a cat? Please answer yes or no.", + "answer": "No", + "image": img, + }, + ] + ) + rows, _ = _vlm_build.build_rows(df, SPECS["mme"], max_image_px=None) + docs = [json.loads(r["raw_doc"]) for r in rows] + assert docs[0]["image_path"] == docs[1]["image_path"] == "existence/0001.jpg" + assert docs[0]["answer"] == "Yes" and docs[1]["answer"] == "No" + assert rows[0]["prompt"] == "Is there a dog? Please answer yes or no." + assert rows[0]["sample_id"] != rows[1]["sample_id"] + + +# --------------------------------------------------------------------------- +# MMMU: VAL filter, open questions, multi-image cells +# --------------------------------------------------------------------------- + + +class TestMmmu: + def test_val_filter_excludes_dev_and_asserts_count(self): + df = pd.DataFrame( + { + "index": range(1050), + "split": ["dev"] * 150 + ["validation"] * 900, + } + ) + out = _SCRIPTS["mmmu_val"].keep_validation(df) + assert len(out) == 900 + assert (out["split"] == "validation").all() + + with pytest.raises(SystemExit): + _SCRIPTS["mmmu_val"].keep_validation(df.head(1000)) + + def test_multi_image_cell_and_placeholder_prompt(self): + img_a, img_b = _png_b64(color=(255, 0, 0)), _png_b64(color=(0, 255, 0)) + df = pd.DataFrame( + [ + { + "index": 1, + "question": "Compare with .", + "split": "validation", + "A": "left", + "B": "right", + "answer": "A", + "category": "Art", + "image": str([img_a, img_b]), + } + ] + ) + rows, _ = _vlm_build.build_rows(df, SPECS["mmmu_val"], max_image_px=None) + assert rows[0]["num_images"] == 2 + assert rows[0]["_images"] == [base64.b64decode(img_a), base64.b64decode(img_b)] + # placeholders stay in the prompt text (interleaving happens at load time) + assert "" in rows[0]["prompt"] and "" in rows[0]["prompt"] + + def test_open_question_options_stay_missing_in_raw_doc(self): + df = pd.DataFrame( + [ + { + "index": 2, + "question": "How many?", + "split": "validation", + "A": float("nan"), + "B": float("nan"), + "answer": "42", + "category": "Math", + "image": _png_b64(), + } + ] + ) + rows, _ = _vlm_build.build_rows(df, SPECS["mmmu_val"], max_image_px=None) + doc = json.loads(rows[0]["raw_doc"]) + assert doc["A"] is None and doc["B"] is None + assert doc["answer"] == "42" + assert rows[0]["prompt"] == "Question: How many?\n" + + +# --------------------------------------------------------------------------- +# OCRBench / POPE raw_doc fields +# --------------------------------------------------------------------------- + + +class TestYornAndOcr: + def test_ocrbench_category_case_and_answer_list(self): + df = pd.DataFrame( + [ + { + "index": 3, + "image": _png_b64(), + "question": "Recognize the expression.", + "answer": "['x^2', 'x**2']", + "category": "Handwritten Mathematical Expression Recognition", + } + ] + ) + rows, _ = _vlm_build.build_rows(df, SPECS["ocrbench"], max_image_px=None) + doc = json.loads(rows[0]["raw_doc"]) + assert doc["category"] == "Handwritten Mathematical Expression Recognition" + assert doc["answer"] == "['x^2', 'x**2']" # stringified list, verbatim + assert rows[0]["prompt"] == "Recognize the expression." + + def test_pope_category_and_bare_prompt(self): + df = pd.DataFrame( + [ + { + "image": _png_b64(), + "question": "Is there a bottle in the image?", + "answer": "Yes", + "category": "adversarial,popular", + "index": 9, + } + ] + ) + rows, _ = _vlm_build.build_rows(df, SPECS["pope"], max_image_px=None) + doc = json.loads(rows[0]["raw_doc"]) + assert doc["category"] == "adversarial,popular" + assert rows[0]["prompt"] == "Is there a bottle in the image?" + assert json.loads(rows[0]["answer"]) == "Yes" + + +# --------------------------------------------------------------------------- +# Image handling +# --------------------------------------------------------------------------- + + +class TestImages: + def test_default_keeps_original_bytes(self): + data = _jpeg_bytes(128) + assert _vlm_build.maybe_resize(data, None) is data + + def test_max_image_px_downscales_oversized(self): + data = _jpeg_bytes(128) + out = _vlm_build.maybe_resize(data, 64) + with Image.open(io.BytesIO(out)) as img: + assert max(img.size) == 64 + assert img.format == "JPEG" + + def test_max_image_px_keeps_small_images_untouched(self): + data = _jpeg_bytes(32) + assert _vlm_build.maybe_resize(data, 64) is data + + def test_sniff_formats(self): + assert _vlm_build.sniff_image_format(_jpeg_bytes()) == "jpeg" + assert _vlm_build.sniff_image_format(base64.b64decode(_png_b64())) == "png" + + +# --------------------------------------------------------------------------- +# Specs, metadata + license fields +# --------------------------------------------------------------------------- + + +class TestMetadataAndSpecs: + def test_all_seven_builders_have_specs(self): + assert sorted(SPECS) == sorted(_EVAL_IDS) + for eval_id, spec in SPECS.items(): + assert spec.eval_id == eval_id + + def test_every_spec_has_license_fields_and_pin(self): + for spec in SPECS.values(): + assert spec.license + assert spec.license_notes + assert spec.url.startswith("https://") + assert len(spec.md5) == 32 + + def test_realworldqa_is_nd_with_prominent_notice(self): + spec = SPECS["realworldqa"] + assert "CC-BY-ND" in spec.license + assert spec.extra_notice and "MUST NOT be redistributed" in spec.extra_notice + + def test_metadata_json_fields(self, tmp_path): + spec = SPECS["pope"] + _vlm_build.write_metadata( + tmp_path, spec, "full", 3, "d" * 64, "e" * 64, {"png": 3}, max_image_px=None, limit=None + ) + meta = json.loads((tmp_path / "metadata.json").read_text()) + assert meta["eval_id"] == "pope" + assert meta["dataset"] == "full" + assert meta["source"]["url"] == spec.url + assert meta["source"]["md5"] == spec.md5 + assert meta["num_samples"] == 3 + assert meta["content_sha256"] == "d" * 64 + assert meta["metadata_content_sha256"] == "e" * 64 + assert meta["max_image_px"] is None + assert meta["license"] == spec.license + assert meta["license_notes"] == spec.license_notes + assert "built_at" in meta + assert "internal use only" in meta["redistribution"] + + def test_content_sha256_is_order_independent_and_image_sensitive(self): + row = { + "sample_id": "abc", + "prompt": "p", + "answer": None, + "raw_doc": "{}", + "_images": [b"\x01\x02"], + } + row2 = {**row, "sample_id": "def"} + assert _vlm_build.content_sha256([row, row2]) == _vlm_build.content_sha256([row2, row]) + assert _vlm_build.content_sha256([row]) != _vlm_build.content_sha256([{**row, "_images": [b"\x01\x03"]}]) + + +# --------------------------------------------------------------------------- +# Round-trip: builder output -> pipette_scores VLM loader +# --------------------------------------------------------------------------- + + +class TestRoundTrip: + def test_parquet_loads_through_vlm_common(self, tmp_path): + img_a, img_b = _png_b64(color=(255, 0, 0)), _png_b64(color=(0, 255, 0)) + df = pd.DataFrame( + [ + { + "index": 1, + "question": "Compare and .", + "split": "validation", + "A": "same", + "B": "different", + "answer": "B", + "category": "Art", + "image": str([img_a, img_b]), + }, + { + "index": 2, + "question": "What is this?", + "split": "validation", + "A": "x", + "B": "y", + "answer": "A", + "category": "Art", + "image": img_a, + }, + ] + ) + rows, _ = _vlm_build.build_rows(df, SPECS["mmmu_val"], max_image_px=None) + _vlm_build.write_dataset(tmp_path, rows) + + eval_samples = _vlm_common.load_eval_samples(tmp_path) + assert len(eval_samples) == 2 + assert eval_samples[0].raw_doc["answer"] == "B" + + prompts = _vlm_common.load_prompt_samples(tmp_path, interleave_images=True) + blocks = prompts[0].messages[0].content + # placeholders are interleaved: text, image, text, image, text + kinds = [b["type"] for b in blocks] + assert kinds == ["text", "image_url", "text", "image_url", "text"] + assert blocks[0]["text"] == "Question: Compare " + # PNG bytes are emitted with a png mime, not the historical jpeg default + assert blocks[1]["image_url"]["url"].startswith("data:image/png;base64,") + + # row without placeholders: images first, then the full prompt text + kinds2 = [b["type"] for b in prompts[1].messages[0].content] + assert kinds2 == ["image_url", "text"] diff --git a/packages/pipette-scores/tests/test_vlm_dataset_loader.py b/packages/pipette-scores/tests/test_vlm_dataset_loader.py new file mode 100644 index 0000000..9016198 --- /dev/null +++ b/packages/pipette-scores/tests/test_vlm_dataset_loader.py @@ -0,0 +1,149 @@ +"""Unit tests for the shared VLM dataset loader (_vlm_common).""" + +from pipette_scores.dataset_catalog._vlm_common import ( + _content_blocks, + _image_format, + _image_to_data_uri, + _interleaved_blocks, +) + +_JPEG = b"\xff\xd8\xff\xe0" + b"\x00" * 8 +_PNG = b"\x89PNG\r\n\x1a\n" + b"\x00" * 8 +_WEBP = b"RIFF\x00\x00\x00\x00WEBP" + b"\x00" * 4 + + +class TestImageMime: + def test_sniffs_common_formats(self): + assert _image_format(_JPEG) == "jpeg" + assert _image_format(_PNG) == "png" + assert _image_format(_WEBP) == "webp" + assert _image_format(b"GIF89a\x00") == "gif" + assert _image_format(b"BM\x00\x00") == "bmp" + + def test_unknown_defaults_to_jpeg(self): + assert _image_format(b"\x00\x01\x02\x03") == "jpeg" + + def test_data_uri_carries_sniffed_mime(self): + assert _image_to_data_uri(_PNG).startswith("data:image/png;base64,") + assert _image_to_data_uri(_JPEG).startswith("data:image/jpeg;base64,") + + +class TestInterleaving: + def test_no_placeholder_returns_none(self): + assert _interleaved_blocks("Question: what?\n", [_PNG]) is None + + def test_no_images_returns_none(self): + assert _interleaved_blocks("See .", []) is None + + def test_mmmu_style_interleave(self): + blocks = _interleaved_blocks("Compare with .", [_PNG, _JPEG]) + assert [b["type"] for b in blocks] == ["text", "image_url", "text", "image_url", "text"] + assert blocks[0]["text"] == "Compare " + assert blocks[1]["image_url"]["url"].startswith("data:image/png;base64,") + assert blocks[3]["image_url"]["url"].startswith("data:image/jpeg;base64,") + assert blocks[4]["text"] == "." + + def test_repeated_reference_reuses_image(self): + blocks = _interleaved_blocks(" vs ", [_PNG]) + assert [b["type"] for b in blocks] == ["image_url", "text", "image_url"] + + def test_malformed_placeholder_falls_back(self): + # not "" shaped -> None (caller uses images-first layout) + assert _interleaved_blocks("a b", [_PNG]) is None + assert _interleaved_blocks("a b", [_PNG]) is None # out of range + + def test_content_blocks_default_layout(self): + blocks = _content_blocks("plain question", [_PNG, _JPEG]) + assert [b["type"] for b in blocks] == ["image_url", "image_url", "text"] + assert blocks[2]["text"] == "plain question" + + def test_placeholder_stays_text_unless_interleaving(self): + # Upstream splits for MMMU only; MMStar keeps it as text. + blocks = _content_blocks("See .", [_PNG]) + assert [b["type"] for b in blocks] == ["image_url", "text"] + assert blocks[1]["text"] == "See ." + interleaved = _content_blocks("See .", [_PNG], interleave=True) + assert [b["type"] for b in interleaved] == ["text", "image_url", "text"] + + +class TestLoadTimeIntegrity: + """The loader re-verifies the cheap image-free digest from metadata.json, + the VLM analogue of the text loaders' content_sha256 re-check. Backward + compatible: no digest in metadata → no check (datasets built before it).""" + + def _write_dataset(self, tmp_path, meta_extra: dict | None): + import json + + import pyarrow as pa + import pyarrow.parquet as pq + + rows = [ + { + "sample_id": "s1", + "doc_id": "d1", + "prompt": "Q1?", + "answer": json.dumps("A"), + "raw_doc": json.dumps({"k": 1}), + "num_images": 0, + }, + { + "sample_id": "s2", + "doc_id": "d2", + "prompt": "Q2?", + "answer": json.dumps("B"), + "raw_doc": json.dumps({"k": 2}), + "num_images": 0, + }, + ] + d = tmp_path / "ds" + d.mkdir() + pq.write_table(pa.Table.from_pylist(rows), d / "test.parquet") + meta = {"eval_id": "x", "num_samples": len(rows)} + if meta_extra: + meta.update(meta_extra) + (d / "metadata.json").write_text(json.dumps(meta)) + return d, rows + + def test_matching_digest_loads(self, tmp_path): + from pipette_scores.dataset_catalog._vlm_common import ( + load_eval_samples, + metadata_content_sha256, + ) + + d, rows = self._write_dataset(tmp_path, {"metadata_content_sha256": None}) + # write the real digest + import json + + digest = metadata_content_sha256(rows) + meta = json.loads((d / "metadata.json").read_text()) + meta["metadata_content_sha256"] = digest + (d / "metadata.json").write_text(json.dumps(meta)) + + samples = load_eval_samples(d) + assert [s.id for s in samples] == ["s1", "s2"] + + def test_mismatched_digest_raises(self, tmp_path): + import pytest + + from pipette_scores.dataset_catalog._vlm_common import load_eval_samples + + d, _ = self._write_dataset(tmp_path, {"metadata_content_sha256": "deadbeef" * 8}) + with pytest.raises(ValueError, match="integrity check failed"): + load_eval_samples(d) + + def test_absent_digest_skips_check(self, tmp_path): + # No metadata_content_sha256 → loads unguarded (pre-feature datasets). + from pipette_scores.dataset_catalog._vlm_common import load_eval_samples + + d, _ = self._write_dataset(tmp_path, None) + assert len(load_eval_samples(d)) == 2 + + def test_digest_is_image_free_and_order_independent(self, tmp_path): + from pipette_scores.dataset_catalog._vlm_common import metadata_content_sha256 + + _, rows = self._write_dataset(tmp_path, None) + # order-independent + assert metadata_content_sha256(rows) == metadata_content_sha256(list(reversed(rows))) + # image bytes don't affect it (adding an image column is a no-op) + with_img = [{**r, "image_0": b"\xff\xd8\xff", "num_images": 1} for r in rows] + assert metadata_content_sha256(with_img) == metadata_content_sha256(rows) diff --git a/packages/pipette-scores/tests/test_vlm_scoring.py b/packages/pipette-scores/tests/test_vlm_scoring.py new file mode 100644 index 0000000..54723c2 --- /dev/null +++ b/packages/pipette-scores/tests/test_vlm_scoring.py @@ -0,0 +1,641 @@ +"""Unit tests for the VLM scorers (extraction + metrics vendored from VLMEvalKit). + +All samples are synthetic (no parquet / LFS data); expected values follow +VLMEvalKit @ 8eca8e5 semantics. +""" + +import pytest + +import pipette_scores.scoring as scoring +from pipette_scores.scoring import ( + mmbench_en_dev, + mme, + mmmu_val, + mmstar, + ocrbench, + pope, + realworldqa, +) +from pipette_scores.scoring.vlm_upstream.matching_util import can_infer +from pipette_scores.scoring.vlm_upstream.yorn import YOrN_Extraction +from pipette_scores.types import SampleCompletion, VlmSample + + +def _vlm(id: str, raw_doc: dict, prompt: str = "q") -> VlmSample: + return VlmSample(id=id, prompt=prompt, answer=None, raw_doc=raw_doc) + + +def _comp(id: str, text: str) -> SampleCompletion: + return SampleCompletion(id=id, completion=text) + + +# --------------------------------------------------------------------------- +# can_infer (VLMEvalKit matching_util) +# --------------------------------------------------------------------------- + +CHOICES = {"A": "teddy bear", "B": "rabbit", "C": "cat", "D": "dog"} + + +class TestCanInfer: + def test_bare_letter(self): + assert can_infer("B", CHOICES) == "B" + + def test_answer_phrase(self): + assert can_infer("The correct choice is B", CHOICES) == "B" + + def test_parenthesized_letter(self): + assert can_infer("The answer is (C).", CHOICES) == "C" + + def test_letter_with_period(self): + assert can_infer("D. dog", CHOICES) == "D" + + def test_option_text_match(self): + assert can_infer("It looks like a rabbit to me", CHOICES) == "B" + + def test_position_constraint_rejects_early_letter(self): + # The single choice-letter token must be among the last 4 tokens. + ans = "B is one option but after long reflection over many tokens I remain unsure" + assert can_infer(ans, CHOICES) is False + + def test_unparseable_is_false(self): + assert can_infer("I cannot tell what this image shows", CHOICES) is False + + def test_two_letters_no_match(self): + assert can_infer("Either B or C", CHOICES) is False + + def test_refusal_maps_to_z(self): + assert can_infer("Sorry, I can't help with images of people yet.", CHOICES) == "Z" + + def test_long_text_answer_rejected(self): + long = "word " * 50 + "rabbit" + assert can_infer(long, CHOICES) is False + + def test_verbose_answer_is_letter(self): + ans = "The correct answer is **B**. Here's why: A and C are both wrong, and D is not shown at all." + assert can_infer(ans, CHOICES) == "B" + + def test_verbose_answer_letter_outside_choices(self): + assert can_infer("The answer is D, I think, or maybe A or C", {"A": "x", "B": "y", "C": "z"}) is False + + +# --------------------------------------------------------------------------- +# Y/N extraction (VLMEvalKit yorn) +# --------------------------------------------------------------------------- + + +class TestYOrNExtraction: + @pytest.mark.parametrize( + "text,expected", + [ + ("The answer is yes.", "Yes"), + ("Yes", "Yes"), + ("yes, there is a dog.", "Yes"), + ("No, there is not.", "No"), + ("The answer is no", "No"), + ("yes and no", "Unknown"), + ("maybe", "Unknown"), + ("", "Unknown"), + ], + ) + def test_extraction(self, text, expected): + assert YOrN_Extraction(text) == expected + + +# --------------------------------------------------------------------------- +# MME — pairing + rating +# --------------------------------------------------------------------------- + + +def _mme_sample(id, category, image, answer): + return _vlm(id, {"category": category, "image_path": image, "answer": answer}) + + +class TestMME: + def test_pairing_and_rating(self): + # existence (perception): img1 both correct, img2 one of two correct + # -> acc = 75, acc_plus = 50, category = 125. No cognition samples. + samples = [ + _mme_sample("1", "existence", "img1.jpg", "Yes"), + _mme_sample("2", "existence", "img1.jpg", "No"), + _mme_sample("3", "existence", "img2.jpg", "Yes"), + _mme_sample("4", "existence", "img2.jpg", "No"), + ] + completions = [ + _comp("1", "The answer is yes."), + _comp("2", "No."), + _comp("3", "Yes, it is."), + _comp("4", "Yes."), # wrong + ] + scores, ctx = mme.score(completions, samples) + assert [s.is_correct for s in scores] == [True, True, True, False] + assert ctx["mme_perception_score"] == 125.0 + assert ctx["mme_cognition_score"] == 0.0 + assert ctx["leaderboard_score"] == 125.0 + assert ctx["accuracy"] == pytest.approx(125.0 / 2800, abs=1e-6) + + def test_acc_plus_requires_both_of_pair(self): + # One of the pair wrong -> acc 50, acc_plus 0 -> category 50. + samples = [ + _mme_sample("1", "code_reasoning", "c1.png", "Yes"), + _mme_sample("2", "code_reasoning", "c1.png", "No"), + ] + completions = [_comp("1", "yes"), _comp("2", "yes")] + _, ctx = mme.score(completions, samples) + assert ctx["mme_cognition_score"] == 50.0 + assert ctx["mme_perception_score"] == 0.0 + + def test_pairs_by_question_id_fallback(self): + # lmms-eval export: question_id is the image relative path. + samples = [ + _vlm("1", {"category": "OCR", "question_id": "ocr/0001.jpg", "answer": "Yes"}), + _vlm("2", {"category": "OCR", "question_id": "ocr/0001.jpg", "answer": "No"}), + ] + completions = [_comp("1", "yes"), _comp("2", "no")] + _, ctx = mme.score(completions, samples) + # both correct: acc 100 + acc_plus 100 + assert ctx["mme_perception_score"] == 200.0 + + +# --------------------------------------------------------------------------- +# POPE — F1 headline +# --------------------------------------------------------------------------- + + +class TestPOPE: + def test_f1_on_imbalanced_set(self): + gts = ["Yes", "Yes", "Yes", "No"] + preds = ["yes", "The answer is yes.", "no", "Yes, there is."] + samples = [_vlm(str(i), {"answer": gt, "category": "adversarial"}) for i, gt in enumerate(gts)] + completions = [_comp(str(i), p) for i, p in enumerate(preds)] + scores, ctx = pope.score(completions, samples) + # tp=2, fp=1, fn=1 -> precision=recall=2/3, f1=2/3; accuracy=0.5 + assert ctx["precision"] == pytest.approx(2 / 3) + assert ctx["recall"] == pytest.approx(2 / 3) + assert ctx["f1"] == pytest.approx(2 / 3) + assert ctx["yes_no_accuracy"] == pytest.approx(0.5) + # Headline metric is F1, not accuracy. + assert ctx["accuracy"] == pytest.approx(2 / 3) + assert ctx["leaderboard_score"] == pytest.approx(200 / 3, abs=1e-3) + assert [s.is_correct for s in scores] == [True, True, False, False] + + def test_unknown_counts_as_no_prediction(self): + samples = [_vlm("1", {"answer": "Yes", "category": "random"})] + completions = [_comp("1", "I am not sure")] + scores, ctx = pope.score(completions, samples) + assert scores[0].is_correct is False + assert ctx["f1"] == 0.0 + + def test_comma_joined_category_exploded(self): + samples = [_vlm("1", {"answer": "Yes", "category": "adversarial,popular"})] + completions = [_comp("1", "yes")] + _, ctx = pope.score(completions, samples) + assert set(ctx["per_category"]) == {"adversarial", "popular"} + + +# --------------------------------------------------------------------------- +# MMMU val — MMMU_preproc + can_infer +# --------------------------------------------------------------------------- + + +class TestMMMUVal: + def test_mcq_question(self): + raw = {"A": "1", "B": "2", "C": "3", "D": "4", "answer": "B", "category": "Math"} + scores, ctx = mmmu_val.score([_comp("1", "The answer is B.")], [_vlm("1", raw)]) + assert scores[0].is_correct is True + assert ctx["accuracy"] == 1.0 + assert ctx["num_open_questions_reformulated"] == 0 + + def test_open_question_reformulated_to_two_choice(self): + raw = {"answer": "42", "category": "Physics"} # no option columns -> open + scores, ctx = mmmu_val.score([_comp("1", "The answer is 42")], [_vlm("1", raw)]) + assert scores[0].is_correct is True + assert ctx["num_open_questions_reformulated"] == 1 + + def test_open_question_wrong_answer(self): + raw = {"answer": "42", "category": "Physics"} + scores, _ = mmmu_val.score([_comp("1", "The result is 41")], [_vlm("1", raw)]) + assert scores[0].is_correct is False + + def test_unparseable_counts_wrong_no_random_fallback(self): + raw = {"A": "1", "B": "2", "C": "3", "D": "4", "answer": "B", "category": "Math"} + completions = [_comp("1", "I have no idea, sorry about that friend")] + for _ in range(3): # deterministic across calls + scores, _ = mmmu_val.score(completions, [_vlm("1", raw)]) + assert scores[0].is_correct is False + + +# --------------------------------------------------------------------------- +# MMStar — micro accuracy +# --------------------------------------------------------------------------- + + +class TestMMStar: + def test_micro_not_macro(self): + # cat X: 2 samples 1 hit; cat Y: 1 sample 1 hit. + # micro = 2/3; macro would be (0.5 + 1.0)/2 = 0.75. + samples = [ + _vlm("1", {"A": "red", "B": "blue", "answer": "A", "l2_category": "X"}), + _vlm("2", {"A": "red", "B": "blue", "answer": "A", "l2_category": "X"}), + _vlm("3", {"A": "one", "B": "two", "answer": "B", "l2_category": "Y"}), + ] + completions = [_comp("1", "A"), _comp("2", "B"), _comp("3", "The answer is B")] + _, ctx = mmstar.score(completions, samples) + assert ctx["accuracy"] == pytest.approx(2 / 3) + assert ctx["per_l2_category_accuracy"] == {"X": 0.5, "Y": 1.0} + + def test_prefix_style_answers(self): + samples = [_vlm("1", {"A": "cat", "B": "dog", "C": "fox", "D": "owl", "answer": "C"})] + _, ctx = mmstar.score([_comp("1", "Option (C) is correct")], samples) + assert ctx["accuracy"] == 1.0 + + +# --------------------------------------------------------------------------- +# RealWorldQA — can_infer over A-D +# --------------------------------------------------------------------------- + + +class TestRealWorldQA: + def test_letter_answer(self): + samples = [_vlm("1", {"A": "left", "B": "right", "C": "straight", "answer": "B"})] + scores, ctx = realworldqa.score([_comp("1", "B.")], samples) + assert scores[0].is_correct is True + assert ctx["leaderboard_score"] == 100.0 + + def test_option_text_answer(self): + samples = [_vlm("1", {"A": "left", "B": "right", "C": "forward", "answer": "C"})] + scores, _ = realworldqa.score([_comp("1", "The car should go forward")], samples) + assert scores[0].is_correct is True + + def test_option_text_substring_collision_is_ambiguous(self): + # Faithful VLMEvalKit behavior: 'cat' is a substring of 'catfish', so + # a text answer naming 'catfish' matches two options -> no inference. + samples = [_vlm("1", {"A": "cat", "B": "catfish", "C": "heron", "answer": "B"})] + scores, _ = realworldqa.score([_comp("1", "It is clearly one catfish")], samples) + assert scores[0].is_correct is False + + def test_unparseable_is_wrong(self): + samples = [_vlm("1", {"A": "left", "B": "right", "answer": "A"})] + scores, _ = realworldqa.score([_comp("1", "hmm, tough to say for sure today")], samples) + assert scores[0].is_correct is False + + +# --------------------------------------------------------------------------- +# MMBench — circular grouping +# --------------------------------------------------------------------------- + + +class TestMMBench: + def test_circular_all_rotations_required(self): + base = {"A": "cat", "B": "dog", "answer": "A", "category": "CP"} + samples = [ + _vlm("1", {**base, "index": 7}), + _vlm("2", {**base, "answer": "B", "index": 1_000_007}), # rotation + ] + # First rotation right, second wrong -> base question is a miss. + scores, ctx = mmbench_en_dev.score([_comp("1", "A"), _comp("2", "A")], samples) + assert [s.is_correct for s in scores] == [True, False] + assert ctx["num_questions"] == 1 + assert ctx["num_hits"] == 0 + assert ctx["accuracy"] == 0.0 + assert ctx["per_rotation_accuracy"] == 0.5 + + def test_circular_hit(self): + base = {"A": "cat", "B": "dog", "category": "CP"} + samples = [ + _vlm("1", {**base, "answer": "A", "index": 7}), + _vlm("2", {**base, "answer": "B", "index": 1_000_007}), + ] + _, ctx = mmbench_en_dev.score([_comp("1", "A"), _comp("2", "The answer is B")], samples) + assert ctx["accuracy"] == 1.0 + assert ctx["num_hits"] == 1 + + +# --------------------------------------------------------------------------- +# OCRBench — category-keyed matching, HME case sensitivity +# --------------------------------------------------------------------------- + +HME = "Handwritten Mathematical Expression Recognition" + + +class TestOCRBench: + def test_hme_case_sensitive(self): + samples = [_vlm("1", {"answer": ["X^{2}+Y"], "category": HME})] + scores, _ = ocrbench.score([_comp("1", "x^{2}+y")], samples) + assert scores[0].is_correct is False # case mismatch + + scores, _ = ocrbench.score([_comp("1", "The expression is X^{2}+Y")], samples) + assert scores[0].is_correct is True + + def test_hme_whitespace_stripped(self): + samples = [_vlm("1", {"answer": ["a + b"], "category": HME})] + scores, _ = ocrbench.score([_comp("1", "a+b")], samples) + assert scores[0].is_correct is True + + def test_other_categories_case_insensitive(self): + samples = [_vlm("1", {"answer": ["Hello"], "category": "Regular Text Recognition"})] + scores, ctx = ocrbench.score([_comp("1", "the sign says hello")], samples) + assert scores[0].is_correct is True + assert ctx["leaderboard_score"] == 1.0 + assert ctx["accuracy"] == pytest.approx(1 / 1000) + assert ctx["final_score_norm"] == pytest.approx(0.1) + + def test_stringified_list_answers(self): + samples = [_vlm("1", {"answer": "['abc', 'def']", "category": "Doc-oriented VQA"})] + scores, _ = ocrbench.score([_comp("1", "it reads DEF")], samples) + assert scores[0].is_correct is True + + def test_question_type_field_accepted(self): + samples = [_vlm("1", {"answer": ["7"], "question_type": "Digit String Recognition"})] + scores, ctx = ocrbench.score([_comp("1", "7")], samples) + assert scores[0].is_correct is True + assert ctx["final_score"] == 1.0 + + +# --------------------------------------------------------------------------- +# Partial submissions — headline metrics use the DATASET denominator +# --------------------------------------------------------------------------- + + +class TestPartialSubmission: + """A submission covering only part of the dataset must not inflate the + headline: missing samples count as wrong, and ``samples_missing`` reports + the gap. Per-sample scores still cover ONLY the submitted completions + (the response contract with mgmt).""" + + MCQ_RAW = {"A": "red", "B": "blue", "answer": "A"} + + def _mcq_samples(self): + return [_vlm(str(i), dict(self.MCQ_RAW)) for i in range(4)] + + @pytest.mark.parametrize("scorer", [mmstar, realworldqa, mmmu_val]) + def test_mcq_dataset_denominator(self, scorer): + samples = self._mcq_samples() + # 2 of 4 samples submitted, 1 correct: 0.25 (dataset), not 0.5. + completions = [_comp("0", "A"), _comp("1", "B")] + scores, ctx = scorer.score(completions, samples) + assert [s.id for s in scores] == ["0", "1"] # submitted-only response + assert [s.is_correct for s in scores] == [True, False] + assert ctx["accuracy"] == pytest.approx(0.25) + assert ctx["leaderboard_score"] == pytest.approx(25.0) + assert ctx["samples_missing"] == 2 + + @pytest.mark.parametrize("scorer", [mmstar, realworldqa, mmmu_val]) + def test_mcq_full_coverage_no_missing(self, scorer): + samples = self._mcq_samples() + completions = [_comp(str(i), "A") for i in range(4)] + _, ctx = scorer.score(completions, samples) + assert ctx["accuracy"] == 1.0 + assert ctx["samples_missing"] == 0 + + def test_mmbench_groups_built_from_dataset(self): + base = {"A": "cat", "B": "dog", "category": "CP"} + samples = [ + _vlm("1", {**base, "answer": "A", "index": 7}), + _vlm("2", {**base, "answer": "B", "index": 1_000_007}), + _vlm("3", {**base, "answer": "A", "index": 8}), + _vlm("4", {**base, "answer": "B", "index": 1_000_008}), + ] + # Only base question 7 fully submitted (both rotations correct); base + # question 8 not submitted at all -> it is a miss, accuracy 0.5 not 1.0. + completions = [_comp("1", "A"), _comp("2", "B")] + scores, ctx = mmbench_en_dev.score(completions, samples) + assert [s.id for s in scores] == ["1", "2"] + assert ctx["num_questions"] == 2 # from the dataset, not the submission + assert ctx["num_hits"] == 1 + assert ctx["accuracy"] == 0.5 + assert ctx["samples_missing"] == 2 + + def test_mmbench_missing_rotation_is_a_miss(self): + base = {"A": "cat", "B": "dog", "category": "CP"} + samples = [ + _vlm("1", {**base, "answer": "A", "index": 7}), + _vlm("2", {**base, "answer": "B", "index": 1_000_007}), + ] + # Only one of the two rotations submitted (and correct): group misses. + _, ctx = mmbench_en_dev.score([_comp("1", "A")], samples) + assert ctx["num_questions"] == 1 + assert ctx["num_hits"] == 0 + assert ctx["accuracy"] == 0.0 + assert ctx["samples_missing"] == 1 + + def test_pope_missing_counts_as_the_wrong_answer(self): + gts = ["Yes", "Yes", "No", "No"] + samples = [_vlm(str(i), {"answer": gt, "category": "adversarial"}) for i, gt in enumerate(gts)] + # Submit only ids 0 and 2, both correct. Missing id 1 (gt Yes) is a + # false negative; missing id 3 (gt No) a false positive. + # tp=1 fp=1 fn=1 -> precision 0.5, recall 0.5, f1 0.5. + completions = [_comp("0", "yes"), _comp("2", "no")] + scores, ctx = pope.score(completions, samples) + assert [s.id for s in scores] == ["0", "2"] + assert [s.is_correct for s in scores] == [True, True] + assert ctx["precision"] == pytest.approx(0.5) + assert ctx["recall"] == pytest.approx(0.5) + assert ctx["f1"] == pytest.approx(0.5) + assert ctx["accuracy"] == pytest.approx(0.5) # headline = F1 + assert ctx["yes_no_accuracy"] == pytest.approx(0.5) + assert ctx["samples_missing"] == 2 + assert ctx["per_sample_accuracy"] == 1.0 # submitted-only diagnostic + + def test_pope_leaving_out_the_negatives_cannot_raise_f1(self): + gts = ["Yes", "Yes", "No", "No"] + samples = [_vlm(str(i), {"answer": gt, "category": "adversarial"}) for i, gt in enumerate(gts)] + _, partial = pope.score([_comp("0", "yes"), _comp("1", "yes")], samples) + _, full = pope.score([_comp(str(i), gt.lower()) for i, gt in enumerate(gts)], samples) + assert full["f1"] == pytest.approx(1.0) + # tp=2 fp=2 fn=0 -> precision 0.5, recall 1, f1 2/3. + assert partial["f1"] == pytest.approx(2 / 3) + + def test_mme_missing_counts_zero_in_acc_and_acc_plus(self): + samples = [ + _mme_sample("1", "existence", "img1.jpg", "Yes"), + _mme_sample("2", "existence", "img1.jpg", "No"), + _mme_sample("3", "existence", "img2.jpg", "Yes"), + _mme_sample("4", "existence", "img2.jpg", "No"), + ] + # Submit only img1's pair, both correct. Over the dataset: acc = 2/4 + # -> 50, acc_plus = img1 hit only -> 50; category = 100 (NOT 200). + completions = [_comp("1", "yes"), _comp("2", "no")] + scores, ctx = mme.score(completions, samples) + assert [s.id for s in scores] == ["1", "2"] + assert ctx["mme_perception_score"] == 100.0 + assert ctx["leaderboard_score"] == 100.0 + assert ctx["accuracy"] == pytest.approx(100.0 / 2800, abs=1e-6) + assert ctx["samples_missing"] == 2 + assert ctx["per_sample_accuracy"] == 1.0 # submitted-only diagnostic + + def test_mme_full_coverage_no_missing(self): + samples = [ + _mme_sample("1", "existence", "img1.jpg", "Yes"), + _mme_sample("2", "existence", "img1.jpg", "No"), + ] + _, ctx = mme.score([_comp("1", "yes"), _comp("2", "no")], samples) + assert ctx["mme_perception_score"] == 200.0 + assert ctx["samples_missing"] == 0 + + def test_ocrbench_fixed_denominator_and_missing_count(self): + raw = {"answer": ["Hello"], "category": "Regular Text Recognition"} + samples = [_vlm(str(i), dict(raw)) for i in range(4)] + completions = [_comp("0", "hello"), _comp("1", "nope")] + scores, ctx = ocrbench.score(completions, samples) + assert [s.id for s in scores] == ["0", "1"] + assert ctx["final_score"] == 1.0 + assert ctx["accuracy"] == pytest.approx(1 / 1000) # /1000 fixed + assert ctx["samples_missing"] == 2 + assert ctx["per_sample_accuracy"] == 0.5 # submitted-only diagnostic + + def test_pope_full_coverage_no_missing(self): + samples = [_vlm("1", {"answer": "Yes", "category": "random"})] + _, ctx = pope.score([_comp("1", "yes")], samples) + assert ctx["samples_missing"] == 0 + + +# --------------------------------------------------------------------------- +# Dispatch wiring +# --------------------------------------------------------------------------- + + +class TestDispatch: + @pytest.mark.parametrize( + "eval_id", ["mme", "mmstar", "ocrbench", "mmmu_val", "realworldqa", "mmbench_en_dev", "pope"] + ) + def test_score_dispatch(self, eval_id): + raw = { + "A": "x", + "B": "y", + "answer": "Yes" if eval_id in ("mme", "pope") else "A", + "category": "existence" if eval_id == "mme" else "Regular Text Recognition", + "image_path": "i.jpg", + "index": 1, + } + samples = [_vlm("1", raw)] + completions = [_comp("1", "yes A")] + scores, ctx = scoring.score(eval_id, completions, samples) + assert len(scores) == 1 + assert "accuracy" in ctx + assert "leaderboard_score" in ctx + + @staticmethod + def _sample(eval_id: str) -> tuple[VlmSample, str]: + yes_no = eval_id in ("mme", "pope") + raw = { + "A": "x", + "B": "y", + "answer": "Yes" if yes_no else ("GV" if eval_id == "ocrbench" else "A"), + "category": "existence" if eval_id == "mme" else "Regular Text Recognition", + "image_path": "i.jpg", + "index": 1, + } + return _vlm("1", raw), "Yes" if yes_no else ("GV" if eval_id == "ocrbench" else "A") + + @pytest.mark.parametrize( + "eval_id", ["mme", "mmstar", "ocrbench", "mmmu_val", "realworldqa", "mmbench_en_dev", "pope"] + ) + def test_reasoning_is_stripped_before_extraction(self, eval_id): + sample, answer = self._sample(eval_id) + completion = _comp("1", f"Not sure: No, or B.{answer}") + scores, ctx = scoring.score(eval_id, [completion], [sample], enable_thinking=True) + assert scores[0].is_correct + assert ctx["blanked_truncated"] == 0 + + @pytest.mark.parametrize( + "eval_id", ["mme", "mmstar", "ocrbench", "mmmu_val", "realworldqa", "mmbench_en_dev", "pope"] + ) + def test_reasoning_cut_at_the_cap_is_graded_empty(self, eval_id): + sample, answer = self._sample(eval_id) + completion = SampleCompletion(id="1", completion=f"It is {answer}", stop_reason="truncated") + scores, ctx = scoring.score(eval_id, [completion], [sample], enable_thinking=True) + assert not scores[0].is_correct + assert ctx["blanked_truncated"] == 1 + + +# --------------------------------------------------------------------------- +# Dataset catalog wiring (_vlm_common, synthetic parquet — no LFS data) +# --------------------------------------------------------------------------- + + +class TestVlmDatasetCatalog: + def _write_dataset(self, tmp_path, eval_id="pope"): + import json + + import pyarrow as pa + import pyarrow.parquet as pq + + d = tmp_path / "datasets" / eval_id / "test_ds" + d.mkdir(parents=True) + rows = [ + { + "sample_id": "s1", + "prompt": "Is there a dog in the image?", + "answer": json.dumps("Yes"), + "raw_doc": json.dumps({"answer": "Yes", "category": "adversarial"}), + "num_images": 1, + "image_0": b"\xff\xd8fakejpeg", + } + ] + pq.write_table(pa.Table.from_pylist(rows), d / "test.parquet") + return tmp_path + + def test_load_eval_samples(self, tmp_path): + from pipette_scores.dataset_catalog import load_eval_samples + from pipette_scores.types import EvalId + + root = self._write_dataset(tmp_path) + samples = load_eval_samples(EvalId.POPE, "test_ds", root_dir=root) + assert len(samples) == 1 + assert isinstance(samples[0], VlmSample) + assert samples[0].raw_doc["answer"] == "Yes" + + def test_load_prompt_samples_multimodal_content(self, tmp_path): + from pipette_scores.dataset_catalog import load_prompt_samples + from pipette_scores.types import EvalId + + root = self._write_dataset(tmp_path) + prompts = load_prompt_samples(EvalId.POPE, "test_ds", root_dir=root) + assert len(prompts) == 1 + content = prompts[0].messages[0].content + assert isinstance(content, list) + assert content[0]["type"] == "image_url" + assert content[0]["image_url"]["url"].startswith("data:image/jpeg;base64,") + assert content[-1] == {"type": "text", "text": "Is there a dog in the image?"} + + +# --------------------------------------------------------------------------- +# Replies as the b11744 smoke runs got them from the new VLM families (no profile +# extraction: the reasoning rules and the fallbacks alone) +# --------------------------------------------------------------------------- + +_MCQ_DOC = {"A": "blue", "B": "green", "C": "red", "D": "black", "answer": "C", "l2_category": "x"} + + +class TestSmokeReplies: + def test_mimo_think_then_boxed_letter(self): + reply = ( + "\nGot it, let's look at the image. There's a red circle on the left. The options are A to D. " + "So the circle is red, which is option C.\n\n\n\\boxed{C}" + ) + _, ctx = mmstar.score([_comp("1", reply)], [_vlm("1", _MCQ_DOC)]) + assert ctx["accuracy"] == 1.0 + + def test_mimo_boxed_ocr_answer(self): + reply = '\nThe text is "PIPETTE 42".\n\n\n\\boxed{PIPETTE 42}' + samples = [_vlm("1", {"answer": ["PIPETTE 42"], "category": "Regular Text Recognition"})] + scores, _ = ocrbench.score([_comp("1", reply)], samples) + assert scores[0].is_correct is True + + def test_kimi_closed_thought_then_the_letter(self): + reply = "◁think▷Got it, let's look at the image. The circle is red. So the answer is C.◁/think▷C" + _, ctx = mmstar.score([_comp("1", reply)], [_vlm("1", _MCQ_DOC)]) + assert ctx["accuracy"] == 1.0 + + def test_kimi_grades_the_letter_after_the_thought_not_the_ones_inside_it(self): + reply = "◁think▷Maybe A, maybe B.◁/think▷C" + _, ctx = mmstar.score([_comp("1", reply)], [_vlm("1", _MCQ_DOC)]) + assert ctx["accuracy"] == 1.0 + reply = "◁think▷Maybe A, maybe C.◁/think▷B" + _, ctx = mmstar.score([_comp("1", reply)], [_vlm("1", _MCQ_DOC)]) + assert ctx["accuracy"] == 0.0 + + def test_ministral_orphan_close_then_yes(self): + reply = 'The image has a red circle, so the answer should be "yes".[/THINK]yes' + scores, _ = pope.score([_comp("1", reply)], [_vlm("1", {"answer": "Yes", "category": "adversarial"})]) + assert scores[0].is_correct is True diff --git a/pyproject.toml b/pyproject.toml index 3cde0f6..65d2bd8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,11 @@ target-version = "py312" exclude = [ "packages/pipette-scores/pipette_scores/vendor", "packages/pipette-scores/pipette_scores/scoring/math_500/prm800k", + # Vendored verbatim from liquid_evals; kept byte-identical to the original. + "packages/pipette-scores/pipette_scores/scoring/math_500/liquid_evals_parser.py", # Build-time-generated bundled IFBench source (force-included into the wheel). "packages/pipette-scores/pipette_scores/scoring/ifbench/_upstream", + # Vendored verbatim from the Open ASR Leaderboard (see its NOTICE.md). + "packages/pipette-scores/pipette_scores/scoring/asr/open_asr_normalizer", "vendor", ] diff --git a/uv.lock b/uv.lock index a0641ee..1d79216 100644 --- a/uv.lock +++ b/uv.lock @@ -201,6 +201,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d6/65/1859fddfabc1cc72548c2269d988819aad96d854e25eae00531517925901/blake3-1.0.8-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:511133bab85ff60ed143424ce484d08c60894ff7323f685d7a6095f43f0c85c3", size = 553805, upload-time = "2025-10-14T06:46:36.532Z" }, ] +[[package]] +name = "boto3" +version = "1.43.123" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, + { name = "jmespath", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, + { name = "s3transfer", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fc/8a/64b46f075149f9160da44aa7e1052dd18431b9ad4435beb0cfb02cc48624/boto3-1.43.123.tar.gz", hash = "sha256:055fabb990c5d2f31fce4ca569255bd640cdd11a3ca68c9ecad35f4f8a5b83b5", size = 112638, upload-time = "2026-10-30T19:18:50.085Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/96/b2/a7986c327f80aa6f1544f4cb8f2fa7da4e1fccd472379a21cad0508b8a0e/boto3-1.43.123-py3-none-any.whl", hash = "sha256:d18d66e1e13faa0c25d0b1b8c5ce946e29fb5fc0b6456b655a93a678ff4cc313", size = 140044, upload-time = "2026-10-30T19:18:47.734Z" }, +] + +[[package]] +name = "botocore" +version = "1.43.123" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jmespath", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, + { name = "python-dateutil", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, + { name = "urllib3", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e0/de/011fc2a47fb066c47064b122498d2fd21f55bc90ea514acbddc37ca0f8c1/botocore-1.43.123.tar.gz", hash = "sha256:80948cb9b1cdf1a1a25809089a42e3a1bc4af0258d198222db6e97dcc660dd0c", size = 16486269, upload-time = "2026-10-30T19:18:44.609Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/db/96/4f2dffd304425c07160ab2c12cb13322d852fd956fe9d4386228c93f02ea/botocore-1.43.123-py3-none-any.whl", hash = "sha256:e30580f9c1282c321529238ff8ff75066096a2adbd0a293c8ae592c21d1733a3", size = 16185987, upload-time = "2026-10-30T19:18:39.794Z" }, +] + [[package]] name = "cachetools" version = "7.0.5" @@ -241,10 +269,12 @@ name = "cffi" version = "2.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pycparser", marker = "implementation_name != 'PyPy' and sys_platform == 'linux'" }, + { name = "pycparser", marker = "(implementation_name != 'PyPy' and sys_platform == 'darwin') or (implementation_name != 'PyPy' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/eb/56/b1ba7935a17738ae8453301356628e8147c79dbb825bcbc73dc7401f9846/cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529", size = 523588, upload-time = "2025-09-08T23:24:04.541Z" } wheels = [ + { url = "https://files.pythonhosted.org/packages/ea/47/4f61023ea636104d4f16ab488e268b93008c3d0bb76893b1b31db1f96802/cffi-2.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d02d6655b0e54f54c4ef0b94eb6be0607b70853c45ce98bd278dc7de718be5d", size = 185271, upload-time = "2025-09-08T23:22:44.795Z" }, + { url = "https://files.pythonhosted.org/packages/df/a2/781b623f57358e360d62cdd7a8c681f074a71d445418a776eef0aadb4ab4/cffi-2.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8eca2a813c1cb7ad4fb74d368c2ffbbb4789d377ee5bb8df98373c2cc0dee76c", size = 181048, upload-time = "2025-09-08T23:22:45.938Z" }, { url = "https://files.pythonhosted.org/packages/ff/df/a4f0fbd47331ceeba3d37c2e51e9dfc9722498becbeec2bd8bc856c9538a/cffi-2.0.0-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:21d1152871b019407d8ac3985f6775c079416c282e431a4da6afe7aefd2bccbe", size = 212529, upload-time = "2025-09-08T23:22:47.349Z" }, { url = "https://files.pythonhosted.org/packages/d5/72/12b5f8d3865bf0f87cf1404d8c374e7487dcf097a1c91c436e72e6badd83/cffi-2.0.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b21e08af67b8a103c71a250401c78d5e0893beff75e28c53c98f4de42f774062", size = 220097, upload-time = "2025-09-08T23:22:48.677Z" }, { url = "https://files.pythonhosted.org/packages/c2/95/7a135d52a50dfa7c882ab0ac17e8dc11cec9d55d2c18dda414c051c5e69e/cffi-2.0.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:1e3a615586f05fc4065a8b22b8152f0c1b00cdbc60596d187c2a74f9e3036e4e", size = 207983, upload-time = "2025-09-08T23:22:50.06Z" }, @@ -252,6 +282,8 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/78/2d/7fa73dfa841b5ac06c7b8855cfc18622132e365f5b81d02230333ff26e9e/cffi-2.0.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3e17ed538242334bf70832644a32a7aae3d83b57567f9fd60a26257e992b79ba", size = 219572, upload-time = "2025-09-08T23:22:52.902Z" }, { url = "https://files.pythonhosted.org/packages/07/e0/267e57e387b4ca276b90f0434ff88b2c2241ad72b16d31836adddfd6031b/cffi-2.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3925dd22fa2b7699ed2617149842d2e6adde22b262fcbfada50e3d195e4b3a94", size = 222963, upload-time = "2025-09-08T23:22:54.518Z" }, { url = "https://files.pythonhosted.org/packages/b6/75/1f2747525e06f53efbd878f4d03bac5b859cbc11c633d0fb81432d98a795/cffi-2.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2c8f814d84194c9ea681642fd164267891702542f028a15fc97d4674b6206187", size = 221361, upload-time = "2025-09-08T23:22:55.867Z" }, + { url = "https://files.pythonhosted.org/packages/4b/8d/a0a47a0c9e413a658623d014e91e74a50cdd2c423f7ccfd44086ef767f90/cffi-2.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:00bdf7acc5f795150faa6957054fbbca2439db2f775ce831222b66f192f03beb", size = 185230, upload-time = "2025-09-08T23:23:00.879Z" }, + { url = "https://files.pythonhosted.org/packages/4a/d2/a6c0296814556c68ee32009d9c2ad4f85f2707cdecfd7727951ec228005d/cffi-2.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:45d5e886156860dc35862657e1494b9bae8dfa63bf56796f2fb56e1679fc0bca", size = 181043, upload-time = "2025-09-08T23:23:02.231Z" }, { url = "https://files.pythonhosted.org/packages/b0/1e/d22cc63332bd59b06481ceaac49d6c507598642e2230f201649058a7e704/cffi-2.0.0-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:07b271772c100085dd28b74fa0cd81c8fb1a3ba18b21e03d7c27f3436a10606b", size = 212446, upload-time = "2025-09-08T23:23:03.472Z" }, { url = "https://files.pythonhosted.org/packages/a9/f5/a2c23eb03b61a0b8747f211eb716446c826ad66818ddc7810cc2cc19b3f2/cffi-2.0.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d48a880098c96020b02d5a1f7d9251308510ce8858940e6fa99ece33f610838b", size = 220101, upload-time = "2025-09-08T23:23:04.792Z" }, { url = "https://files.pythonhosted.org/packages/f2/7f/e6647792fc5850d634695bc0e6ab4111ae88e89981d35ac269956605feba/cffi-2.0.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:f93fd8e5c8c0a4aa1f424d6173f14a892044054871c771f8566e4008eaa359d2", size = 207948, upload-time = "2025-09-08T23:23:06.127Z" }, @@ -472,6 +504,12 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ba/5a/18ad964b0086c6e62e2e7500f7edc89e3faa45033c71c1893d34eed2b2de/dnspython-2.8.0-py3-none-any.whl", hash = "sha256:01d9bbc4a2d76bf0db7c1f729812ded6d912bd318d3b1cf81d30c0f845dbf3af", size = 331094, upload-time = "2025-09-07T18:57:58.071Z" }, ] +[[package]] +name = "docopt" +version = "0.6.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/55/8f8cab2afd404cf578136ef2cc5dfb50baa1761b68c9da1fb1e4eed343c9/docopt-0.6.2.tar.gz", hash = "sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491", size = 25901, upload-time = "2014-06-16T11:18:57.406Z" } + [[package]] name = "docstring-parser" version = "0.18.0" @@ -1059,6 +1097,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" }, ] +[[package]] +name = "kaldialign" +version = "0.12.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b0/58/bd83457ab1f4296cb48da2057330fd669e14d0139353d87f9c138fc055af/kaldialign-0.12.0.tar.gz", hash = "sha256:f25743ef7dcf15716c5ac47e164dd7191cab791196a78c27cff6a99c253e1a14", size = 31415, upload-time = "2026-06-15T13:16:40.7Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8a/cd/f7c852333a77887ce9166415512d19da329c4896ec0a16127b250c20fd5e/kaldialign-0.12.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:3a4f914f4971d20784710944a965de5ce0d1aa1ce26934767c0aee95194e0d98", size = 166411, upload-time = "2026-06-15T13:16:45.578Z" }, + { url = "https://files.pythonhosted.org/packages/35/c9/a01c036b6e0afef2139bcc3088d3ce9fbad111864457c89ac059b8fbaee4/kaldialign-0.12.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:62694144a5e0ae086910bd979bc6b99b36c97a4276614bd28b5b06fed133abbf", size = 99987, upload-time = "2026-06-15T13:18:05.331Z" }, + { url = "https://files.pythonhosted.org/packages/d0/00/77885350408ef2ec6281cafd3dfbec099c25893d502bcc7ac819637309c5/kaldialign-0.12.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8189d32046b3a865b6f147662620feba736580b5424368164c6a7bc927d3e0a8", size = 110985, upload-time = "2026-06-15T13:16:46.189Z" }, + { url = "https://files.pythonhosted.org/packages/7f/b8/190bc4b3a5a208cca114ac46df2954fb3f6eee02700603b44235c7a0219b/kaldialign-0.12.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:30c7f0aefc840ac0dea27bedbf71dc103df7e66682a3dd4f26c9c390feb27143", size = 166526, upload-time = "2026-06-15T13:17:30.188Z" }, + { url = "https://files.pythonhosted.org/packages/43/21/3254f35f410ba850b90bd2bdd0df2d8de187dc570e1607088fb3f9f4421c/kaldialign-0.12.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5f3692aa70da005624c42230a2af8bad01f1adba7468436186ceaa12604c78e8", size = 100016, upload-time = "2026-06-15T13:16:56.169Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8f/e2e9d71c26a3316785d6c14cfffd25fccb021442c8aebd4507c57eece25c/kaldialign-0.12.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d488613310771e7f9f749320f26afcdfadf476e08b8821f2b309a046e7ff2c62", size = 110995, upload-time = "2026-06-15T13:16:30.972Z" }, +] + [[package]] name = "langdetect" version = "1.0.9" @@ -1377,6 +1429,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9d/91/04e965f8e717ba0ab4bdca5c112deeab11c9e750d94c4d4602f050295d39/nltk-3.9.4-py3-none-any.whl", hash = "sha256:f2fa301c3a12718ce4a0e9305c5675299da5ad9e26068218b69d692fda84828f", size = 1552087, upload-time = "2026-03-24T06:13:38.47Z" }, ] +[[package]] +name = "num2words" +version = "0.5.14" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "docopt", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f6/58/ad645bd38b4b648eb2fc2ba1b909398e54eb0cbb6a7dbd2b4953e38c9621/num2words-0.5.14.tar.gz", hash = "sha256:b066ec18e56b6616a3b38086b5747daafbaa8868b226a36127e0451c0cf379c6", size = 218213, upload-time = "2024-12-17T20:17:10.191Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d6/5b/545e9267a1cc080c8a1be2746113a063e34bcdd0f5173fd665a5c13cb234/num2words-0.5.14-py3-none-any.whl", hash = "sha256:1c8e5b00142fc2966fd8d685001e36c4a9911e070d1b120e1beb721fa1edb33d", size = 163525, upload-time = "2024-12-17T20:17:06.074Z" }, +] + [[package]] name = "numba" version = "0.61.2" @@ -1866,6 +1930,8 @@ version = "12.2.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/8c/21/c2bcdd5906101a30244eaffc1b6e6ce71a31bd0742a01eb89e660ebfac2d/pillow-12.2.0.tar.gz", hash = "sha256:a830b1a40919539d07806aa58e1b114df53ddd43213d9c8b75847eee6c0182b5", size = 46987819, upload-time = "2026-04-01T14:46:17.687Z" } wheels = [ + { url = "https://files.pythonhosted.org/packages/58/be/7482c8a5ebebbc6470b3eb791812fff7d5e0216c2be3827b30b8bb6603ed/pillow-12.2.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2d192a155bbcec180f8564f693e6fd9bccff5a7af9b32e2e4bf8c9c69dbad6b5", size = 5308279, upload-time = "2026-04-01T14:43:13.246Z" }, + { url = "https://files.pythonhosted.org/packages/d8/95/0a351b9289c2b5cbde0bacd4a83ebc44023e835490a727b2a3bd60ddc0f4/pillow-12.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f3f40b3c5a968281fd507d519e444c35f0ff171237f4fdde090dd60699458421", size = 4695490, upload-time = "2026-04-01T14:43:15.584Z" }, { url = "https://files.pythonhosted.org/packages/de/af/4e8e6869cbed569d43c416fad3dc4ecb944cb5d9492defaed89ddd6fe871/pillow-12.2.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:03e7e372d5240cc23e9f07deca4d775c0817bffc641b01e9c3af208dbd300987", size = 6284462, upload-time = "2026-04-01T14:43:18.268Z" }, { url = "https://files.pythonhosted.org/packages/e9/9e/c05e19657fd57841e476be1ab46c4d501bffbadbafdc31a6d665f8b737b6/pillow-12.2.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b86024e52a1b269467a802258c25521e6d742349d760728092e1bc2d135b4d76", size = 8094744, upload-time = "2026-04-01T14:43:20.716Z" }, { url = "https://files.pythonhosted.org/packages/2b/54/1789c455ed10176066b6e7e6da1b01e50e36f94ba584dc68d9eebfe9156d/pillow-12.2.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7371b48c4fa448d20d2714c9a1f775a81155050d383333e0a6c15b1123dda005", size = 6398371, upload-time = "2026-04-01T14:43:23.443Z" }, @@ -1875,12 +1941,16 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4a/01/53d10cf0dbad820a8db274d259a37ba50b88b24768ddccec07355382d5ad/pillow-12.2.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:8297651f5b5679c19968abefd6bb84d95fe30ef712eb1b2d9b2d31ca61267f4c", size = 4100837, upload-time = "2026-04-01T14:43:41.506Z" }, { url = "https://files.pythonhosted.org/packages/0f/98/f3a6657ecb698c937f6c76ee564882945f29b79bad496abcba0e84659ec5/pillow-12.2.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:50d8520da2a6ce0af445fa6d648c4273c3eeefbc32d7ce049f22e8b5c3daecc2", size = 4176528, upload-time = "2026-04-01T14:43:43.773Z" }, { url = "https://files.pythonhosted.org/packages/69/bc/8986948f05e3ea490b8442ea1c1d4d990b24a7e43d8a51b2c7d8b1dced36/pillow-12.2.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:766cef22385fa1091258ad7e6216792b156dc16d8d3fa607e7545b2b72061f1c", size = 3640401, upload-time = "2026-04-01T14:43:45.87Z" }, + { url = "https://files.pythonhosted.org/packages/34/46/6c717baadcd62bc8ed51d238d521ab651eaa74838291bda1f86fe1f864c9/pillow-12.2.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5d2fd0fa6b5d9d1de415060363433f28da8b1526c1c129020435e186794b3795", size = 5308094, upload-time = "2026-04-01T14:43:48.438Z" }, + { url = "https://files.pythonhosted.org/packages/71/43/905a14a8b17fdb1ccb58d282454490662d2cb89a6bfec26af6d3520da5ec/pillow-12.2.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:56b25336f502b6ed02e889f4ece894a72612fe885889a6e8c4c80239ff6e5f5f", size = 4695402, upload-time = "2026-04-01T14:43:51.292Z" }, { url = "https://files.pythonhosted.org/packages/73/dd/42107efcb777b16fa0393317eac58f5b5cf30e8392e266e76e51cff28c3d/pillow-12.2.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f1c943e96e85df3d3478f7b691f229887e143f81fedab9b20205349ab04d73ed", size = 6280005, upload-time = "2026-04-01T14:43:54.242Z" }, { url = "https://files.pythonhosted.org/packages/a8/68/b93e09e5e8549019e61acf49f65b1a8530765a7f812c77a7461bca7e4494/pillow-12.2.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:03f6fab9219220f041c74aeaa2939ff0062bd5c364ba9ce037197f4c6d498cd9", size = 8090669, upload-time = "2026-04-01T14:43:57.335Z" }, { url = "https://files.pythonhosted.org/packages/4b/6e/3ccb54ce8ec4ddd1accd2d89004308b7b0b21c4ac3d20fa70af4760a4330/pillow-12.2.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5cdfebd752ec52bf5bb4e35d9c64b40826bc5b40a13df7c3cda20a2c03a0f5ed", size = 6395194, upload-time = "2026-04-01T14:43:59.864Z" }, { url = "https://files.pythonhosted.org/packages/67/ee/21d4e8536afd1a328f01b359b4d3997b291ffd35a237c877b331c1c3b71c/pillow-12.2.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:eedf4b74eda2b5a4b2b2fb4c006d6295df3bf29e459e198c90ea48e130dc75c3", size = 7082423, upload-time = "2026-04-01T14:44:02.74Z" }, { url = "https://files.pythonhosted.org/packages/78/5f/e9f86ab0146464e8c133fe85df987ed9e77e08b29d8d35f9f9f4d6f917ba/pillow-12.2.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:00a2865911330191c0b818c59103b58a5e697cae67042366970a6b6f1b20b7f9", size = 6505667, upload-time = "2026-04-01T14:44:05.381Z" }, { url = "https://files.pythonhosted.org/packages/ed/1e/409007f56a2fdce61584fd3acbc2bbc259857d555196cedcadc68c015c82/pillow-12.2.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1e1757442ed87f4912397c6d35a0db6a7b52592156014706f17658ff58bbf795", size = 7208580, upload-time = "2026-04-01T14:44:08.39Z" }, + { url = "https://files.pythonhosted.org/packages/4d/a4/b342930964e3cb4dce5038ae34b0eab4653334995336cd486c5a8c25a00c/pillow-12.2.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:042db20a421b9bafecc4b84a8b6e444686bd9d836c7fd24542db3e7df7baad9b", size = 5309927, upload-time = "2026-04-01T14:44:18.89Z" }, + { url = "https://files.pythonhosted.org/packages/9f/de/23198e0a65a9cf06123f5435a5d95cea62a635697f8f03d134d3f3a96151/pillow-12.2.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:dd025009355c926a84a612fecf58bb315a3f6814b17ead51a8e48d3823d9087f", size = 4698624, upload-time = "2026-04-01T14:44:21.115Z" }, { url = "https://files.pythonhosted.org/packages/01/a6/1265e977f17d93ea37aa28aa81bad4fa597933879fac2520d24e021c8da3/pillow-12.2.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:88ddbc66737e277852913bd1e07c150cc7bb124539f94c4e2df5344494e0a612", size = 6321252, upload-time = "2026-04-01T14:44:23.663Z" }, { url = "https://files.pythonhosted.org/packages/3c/83/5982eb4a285967baa70340320be9f88e57665a387e3a53a7f0db8231a0cd/pillow-12.2.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d362d1878f00c142b7e1a16e6e5e780f02be8195123f164edf7eddd911eefe7c", size = 8126550, upload-time = "2026-04-01T14:44:26.772Z" }, { url = "https://files.pythonhosted.org/packages/4e/48/6ffc514adce69f6050d0753b1a18fd920fce8cac87620d5a31231b04bfc5/pillow-12.2.0-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2c727a6d53cb0018aadd8018c2b938376af27914a68a492f59dfcaca650d5eea", size = 6433114, upload-time = "2026-04-01T14:44:29.615Z" }, @@ -1943,14 +2013,17 @@ name = "pipette-scores" version = "0.1.0" source = { editable = "packages/pipette-scores" } dependencies = [ + { name = "boto3", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "datasets", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "emoji", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "fastapi", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "gunicorn", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "ifstruct", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "immutabledict", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, + { name = "kaldialign", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "langdetect", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "nltk", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, + { name = "num2words", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform == 'linux'" }, { name = "numpy", version = "2.4.4", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform == 'darwin'" }, { name = "pandas", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, @@ -1958,7 +2031,10 @@ dependencies = [ { name = "pydantic", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "pylatexenc", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "pyyaml", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, + { name = "regex", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "setuptools", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, + { name = "soundfile", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, + { name = "soxr", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "syllapy", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "sympy", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "tabulate", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, @@ -1966,15 +2042,24 @@ dependencies = [ ] [package.optional-dependencies] +build = [ + { name = "pillow", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, + { name = "pycountry", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, + { name = "requests", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, +] dev = [ { name = "httpx", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, + { name = "pillow", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, + { name = "pycountry", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "pytest", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "pytest-asyncio", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, + { name = "requests", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "ruff", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, ] [package.metadata] requires-dist = [ + { name = "boto3" }, { name = "datasets", specifier = ">=2.14.0" }, { name = "emoji", specifier = ">=2.0.0" }, { name = "fastapi", specifier = "==0.128.1" }, @@ -1982,24 +2067,33 @@ requires-dist = [ { name = "httpx", marker = "extra == 'dev'", specifier = ">=0.24.0" }, { name = "ifstruct", directory = "vendor/ifstruct" }, { name = "immutabledict", specifier = ">=4.0.0" }, + { name = "kaldialign", specifier = "==0.12.0" }, { name = "langdetect", specifier = ">=1.0.9" }, - { name = "nltk", specifier = ">=3.8.0" }, + { name = "nltk", specifier = "==3.9.4" }, + { name = "num2words", specifier = "==0.5.14" }, { name = "numpy", specifier = ">=1.26.0" }, { name = "pandas", specifier = ">=2.0.0" }, + { name = "pillow", marker = "extra == 'build'", specifier = ">=10.0" }, + { name = "pipette-scores", extras = ["build"], marker = "extra == 'dev'" }, { name = "pyarrow", specifier = ">=14.0.0" }, + { name = "pycountry", marker = "extra == 'build'", specifier = "==26.2.16" }, { name = "pydantic", specifier = ">=2.0.0" }, - { name = "pylatexenc", specifier = ">=2.10" }, + { name = "pylatexenc", specifier = "==2.10" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0" }, { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.24.0" }, { name = "pyyaml", specifier = ">=6.0" }, + { name = "regex", specifier = "==2026.4.4" }, + { name = "requests", marker = "extra == 'build'", specifier = ">=2.31.0" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.8.0" }, { name = "setuptools", specifier = ">=69.0.0,<81" }, + { name = "soundfile", specifier = "==0.14.0" }, + { name = "soxr", specifier = "==1.1.0" }, { name = "syllapy", specifier = ">=0.7.2" }, - { name = "sympy", specifier = ">=1.12" }, + { name = "sympy", specifier = "==1.14.0" }, { name = "tabulate", specifier = ">=0.9.0" }, { name = "uvicorn", extras = ["standard"], specifier = "==0.40.0" }, ] -provides-extras = ["dev"] +provides-extras = ["build", "dev"] [[package]] name = "pluggy" @@ -2649,6 +2743,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/11/08/874da392558ce087a0f9b709dc6ec0d60cbc694c1c772dab8d5f31efe8cb/ruff-0.15.10-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:b0c52744cf9f143a393e284125d2576140b68264a93c6716464e129a3e9adb48", size = 11358051, upload-time = "2026-04-09T14:06:18.948Z" }, ] +[[package]] +name = "s3transfer" +version = "0.19.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/49/c5/0a5ce655109cb96b8dfad36908a4e108d15f5dc2eff8fbc3146c4312e704/s3transfer-0.19.3.tar.gz", hash = "sha256:8326acc3d056d7f7b8a37110c7d2aea41f67e83f6ae79cd3b5fbcca34c726406", size = 166087, upload-time = "2026-10-19T19:19:49.751Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e2/a4/88f9e10498759121060b8050a1554753dfc04df39bbbcc7aad9ecc8d9440/s3transfer-0.19.3-py3-none-any.whl", hash = "sha256:24eb7535804306355e31c952a90022bf5b2f9d760f9a11062c7493a217c08553", size = 90252, upload-time = "2026-10-19T19:19:48.309Z" }, +] + [[package]] name = "safetensors" version = "0.7.0" @@ -2756,6 +2862,41 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" }, ] +[[package]] +name = "soundfile" +version = "0.14.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform == 'linux'" }, + { name = "numpy", version = "2.4.4", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform == 'darwin'" }, + { name = "typing-extensions", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d2/db/949331952a6fb1c5b12e9de80fd08747966c2039d1a61db4764fbd3981c2/soundfile-0.14.0.tar.gz", hash = "sha256:ba1c1a2d618bca5c406647c83b89f07cc8810fa506a50622a6993ba130c1de11", size = 47842, upload-time = "2026-06-06T08:58:47.869Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b1/d1/5e338af9ca6ed0786cd5bb03f6d60de1c325728c1189014f3b59aae7403c/soundfile-0.14.0-py2.py3-none-any.whl", hash = "sha256:8ba81ae3a89fd5ab3bef8a8eb481fbbe794e806309675a89b4df48b8d31908a8", size = 26799, upload-time = "2026-06-06T08:58:33.269Z" }, + { url = "https://files.pythonhosted.org/packages/7e/72/c6b21e58d3113596e7e8de0a08d6f1d95173492cfbca0a4db14148cbba2a/soundfile-0.14.0-py2.py3-none-macosx_10_9_x86_64.whl", hash = "sha256:19be05428da76ed61a4cad29b8e4bcf43a3e5c100089d2ec81dc961eed1b0dd4", size = 1144568, upload-time = "2026-06-06T08:58:35.231Z" }, + { url = "https://files.pythonhosted.org/packages/63/7a/dfdd6f8c748988427119f75eb860a3cedd858d1aea1fe28f39ad8559ef22/soundfile-0.14.0-py2.py3-none-macosx_11_0_arm64.whl", hash = "sha256:d828d35a059626da52f1415b5faee610aeab393319cb3fc4a9aef47b619fc14c", size = 1103726, upload-time = "2026-06-06T08:58:37.948Z" }, + { url = "https://files.pythonhosted.org/packages/4a/f8/fc39fad6f879633461d27394cd1ddaf1f769ffa0597dca35872f51b16461/soundfile-0.14.0-py2.py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:e85724a90bc99a6e8062c0b4ddf725f53b2a3b70afd4da875e9d2cfc4e92f377", size = 1238050, upload-time = "2026-06-06T08:58:39.932Z" }, + { url = "https://files.pythonhosted.org/packages/7b/a2/70fd4432b924684c372df8b0a45708c36c057ef3596c9eb53e0a806b980b/soundfile-0.14.0-py2.py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:1e38bac1853412871318e82a1ba69a8be677619b56025bbfcccdb41b6cafe82d", size = 1315963, upload-time = "2026-06-06T08:58:41.716Z" }, +] + +[[package]] +name = "soxr" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform == 'linux'" }, + { name = "numpy", version = "2.4.4", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform == 'darwin'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ed/11/27cebce4a108f77afea7c80545115536b45e3f11ebfb914f638fdd9ba847/soxr-1.1.0.tar.gz", hash = "sha256:9f228ae21c78fa9359ca98d8a5e8e91f30639e438e574133dace62c5b5309e44", size = 173067, upload-time = "2026-05-03T00:15:18.214Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/06/8a/f3da7973b5f1b05d2d7e94d5376b881dcbc05297900cae6c3d33d95b209b/soxr-1.1.0-cp312-abi3-macosx_10_14_x86_64.whl", hash = "sha256:e0e09fa633ce2e67df08b298afced4d184f6e753fc330f241022250f1d0d61da", size = 204124, upload-time = "2026-05-03T00:14:54.505Z" }, + { url = "https://files.pythonhosted.org/packages/03/dc/200013a74641f8774664bbcd2346c695c05c2e300ea792adcb40a293eed0/soxr-1.1.0-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:d6a7ad82b8d5f3fcc04b1d2ca055562b96af571e1d4fa7c6c61d0fb509ac43b4", size = 165457, upload-time = "2026-05-03T00:14:56.007Z" }, + { url = "https://files.pythonhosted.org/packages/88/2b/2e5eba817a762a2ec589ff165b8bc5955b25a0ad140045f7cd8e45410543/soxr-1.1.0-cp312-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bf98c0d7b7d5ef5bf072fee8d3020e8b664f2d195933ea7bc5089267c2e22a06", size = 206529, upload-time = "2026-05-03T00:14:57.646Z" }, + { url = "https://files.pythonhosted.org/packages/5c/f1/0e55195893228609c9a08c3b13b7a83a46c3a992cd00d3304f0f320cfb07/soxr-1.1.0-cp312-abi3-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3b033078e86f3c4a658e5697fac8995764fad9e799563616b630136b613167f1", size = 240413, upload-time = "2026-05-03T00:14:59.363Z" }, +] + [[package]] name = "sse-starlette" version = "3.3.4" diff --git a/vendor/ifbench-checkers/LICENSE b/vendor/ifbench-checkers/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/vendor/ifbench-checkers/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/ifbench-checkers/README.md b/vendor/ifbench-checkers/README.md new file mode 100644 index 0000000..174200e --- /dev/null +++ b/vendor/ifbench-checkers/README.md @@ -0,0 +1,73 @@ +# IFBench checkers + +The IFBench instruction checkers that `pipette-scores` scores with, from +[allenai/IFBench](https://github.com/allenai/IFBench) @ +`1a902f9e824259f044b6554f6ccb95e2bb322e7b` (2025-11-13). + +That is the revision Liquid4All/liquid_evals scores its `ifbench_aa` task with, +the key that the internal eval scheduler runs through liquid_evals. A completion gets +the same verdict here as it does there, with the exceptions listed below. +liquid_evals runs its own copy of this revision with two small guards: a +guarded spacy import, and `False` for a response with no trigrams. Changes 2 +and 4 below cover the same cases. The checkers were reworked after this +revision, which is why they are vendored at a fixed commit rather than tracked +through the `vendor/ifbench` submodule. That submodule is still the source of +the IFBench **test set**; only the checker code comes from here. + +Git blob ids of the upstream files (the first commit that added this directory +holds them unmodified): + +| File | Blob | +|---|---| +| `evaluation_lib.py` | `a0db9e728d067672eb116dd3e769774ae728c916` | +| `instructions.py` | `f32ff486907490e88b77b014fa3ce5fe35c357c3` | +| `instructions_registry.py` | `4bdbdf52e776a6592dc60285954d6eb771dbd952` | +| `instructions_util.py` | `218247ae86d070ac0e9248d6a0a35a92453a6e4e` | +| `LICENSE` | `261eeb9e9f8b2b4b0d119366dda99c6fd7d35c64` | + +## Local changes + +The edited files carry a header notice, and each added block is commented +`pipette-scores:`. The first two changes alter no verdicts. The last two only +affect inputs on which liquid_evals gives no stable verdict. + +1. **No network at scoring time** (`instructions.py`, `instructions_util.py`). + Upstream calls `nltk.download()` from eleven `build_description()` methods + and from every `count_stopwords()` call; those calls are removed. Instead, + the resources the checkers use (`punkt_tab`, `stopwords`, + `averaged_perceptron_tagger_eng`) are fetched once at import, only if + missing. So is `punkt`, which upstream fetched at import although no + checker uses it. If a resource is still missing after that, the import + fails rather than scoring without it. The Docker image pre-downloads all + four. +2. **No spacy** (`instructions.py`). Upstream runs + `spacy.cli.download('en_core_web_sm')` on every import. No checker uses + spacy, so the import and the download are dropped. +3. **Deterministic `repeat:repeat_span`** (`instructions.py`). Upstream treats + `n_start=0` as unset (`if not n_start`) and picks a random start on every + call. The one dataset row with `n_start=0` (key 286) therefore gets a + random verdict, and a correct answer passes there about 1 time in 42. Here + `0` is used as given, and `n_end` gets the same `is None` check. On that + row this scorer can report up to 5 of 1500 attempts (0.33 pp) more than + liquid_evals. +4. **A checker that raises counts as not followed** (`evaluation_lib.py`). + Some checkers raise on degenerate responses, for example `IndexError` on + `Wow!! Wow is great.` for `words:last_first`, or `ZeroDivisionError` on + `...` for `ratio:stop_words`. Upstream lets that abort the run with no + score. Here the check counts as not followed for that response. In loose + mode the other response variants are still tried, so the instruction can + still pass. The trigram-overlap checker raises `ZeroDivisionError` when a + response, or a loose variant of it, is shorter than three characters. + Catching it gives the `False` that liquid_evals' guard returns, and the + remaining variants are tried on both sides. Missing NLTK data + (`LookupError`) is not caught; it still raises. + +Verdicts also depend on the nltk version: 3.9.4 splits dashes (U+2012–U+2015) +into their own tokens, which changes `word_tokenize` for four checkers. +`pipette-scores` pins nltk 3.9.4, the version that the internal liquid_evals +environments lock. + +## License + +Apache-2.0, (c) Allen Institute for AI and The Google Research Authors; full +text in `LICENSE`. See the repository's `NOTICE` and `THIRD_PARTY_LICENSES.md`. diff --git a/vendor/ifbench-checkers/evaluation_lib.py b/vendor/ifbench-checkers/evaluation_lib.py new file mode 100644 index 0000000..4c3ebdc --- /dev/null +++ b/vendor/ifbench-checkers/evaluation_lib.py @@ -0,0 +1,240 @@ +# coding=utf-8 +# Copyright 2025 The Google Research Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Modified by Liquid AI for pipette-scores; the changes are marked +# "pipette-scores:" and listed in this directory's README.md. + +"""Binary of evaluating instruction following. See README.md.""" + +import collections +import dataclasses +import json +from typing import Dict, Optional, Union + +import instructions_registry + + +def _check_following(instruction, response): + """pipette-scores: count a checker that raises as not followed. + + Some checkers raise on degenerate responses (e.g. IndexError or + ZeroDivisionError on punctuation- or emoji-only text). Upstream lets that + abort the whole run, so it produces no verdict; a scoring service has to + return one. Missing NLTK data is an environment fault, not a verdict, so it + still raises: nltk reports it as a bare LookupError, which is matched by exact + type because IndexError and KeyError subclass LookupError. + """ + try: + return instruction.check_following(response) + except Exception as e: # pylint: disable=broad-except + if type(e) is LookupError: # pylint: disable=unidiomatic-typecheck + raise + return False + + +@dataclasses.dataclass +class InputExample: + key: int + instruction_id_list: list[str] + prompt: str + kwargs: list[Dict[str, Optional[Union[str, int]]]] + + +@dataclasses.dataclass +class OutputExample: + instruction_id_list: list[str] + prompt: str + response: str + follow_all_instructions: bool + follow_instruction_list: list[bool] + + +def read_prompt_list(input_jsonl_filename): + """Read inputs from jsonl.""" + inputs = [] + with open(input_jsonl_filename, "r") as f: + for l in f: + example = json.loads(l) + inputs.append( + InputExample(key=example["key"], + instruction_id_list=example["instruction_id_list"], + prompt=example["prompt"], + kwargs=example["kwargs"])) + return inputs + + +def write_outputs(output_jsonl_filename, outputs): + """Writes outputs to jsonl.""" + assert outputs + with open(output_jsonl_filename, "w") as f: + for o in outputs: + f.write( + json.dumps( + { + attr_name: o.__getattribute__(attr_name) + for attr_name in [ + name for name in dir(o) if not name.startswith("_") + ] + } + ) + ) + f.write("\n") + + +def test_instruction_following_strict( + inp, + prompt_to_response, +): + """Tests response to see if instrutions are followed.""" + response = prompt_to_response[inp.prompt] + instruction_list = inp.instruction_id_list + is_following_list = [] + + for index, instruction_id in enumerate(instruction_list): + instruction_cls = instructions_registry.INSTRUCTION_DICT[instruction_id] + instruction = instruction_cls(instruction_id) + inp.kwargs[index] = {key: value for key, value in inp.kwargs[index].items() if value is not None} + instruction.build_description(**inp.kwargs[index]) + args = instruction.get_instruction_args() + if args and "prompt" in args: + instruction.build_description(prompt=inp.prompt) + + if response.strip() and _check_following(instruction, response): + is_following_list.append(True) + else: + is_following_list.append(False) + + return OutputExample( + instruction_id_list=inp.instruction_id_list, + prompt=inp.prompt, + response=response, + follow_all_instructions=all(is_following_list), + follow_instruction_list=is_following_list, + ) + + +def test_instruction_following_loose( + inp, + prompt_to_response, +): + """Tests response for an upper bound for following instructions.""" + response = prompt_to_response[inp.prompt] + r = response.split("\n") + response_remove_first = "\n".join(r[1:]).strip() + response_remove_last = "\n".join(r[:-1]).strip() + response_remove_both = "\n".join(r[1:-1]).strip() + revised_response = response.replace("*", "") + revised_response_remove_first = response_remove_first.replace("*", "") + revised_response_remove_last = response_remove_last.replace("*", "") + revised_response_remove_both = response_remove_both.replace("*", "") + all_responses = [ + response, + revised_response, + response_remove_first, + response_remove_last, + response_remove_both, + revised_response_remove_first, + revised_response_remove_last, + revised_response_remove_both, + ] + instruction_list = inp.instruction_id_list + is_following_list = [] + + for index, instruction_id in enumerate(instruction_list): + instruction_cls = instructions_registry.INSTRUCTION_DICT[instruction_id] + instruction = instruction_cls(instruction_id) + + instruction.build_description(**inp.kwargs[index]) + args = instruction.get_instruction_args() + if args and "prompt" in args: + instruction.build_description(prompt=inp.prompt) + + is_following = False + for r in all_responses: + if r.strip() and _check_following(instruction, r): + is_following = True + break + + is_following_list.append(is_following) + + return OutputExample( + instruction_id_list=inp.instruction_id_list, + prompt=inp.prompt, + response=response, + follow_all_instructions=all(is_following_list), + follow_instruction_list=is_following_list, + ) + + +def read_prompt_to_response_dict(input_jsonl_filename): + """Creates dictionary matching prompt and response.""" + return_dict = {} + with open(input_jsonl_filename, "r") as f: + for l in f: + example = json.loads(l) + return_dict[example["prompt"]] = example["response"] + return return_dict + + +def print_report(outputs): + """Prints a report on accuracy scores.""" + + prompt_total = 0 + prompt_correct = 0 + instruction_total = 0 + instruction_correct = 0 + + tier0_total = collections.defaultdict(int) + tier0_correct = collections.defaultdict(int) + + tier1_total = collections.defaultdict(int) + tier1_correct = collections.defaultdict(int) + + for example in outputs: + follow_instruction_list = example.follow_instruction_list + instruction_id_list = example.instruction_id_list + + prompt_total += 1 + if all(follow_instruction_list): + prompt_correct += 1 + + instruction_total += len(instruction_id_list) + instruction_correct += sum(follow_instruction_list) + + for instruction_id, followed_or_not in zip( + instruction_id_list, follow_instruction_list + ): + instruction_id = instruction_id.split(":")[0] + tier0_total[instruction_id] += 1 + if followed_or_not: + tier0_correct[instruction_id] += 1 + + for instruction_id, followed_or_not in zip( + instruction_id_list, follow_instruction_list + ): + tier1_total[instruction_id] += 1 + if followed_or_not: + tier1_correct[instruction_id] += 1 + + print(f"prompt-level: {prompt_correct / prompt_total}") + print(f"instruction-level: {instruction_correct / instruction_total}") + print() + for instruction_id in sorted(tier0_total.keys()): + accuracy = tier0_correct[instruction_id] / tier0_total[instruction_id] + print(f"{instruction_id} {accuracy}") + print() + for instruction_id in sorted(tier1_total.keys()): + accuracy = tier1_correct[instruction_id] / tier1_total[instruction_id] + print(f"{instruction_id} {accuracy}") diff --git a/vendor/ifbench-checkers/instructions.py b/vendor/ifbench-checkers/instructions.py new file mode 100644 index 0000000..4a1874e --- /dev/null +++ b/vendor/ifbench-checkers/instructions.py @@ -0,0 +1,2250 @@ +# Copyright 2025 Allen Institute for AI. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Modified by Liquid AI for pipette-scores; the changes are marked +# "pipette-scores:" and listed in this directory's README.md. + +"""Library of instructions.""" + +import logging +import random +import re +import string +from typing import Dict, Optional, Sequence, Union +import nltk +import emoji +import syllapy +import unicodedata +from collections import Counter +import csv +import io + +import instructions_util + +# pipette-scores: dropped `download('en_core_web_sm')` and its spacy import. No +# checker uses spacy, and the download goes to the network on every import. + +logger = logging.getLogger(__name__) + +_InstructionArgsDtype = Optional[Dict[str, Union[int, str, Sequence[str]]]] + +# The number of keywords. +_NUM_KEYWORDS = 2 + +# The number of words in the response. +_NUM_WORDS_LOWER_LIMIT = 100 +_NUM_WORDS_UPPER_LIMIT = 500 + +# The number of numbers. +_NUM_NUMBERS = 6 + +# Period length for periodic words. +_NUM_WORD_CYCLE = 30 + +# Maximum number of times a word can be repeated. +_MAX_REPEATS = 5 + +# Which sentence must contain a keyword. +_NUM_KEYWORD_SENTENCE = 20 + +# Minimum number of pronouns. +_NUM_PRONOUNS = 25 + +# The size of increment for lengths. +_NUM_INCREMENT = 5 + +# The number of coordinating conjunctions. +_NUM_CONJUNCTIONS = 6 + + +class Instruction: + """An instruction template.""" + + def __init__(self, instruction_id): + self.id = instruction_id + + def build_description(self, **kwargs): + raise NotImplementedError("`build_description` not implemented.") + + def get_instruction_args(self): + raise NotImplementedError("`get_instruction_args` not implemented.") + + def get_instruction_args_keys(self): + raise NotImplementedError("`get_instruction_args_keys` not implemented.") + + def check_following(self, value): + raise NotImplementedError("`check_following` not implemented.") + + +# Everything as follows is part of OOD IFEval + +class WordCountRangeChecker(Instruction): + """Word Count Range: The response must contain between X and Y words.""" + + def build_description(self, *, min_words=None, max_words=None): + """Build the instruction description. + + Args: + min_words: An integer specifying the minimum number of words contained in the response. + max_words: An integer specifying the maximum number of words contained in the response. + + Returns: + A string representing the instruction description. + """ + self._min_words = min_words + self._max_words = max_words + + if self._min_words is None or self._min_words < 0: + self._min_words = random.randint( + _NUM_WORDS_LOWER_LIMIT, _NUM_WORDS_UPPER_LIMIT + ) + + # Make the range small + if self._max_words is None or self._max_words < 0: + self._max_words = self._min_words + random.randint(int(self._min_words * 0.05), int(self._min_words * 0.1)) + + self._description_pattern = "The response must contain between {min_words} and {max_words} words." + + return self._description_pattern.format( + min_words=self._min_words, max_words=self._max_words + ) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"min_words": self._min_words, "max_words": self._max_words} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["min_words", "max_words"] + + def check_following(self, value): + """Checks if the response contains the expected number of words.""" + num_words = instructions_util.count_words(value) + return self._min_words <= num_words <= self._max_words + + +class UniqueWordCountChecker(Instruction): + """Unique Word Count: The response must contain X unique words.""" + + def build_description(self, *, N=None): + """Build the instruction description. + + Args: + n: An integer specifying the number of unique words contained in the response. + + Returns: + A string representing the instruction description. + """ + self._num_unique_words = N + + if self._num_unique_words is None or self._num_unique_words < 0: + self._num_unique_words = random.randint( + _NUM_WORDS_LOWER_LIMIT, _NUM_WORDS_UPPER_LIMIT + ) + + self._description_pattern = "Use at least {N} unique words in the response." + + return self._description_pattern.format(N=self._num_unique_words) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"N": self._num_unique_words} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["N"] + + def check_following(self, value): + """Checks if the response contains the expected number of unique words.""" + words = value.lower().split() + unique_words = set() + for word in words: + unique_words.add(word.strip(''.join(string.punctuation) + ' ')) + # Convert to set to get unique words + return len(unique_words) >= self._num_unique_words + + +class StopWordPercentageChecker(Instruction): + """Ensure that stop words constitute no more than {percentage}% of the total words in your response.""" + + def build_description(self, *, percentage=None): + """Build the instruction description. + + Args: + percentage: An integer specifying the percentage of stop words that are allowed in the response. + + Returns: + A string representing the instruction description. + """ + self._percentage = percentage + + if self._percentage is None or self._percentage < 0: + self._percentage = random.randint(1, 100) + + self._description_pattern = "Ensure that stop words constitute no more than {percentage}% of the total words in your response." + + return self._description_pattern.format(percentage=self._percentage) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"percentage": self._percentage} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["percentage"] + + def check_following(self, value): + """Checks if the response contains the expected percentage of stop words.""" + num_words = instructions_util.count_words(value) + num_stopwords = instructions_util.count_stopwords(value) + stopword_percentage = (num_stopwords / num_words) * 100 + return stopword_percentage <= self._percentage + + +class SentTypeRatioChecker(Instruction): + """Maintain a 2:1 ratio of declarative to interrogative sentences.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = "Maintain a 2:1 ratio of declarative to interrogative sentences." + return self._description_pattern + + def get_instruction_args(self): + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response contains the expected ratio of declarative to interrogative sentences.""" + # Split the text into sentences + sentences = instructions_util.split_into_sentences(value) + # Count the number of declarative and interrogative sentences + declarative_count = sum(1 for sentence in sentences if sentence.endswith('.')) + interrogative_count = sum(1 for sentence in sentences if sentence.endswith('?')) + # Check if the ratio is 2:1 + return declarative_count == 2 * interrogative_count + + +class SentBalanceChecker(Instruction): + """Ensure that the ratio of sentence types (declarative, interrogative, exclamatory) is balanced.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = "Ensure that the ratio of sentence types (declarative, interrogative, exclamatory) is balanced." + return self._description_pattern + + def get_instruction_args(self): + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response contains a balanced ratio of sentence types.""" + # Split the text into sentences + sentences = instructions_util.split_into_sentences(value) + # Count the number of each sentence type + declarative_count = sum(1 for sentence in sentences if sentence.endswith('.')) + interrogative_count = sum(1 for sentence in sentences if sentence.endswith('?')) + exclamatory_count = sum(1 for sentence in sentences if sentence.endswith('!')) + # Check if the ratio of sentence types is balanced + return declarative_count == interrogative_count == exclamatory_count + + +class ConjunctionCountChecker(Instruction): + """Use at least {small_n} different coordinating conjunctions in the response.""" + + def build_description(self, *, small_n=None): + """Build the instruction description. + + Args: + small_n: An integer specifying the number of different coordinating conjunctions contained in the response. + + Returns: + A string representing the instruction description. + """ + self._num_conjunctions = small_n + + if self._num_conjunctions is None or self._num_conjunctions < 0: + self._num_conjunctions = random.randint(2, _NUM_CONJUNCTIONS) + + self._description_pattern = "Use at least {small_n} different coordinating conjunctions in the response." + + return self._description_pattern.format(small_n=self._num_conjunctions) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"small_n": self._num_conjunctions} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["small_n"] + + def check_following(self, value): + """Checks if the response contains the expected number of different coordinating conjunctions.""" + # Split the text into words + words = value.split() + # Count the number of coordinating conjunctions + conjunctions = [word for word in words if + word.strip(''.join(string.punctuation) + ' ').lower() in ['and', 'but', 'for', 'nor', 'or', + 'so', 'yet']] + unique_conjunctions = set(conjunctions) + return len(unique_conjunctions) >= self._num_conjunctions + + +class PersonNameCountChecker(Instruction): + """Mention at least {N} different person names in the response, from this list of person names: Emma, Liam, Sophia...""" + + def build_description(self, *, N=None): + """Build the instruction description. + + Args: + N: An integer specifying the minimum number of unique person names contained in the response. + + Returns: + A string representing the instruction description. + """ + self._num_person_names = N + + if self._num_person_names is None or self._num_person_names < 0: + self._num_person_names = random.randint(1, 50) + + self._description_pattern = "Mention at least {N} different person names in the response, from this list of person names: Emma, Liam, Sophia, Jackson, Olivia, Noah, Ava, Lucas, Isabella, Mason, Mia, Ethan, Charlotte, Alexander, Amelia, Benjamin, Harper, Leo, Zoe, Daniel, Chloe, Samuel, Lily, Matthew, Grace, Owen, Abigail, Gabriel, Ella, Jacob, Scarlett, Nathan, Victoria, Elijah, Layla, Nicholas, Audrey, David, Hannah, Christopher, Penelope, Thomas, Nora, Andrew, Aria, Joseph, Claire, Ryan, Stella, Jonathan ." + return self._description_pattern.format(N=self._num_person_names) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"N": self._num_person_names} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["N"] + + def check_following(self, value): + """Checks if the response contains at least the expected number of unique person names.""" + person_name_list = ["Emma", "Liam", "Sophia", "Jackson", "Olivia", "Noah", "Ava", "Lucas", "Isabella", "Mason", + "Mia", "Ethan", "Charlotte", + "Alexander", + "Amelia", + "Benjamin", + "Harper", + "Leo", + "Zoe", + "Daniel", + "Chloe", + "Samuel", + "Lily", + "Matthew", + "Grace", + "Owen", + "Abigail", + "Gabriel", + "Ella", + "Jacob", + "Scarlett", + "Nathan", + "Victoria", + "Elijah", + "Layla", + "Nicholas", + "Audrey", + "David", + "Hannah", + "Christopher", + "Penelope", + "Thomas", + "Nora", + "Andrew", + "Aria", + "Joseph", + "Claire", + "Ryan", + "Stella", + "Jonathan" + ] + # Extract the named entities + person_names = [] + for name in person_name_list: + if name in value: + person_names.append(name) + unique_person_names = set(person_names) + + return len(unique_person_names) >= self._num_person_names + + +class NGramOverlapChecker(Instruction): + """Maintain a trigram overlap of {percentage}% (±2%) with the provided reference text.""" + + def build_description(self, *, reference_text=None, percentage=None): + """Build the instruction description. + + Args: + reference_text: A string representing the reference text. + percentage: An integer specifying the percent trigram overlap + to maintain in the response. + + Returns: + A string representing the instruction description. + """ + self._reference_text = reference_text + self._percentage = percentage + if self._percentage is None or self._percentage < 0: + self._percentage = random.randint(1, 100) + + self._description_pattern = "Maintain a trigram overlap of {percentage}% (±2%) with the provided reference text." + return self._description_pattern.format(percentage=self._percentage) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"reference_text": self._reference_text, "percentage": self._percentage} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["reference_text", "percentage"] + + def check_following(self, value): + """Checks if the response maintains a trigram overlap with the reference text within 2% of {percent}.""" + n = 3 + ngrams = set(nltk.ngrams(value, n)) + ref_ngrams = set(nltk.ngrams(self._reference_text, n)) + overlap = len(ngrams.intersection(ref_ngrams)) / len(ngrams) + return self._percentage - 2 <= overlap * 100 <= self._percentage + 2 + + +class NumbersCountChecker(Instruction): + """Include exactly {N} numbers in the response.""" + + def build_description(self, *, N=None): + """Build the instruction description. + + Args: + N: An integer specifying the exact number of numbers + that is required to appear in the response. + + Returns: + A string representing the instruction description. + """ + self._count_numbers = N + if self._count_numbers is None or self._count_numbers < 0: + self._count_numbers = random.randint(1, _NUM_NUMBERS) + + self._description_pattern = "Include exactly {N} numbers in the response." + return self._description_pattern.format(N=self._count_numbers) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"N": self._count_numbers} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["N"] + + def check_following(self, value): + """Checks if the response includes exactly {N} numbers.""" + # Strip punctuation to handle decimals and commas in numbers correctly + value = value.translate(str.maketrans('', '', string.punctuation)) + numbers = re.findall(r'\d+', value) + return len(numbers) == self._count_numbers + + +class AlphabetLoopChecker(Instruction): + """Each word must start with the next letter of the alphabet, looping back to 'A' after 'Z'.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = "Each word must start with the next letter of the alphabet, looping back to 'A' after 'Z'." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if each word of the response starts with the next letter of the alphabet.""" + value = value.translate(str.maketrans('', '', string.punctuation)) + words = value.strip(''.join(string.punctuation) + ' ').split() + if not words: + return False + alphabet = string.ascii_lowercase + correct_letter = words[0][0].lower() + if correct_letter not in alphabet: # numbers are fails + return False + for word in words[1:]: + word = word.strip(''.join(string.punctuation) + ' ').lower() + if not word: + continue + correct_letter = alphabet[(alphabet.index(correct_letter) + 1) % 26] + if word[0] != correct_letter: + return False + return True + + +class SingleVowelParagraphChecker(Instruction): + """Write a paragraph using words that contain only three type of vowels.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = "Write a paragraph using words that contain only three types of vowels." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if no more than three types of vowels are used in the response and the response is only 1 paragraph.""" + paragraphs = value.strip().split('\n') + if len(paragraphs) != 1: + return False + paragraph = paragraphs[0].lower() + + vowels = set('aeiou') + paragraph_vowels = set([char for char in paragraph if char in vowels]) + return len(paragraph_vowels) <= 3 + + +class ConsonantClusterChecker(Instruction): + """Ensure each word in your response has at least one consonant cluster (two or more consonants together).""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = "Ensure each word in your response has at least one consonant cluster (two or more consonants together)." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if each word in the response includes at least one consonant cluster.""" + words = value.lower().strip().split() + consonants = set('bcdfghjklmnpqrstvwxyz') + for word in words: + cluster = False + for i in range(len(word) - 1): + if word[i] in consonants and word[i + 1] in consonants: + cluster = True + break + if not cluster: + return False + return True + + +class IncrementingAlliterationChecker(Instruction): + """Each sentence must have a longer sequence of consecutive alliterative words than the previous one.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = "Each sentence must have a longer sequence of consecutive alliterative words than the previous one." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if each sentence in the response has more alliterative words (determined by common first letter) than the previous sentence.""" + sentences = instructions_util.split_into_sentences(value) + prev_alliteration = -1 + for sentence in sentences: + words = sentence.lower().split() + alliteration = 0 + prev_alliterative = False + new_words = [] + for word in words: + clean = word.lstrip(''.join(string.punctuation) + ' ') + if clean: + new_words.append(clean) + for i in range(len(new_words) - 1): + if new_words[i][0] == new_words[i + 1][0]: + if prev_alliterative: + alliteration += 1 + else: + alliteration += 2 + prev_alliterative = True + else: + prev_alliterative = False + if alliteration <= prev_alliteration: + return False + prev_alliteration = alliteration + return True + + +class PalindromeChecker(Instruction): + """Include at least 10 single-word palindromes, each at least 5 characters long.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = "Include at least 10 single-word palindromes, each at least 5 characters long." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response includes at least 10 single-word palindromes of length at least 5.""" + value = value.translate(str.maketrans('', '', string.punctuation)) + words = value.lower().split() + palindromes = [word for word in words if word == word[::-1] and len(word) >= 5] + return len(palindromes) >= 10 + + +class PunctuationCoverChecker(Instruction): + """Use every standard punctuation mark at least once, including semicolons, colons, and the interrobang (?!).""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = "Use every standard punctuation mark at least once, including semicolons, colons, and the interrobang (?!)." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response includes every standard punctuation mark at least once, including the interrobang (?!).""" + punctuation = {".", ",", "!", "?", ";", ":"} + if not ('!?' in value or '?!' in value or '‽' in value): + return False + new_value = value.replace('?!', '', 1) + if len(new_value) == len(value): + new_value = value.replace('!?', '', 1) + for char in new_value: + if char in punctuation: + punctuation.remove(char) + return not punctuation + + +class NestedParenthesesChecker(Instruction): + """Nest parentheses (and [brackets {and braces}]) at least 5 levels deep.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = "Nest parentheses (and [brackets {and braces}]) at least 5 levels deep." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response includes a correctly closed set of at least 5 nested brackets.""" + levels = [] + min_levels = 5 + max_depth = 0 + depth_stack = [] # Track depth per matched group + + for char in value: + if char in "([{": + levels.append(char) + if len(levels) > max_depth: + max_depth = len(levels) + elif char in ")]}": + if levels and ( + (levels[-1] == '(' and char == ')') or + (levels[-1] == '[' and char == ']') or + (levels[-1] == '{' and char == '}') + ): + levels.pop() + # Check if we just closed a group that reached 5+ depth + if max_depth >= min_levels and len(levels) < max_depth: + return True + else: + # Mismatch — reset + levels = [] + max_depth = 0 + + return False + + +class NestedQuotesChecker(Instruction): + """Include quotes within quotes within quotes, at least 3 levels deep, alternating between double quotes and single quotes.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = "Include quotes within quotes within quotes, at least 3 levels deep, alternating between double quotes and single quotes." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response includes nested quotes to at least 3 levels + alternating between " and ' starting with either character.""" + levels = [] + min_levels = 3 + reached_depth = 0 + current_depth = 0 + for char in value: + if len(levels) != 0 and char == levels[-1]: + levels.pop() + current_depth -= 1 + if reached_depth - current_depth >= min_levels: + return True + elif char == '"' or char == "'": + levels.append(char) + current_depth += 1 + if current_depth > reached_depth: + reached_depth = current_depth + return False + + +class PrimeLengthsChecker(Instruction): + """Use only words with lengths that are prime numbers.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = "Use only words with lengths that are prime numbers." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response only includes words with prime length.""" + value = value.translate(str.maketrans('', '', string.punctuation)) + words = value.split() + primes = set([2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97]) + for word in words: + if len(word) not in primes: + return False + return True + + +class OptionsResponseChecker(Instruction): + """Answer with one of the following options: {options}. Do not give any explanation.""" + + def build_description(self, *, options=None): + """Build the instruction description. + + Args: + options: A string specifying the permitted options for + the response. + + Returns: + A string representing the instruction description. + """ + # Options string may be: yes/no/maybe, I know or I don't know, a), b), c), d) + # Can be separated by "/", "or", "," + options_bank = ["yes/no/maybe", "I know or I don't know", "a), b), c), d)"] + if options is None: + options = random.choice(options_bank) + + # Be more strict about format for multiple choice letters than for text options + self._strict = False + if re.match(r"\W*[aA]\W*[bB]\W*[cC]\W*", options) is not None: + self._strict = True + if "/" in options: + separator = "/" + elif "or" in options: + separator = "or" + else: + separator = "," + self._options = [option.strip() for option in options.split(separator)] + self._options_text = options # in text, shouldn't be formatted as a list + self._description_pattern = "Answer with one of the following options: {options}. Do not give any explanation." + return self._description_pattern.format(options=self._options_text) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"options": self._options_text} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["options"] + + def check_following(self, value): + """Checks if the response is exactly one of {options}.""" + if self._strict: + return value in self._options + value = value.strip(''.join(string.punctuation) + ' ').lower() + for option in self._options: + if option.strip(''.join(string.punctuation) + ' ').lower() == value: + return True + return False + + +class NewLineWordsChecker(Instruction): + """Write each word on a new line.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = "Write each word on a new line." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response has each word on a new line.""" + value = value.translate(str.maketrans('', '', string.punctuation)) + lines = value.strip().split('\n') + while '' in lines: + lines.remove('') + return len(lines) == len(value.strip().split()) + + +class EmojiSentenceChecker(Instruction): + """Please use an emoji at the end of every sentence.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = "Please use an emoji at the end of every sentence." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response includes an emoji at the end of every sentence.""" + + sentences = instructions_util.split_into_sentences(value) + for i, sentence in enumerate(sentences): + stripped = sentence.translate(str.maketrans('', '', string.punctuation)).strip() + # check for empty string + if not stripped: + return False + last_char = stripped[-1] + # because blank spaces are treated oddly + second_last_char = stripped[-2] if len(stripped) > 1 else stripped[-1] + if not emoji.is_emoji(last_char) and not emoji.is_emoji(second_last_char): + if i < len(sentences) - 1: + stripped = sentences[i + 1].translate(str.maketrans('', '', string.punctuation)).strip() + # fixed empty string + if not stripped: + return False + first_char = stripped[0] + if not emoji.is_emoji(first_char): + return False + else: + return False + return True + + +class CharacterCountUniqueWordsChecker(Instruction): + """Respond with three sentences, all containing the same number of characters but using all different words.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = "Respond with three sentences, all containing the same number of characters but using all different words." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response has exactly 3 sentences containing the same number of characters but different words.""" + sentences = instructions_util.split_into_sentences(value) + if len(sentences) != 3: + return False + char_count = len(sentences[0].strip()) + for sentence in sentences: + if len(sentence.strip()) != char_count: + return False + return True + + +class NthWordJapaneseChecker(Instruction): + """Every {N}th word of your response must be in Japanese.""" + + def build_description(self, *, N=None): + """Build the instruction description. + + Args: + N: An integer specifying the cycle length for + Japanese words to appear in the response. + + Returns: + A string representing the instruction description. + """ + self._japanese_position = N + if self._japanese_position is None or self._japanese_position < 0: + self._japanese_position = random.randint(1, _NUM_WORD_CYCLE) + + self._description_pattern = "Every {N}th word of your response must be in Japanese." + if N % 10 == 1: + self._description_pattern = "Every {N}st of your response must be in Japanese." + if N % 10 == 2: + self._description_pattern = "Every {N}nd of your response must be in Japanese." + elif N % 10 == 3: + self._description_pattern = "Every {N}rd of your response must be in Japanese." + return self._description_pattern.format(N=self._japanese_position) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"N": self._japanese_position} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["N"] + + def check_following(self, value): + """Checks if every {N}th word of the response is in Japanese.""" + + def is_japanese(text): + """ + Checks if a string contains Japanese characters (Hiragana, Katakana, or Kanji). + + Args: + text: The string to check. + + Returns: + True if the string contains Japanese characters, False otherwise. + """ + japanese_pattern = re.compile(r'[\u3040-\u30ff\u4e00-\u9fff]') + return bool(japanese_pattern.search(text)) + + words = value.split() + for i, word in enumerate(words): + word = word.strip(''.join(string.punctuation) + ' ') + if (i + 1) % self._japanese_position == 0 and word and not word.isdigit(): + if not is_japanese(word): + return False + return True + + +class StartWithVerbChecker(Instruction): + """The response must start with a verb.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = "The response must start with a verb." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response starts with a verb.""" + text = nltk.word_tokenize(value) + return len(text) > 0 and len(nltk.pos_tag(text)) > 0 and 'VB' in nltk.pos_tag(text)[0][1] + + +class LimitedWordRepeatChecker(Instruction): + """The response should not repeat any word more than {small_n} times.""" + + def build_description(self, *, small_n=None): + """Build the instruction description. + + Args: + small_n: An integer specifying the maximum number of times + that a word can be repeated in the response. + + Returns: + A string representing the instruction description. + """ + self._max_repeats = small_n + if self._max_repeats is None or self._max_repeats < 0: + self._max_repeats = random.randint(1, _MAX_REPEATS) + + self._description_pattern = "The response should not repeat any word more than {small_n} times." + return self._description_pattern.format(small_n=self._max_repeats) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"small_n": self._max_repeats} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["small_n"] + + def check_following(self, value): + """Checks if the response repeats any word more than {small_n} times.""" + words = value.lower().translate(str.maketrans('', '', string.punctuation)).split() + word_count = Counter(words) + for word, count in word_count.items(): + if count > self._max_repeats: + return False + return True + + +class IncludeKeywordChecker(Instruction): + """The response must include keyword {word} in the {N}-th sentence.""" + + def build_description(self, *, word=None, N=None): + """Build the instruction description. + + Args: + word: A string specifying the keyword that is + required to appear in the response. + N: An integer specifying which sentence of the + response is required to have the keyword. + + Returns: + A string representing the instruction description. + """ + + if not word: + self._keyword = instructions_util.generate_keywords( + num_keywords=1 + )[0] + else: + self._keyword = word + self._keyword_position = N + if self._keyword_position is None or self._keyword_position < 0: + self._keyword_position = random.randint(1, _NUM_KEYWORD_SENTENCE) + + self._description_pattern = "The response must include keyword \"{word}\" in the {N}-th sentence." + return self._description_pattern.format(word=self._keyword, N=self._keyword_position) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"word": self._keyword, "N": self._keyword_position} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["word", "N"] + + def check_following(self, value): + """Checks if the {N}th sentence of the response includes keyword {word}.""" + sentences = instructions_util.split_into_sentences(value) + if len(sentences) < self._keyword_position: + return False + return self._keyword.lower() in sentences[int(self._keyword_position - 1)].lower() + + +class PronounCountChecker(Instruction): + """The response should include at least {N} pronouns.""" + + def build_description(self, *, N=None): + """Build the instruction description. + + Args: + N: An integer specifying the minimum number of pronouns + that is required to appear in the response. + + Returns: + A string representing the instruction description. + """ + self._num_pronouns = N + if self._num_pronouns is None or self._num_pronouns < 0: + self._num_pronouns = random.randint(1, _NUM_PRONOUNS) + + self._description_pattern = "The response should include at least {N} pronouns." + return self._description_pattern.format(N=self._num_pronouns) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"N": self._num_pronouns} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["N"] + + def check_following(self, value): + """Checks if the response includes at least {N} pronouns.""" + pronouns = set( + ['i', 'me', 'my', 'mine', 'myself', 'we', 'us', 'our', 'ours', 'ourselves', 'you', 'your', 'yours', + 'yourself', 'yourselves', 'he', 'him', 'his', 'himself', 'she', 'her', 'hers', 'herself', 'it', 'its', + 'itself', 'they', 'them', 'their', 'theirs', 'themselves']) + value = value.replace('/', + ' ') # to correctly count pronoun sets like she/her/hers, a common use case of pronouns + value = value.lower().translate(str.maketrans('', '', string.punctuation)) + words = value.split() + pronoun_count = sum(1 for word in words if word in pronouns) + return pronoun_count >= self._num_pronouns + + +class AlternateParitySyllablesChecker(Instruction): + """Alternate between words with odd and even numbers of syllables.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = "Alternate between words with odd and even numbers of syllables." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response alternates between words with odd and even numbers of syllables.""" + words = value.translate(str.maketrans('', '', string.punctuation)).lower().split() + syllables = [syllapy.count(word) % 2 for word in words if word.strip()] + return all(syllables[i] != syllables[i + 1] for i in range(len(syllables) - 1)) + + +class LastWordFirstNextChecker(Instruction): + """The last word of each sentence must become the first word of the next sentence.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = "The last word of each sentence must become the first word of the next sentence." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the last word of each sentence in the response is the first word of the next sentence.""" + sentences = instructions_util.split_into_sentences(value) + for i in range(len(sentences) - 1): + last_word = sentences[i].rstrip(''.join(string.punctuation) + ' ').split()[-1] + first_word = sentences[i + 1].lstrip(''.join(string.punctuation) + ' ').split()[0] + if last_word.lower() != first_word.lower(): + return False + return True + + +class ParagraphLastFirstWordMatchChecker(Instruction): + """Each paragraph must end with the same word it started with, separate paragraphs with a newline.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = "Each paragraph must end with the same word it started with, separate paragraphs with a newline." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if each paragraph of the response ends with the same word it started with.""" + paragraphs = value.split('\n') + for paragraph in paragraphs: + paragraph = paragraph.strip().lower() + if not paragraph: + continue + words = paragraph.strip(''.join(string.punctuation) + ' ').split() + if not words: + continue + if words[0] != words[-1]: + return False + return True + + +class IncrementingWordCountChecker(Instruction): + """Each sentence must contain exactly {small_n} more words than the previous one.""" + + def build_description(self, *, small_n=None): + """Build the instruction description. + + Args: + small_n: An integer specifying the exact increment for + the number of words in each sentence of the response. + + Returns: + A string representing the instruction description. + """ + self._num_increment = small_n + if self._num_increment is None or self._num_increment < 0: + self._num_increment = random.randint(1, _NUM_INCREMENT) + + + self._description_pattern = "Each sentence must contain exactly {small_n} more words than the previous one." + return self._description_pattern.format(small_n=self._num_increment) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"small_n": self._num_increment} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["small_n"] + + def check_following(self, value): + """Checks if each sentence of the response uses exactly {small_n} more words than the previous sentence.""" + sentences = instructions_util.split_into_sentences(value) + words = sentences[0].translate(str.maketrans('', '', string.punctuation)).strip().split() + while '' in words: + words.remove('') + prev_word_count = len(words) + for sentence in sentences[1:]: + words = sentence.translate(str.maketrans('', '', string.punctuation)).strip().split() + while '' in words: + words.remove('') + if len(words) != prev_word_count + self._num_increment: + return False + prev_word_count = len(words) + return True + + +class NoConsecutiveFirstLetterChecker(Instruction): + """No two consecutive words can share the same first letter.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = "No two consecutive words can share the same first letter." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if no two consecutive words in the response share the same first letter.""" + words = value.lower().translate(str.maketrans('', '', string.punctuation)).split() + while '' in words: + words.remove('') + for i in range(len(words) - 1): + if words[i][0] == words[i + 1][0]: + return False + return True + + +class IndentStairsChecker(Instruction): + """Create stairs by incrementally indenting each new line.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = "Create stairs by incrementally indenting each new line." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response incrementally indents each new line.""" + lines = value.split('\n') + for line in lines: + if not line.strip(): + lines.remove(line) + for i in range(len(lines) - 1): + if len(lines[i + 1]) - len(lines[i + 1].lstrip(' ')) <= len(lines[i]) - len(lines[i].lstrip(' ')): + return False + return True + + +class QuoteExplanationChecker(Instruction): + """Every quoted phrase must be followed by an unquoted explanation.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = "Every quoted phrase must be followed by an unquoted explanation." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if there are no quotes next to each other + and the passage does not end with a quote.""" + value = value.replace('“', '"').replace('”', '"') + value = value.replace("'\"'", '') # remove references to the character '"' + value = ''.join(value.split()) # remove all whitespace + if '""' in value: + return False + if value.strip(string.digits + string.punctuation.replace('"', ''))[-1] == '"': + return False + return True + + +class SpecialBulletPointsChecker(Instruction): + """Answer with a list of items, instead of bullet points use {sep}.""" + + def build_description(self, *, sep=None): + """Build the instruction description. + + Args: + sep: A string specifying the bullet point marker for + the list in the response. + + Returns: + A string representing the instruction description. + """ + self._bullet_marker = sep + if sep is None: + self._bullet_marker = random.choice(['...', 'SEPARATOR', '!?!?', '-']) + self._description_pattern = "Answer with a list of items, instead of bullet points use {sep}." + return self._description_pattern.format(sep=self._bullet_marker) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"sep": self._bullet_marker} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["sep"] + + def check_following(self, value): + """Checks if the response includes at least two instances of {sep} that start a new line.""" + return len(re.findall(re.escape(self._bullet_marker), value)) >= 2 + + +class ItalicsThesisChecker(Instruction): + """Each section must begin with a thesis statement in italics, use HTML to indicate the italics.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = "Each section must begin with a thesis statement in italics, use HTML to indicate the italics." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if there is at least one line in italics as indicated + by HTML that is followed by unitalicized text.""" + index = value.find('') + if index == -1: + index = value.find('') + if index == -1: + return False + value = value[index:] + end_thesis = value.find('') + if end_thesis == -1: + end_thesis = value.find('') + if end_thesis == -1: + return False + thesis = value[3:end_thesis] + if thesis.strip() == '': + return False + text = value[end_thesis + 4:] + return text.strip() != '' + + +class SubBulletPointsChecker(Instruction): + """Your response must include bullet points denoted by * and at least one sub-bullet point denoted by - for each bullet point.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = "Your response must include bullet points denoted by * and at least one sub-bullet point denoted by - for each bullet point." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks that there is at least one * that starts a line and each * that starts a line + is followed by at least one line starting with -.""" + bullets = value.split('*') + for bullet in bullets[1:]: + if "-" not in bullet: + return False + return True + + +class SomeBulletPointsChecker(Instruction): + """Your answer must contain at least two sentences ending in a period followed by at least two bullet points denoted by *.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = "Your answer must contain at least two sentences ending in a period followed by at least two bullet points denoted by *." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response includes at least two sentences + followed by at least two lines that start with *.""" + lines = value.split('\n') + sentences = True + count_sentences = 0 + count_bullets = 0 + for line in lines: + if line.strip().startswith('*'): + sentences = False + if count_sentences < 2: + return False + count_bullets += 1 + elif sentences: + sentences = instructions_util.split_into_sentences(line.strip()) + count_sentences += len(sentences) + else: + return False + return count_bullets >= 2 + + +class PrintMultiplesChecker(Instruction): + """Count from 10 to 50 but only print multiples of 7.""" + + def build_description(self, **kwargs): + self._description_pattern = "Count from 10 to 50 but only print multiples of 7." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response prints multiples of 7 from 10 to 50.""" + value = value.replace(',', ', ') + numbers = re.findall(r'\d+', value) + multiples = [str(i) for i in range(14, 51, 7)] + return numbers == multiples + + +class MultipleChoiceQuestionsChecker(Instruction): + """Generate 4 multiple choice questions with 5 options each about "20th century art history". Each question should start with the label "Question". The questions should get progressively longer. Do not provide an explanation.""" + + def build_description(self, **kwargs): + self._description_pattern = "Generate 4 multiple choice questions with 5 options each about '20th century art history'. Each question should start with the label \"Question\". The questions should get progressively longer. Do not provide an explanation." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response generates 4 multiple choice questions with 5 options.""" + # Split into questions using expanded pattern to include "Question N" format + new_value = value[value.find('Question'):] + if new_value != value: + return False # failed no explanation + value = new_value + questions = re.split(r'\n*(?:Question \d+[\.|\):;]?\s*)', value) + if questions[0] == '': + questions = questions[1:] + questions = [q.strip() for q in questions if q.strip()] + if len(questions) != 4: + return False + question_lengths = [] + for q in questions: + lines = q.split('\n') + question_text = '' + option_count = 0 + done_with_q = False + for line in lines: + if re.match(r'^[A-Ea-e][\.|\)]\s*\w+', line.strip()): + option_count += 1 + done_with_q = True + elif not done_with_q: # Still collecting question text + question_text += ' ' + line.strip() + if option_count != 5: + return False + question_lengths.append(len(question_text.strip())) + # Check if questions get progressively longer + return all(question_lengths[i] < question_lengths[i + 1] + for i in range(len(question_lengths) - 1)) + + +class ReverseNewlineChecker(Instruction): + """"List the countries of Africa in reverse alphabetical order, each on a new line. """ + + def build_description(self, **kwargs): + self._description_pattern = "List the countries of Africa in reverse alphabetical order, each on a new line." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """ + Checks if text satisfies the following constraints: + 1. Contains at least 53 newlines with text + 2. Lines are in reverse alphabetical order + 3. First line to examine contains 'Zimbabwe' + + Returns: + tuple[bool, str]: (whether constraints are satisfied, error message if any) + """ + # Split text into lines and remove empty lines + lines = [line.strip(''.join(string.punctuation) + ' ') for line in value.split('\n') if + line.strip(''.join(string.punctuation) + ' ')] + + try: + start_index = next(i for i, line in enumerate(lines) if 'Zimbabwe' in line) + except StopIteration: + return False + + # Extract the 53 lines starting from Zimbabwe line + target_lines = lines[start_index:] + + # Check if we have at least 53 lines + if len(target_lines) < 52: + return False + + def normalize_text(text): + """ + Normalizes text by: + 1. Converting to NFKD form (separates combined characters) + 2. Removes diacritical marks + 3. Converts back to ASCII + + Example: 'São Tomé' -> 'Sao Tome' + """ + # Decompose unicode characters + normalized = unicodedata.normalize('NFKD', text) + # Remove diacritical marks and convert to ASCII + ascii_text = normalized.encode('ASCII', 'ignore').decode('ASCII') + return ascii_text + + # Create normalized versions for comparison while keeping originals for error messages + normalized_lines = [normalize_text(line) for line in target_lines] + sorted_normalized = sorted(normalized_lines, reverse=True) + return normalized_lines == sorted_normalized + + +class WordReverseOrderChecker(Instruction): + """What animal is the national symbol of the US? Respond to this query, but make your sentence in reverse order of what it should be, per word.""" + + def build_description(self, **kwargs): + self._description_pattern = "What animal is the national symbol of the US? Respond to this query, but make your sentence in reverse order of what it should be, per word." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the reverse of the sentence is a valid English sentence.""" + value = value.lower().strip().translate(str.maketrans('', '', string.punctuation)) + value = ' '.join(value.split()[::-1]) + if 'bald eagle' not in value: + return False + return value in instructions_util.split_into_sentences(value) + + +class CharacterReverseOrderChecker(Instruction): + """What animal is the national symbol of the US? Respond to this query, but make your sentence in reverse order of what it should be, per letter.""" + + def build_description(self, **kwargs): + self._description_pattern = "What animal is the national symbol of the US? Respond to this query, but make your sentence in reverse order of what it should be, per letter." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + value = value.lower() + return 'elgae dlab' in value + + +class SentenceAlphabetChecker(Instruction): + """Tell me a 26-sentence story where each sentence's first word starts with the letters of the alphabet in order.""" + + def build_description(self, **kwargs): + self._description_pattern = "Tell me a 26-sentence story where each sentence's first word starts with the letters of the alphabet in order." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + sentences = instructions_util.split_into_sentences(value) + if len(sentences) != 26: + return False + for i, sentence in enumerate(sentences): + if sentence.lstrip().split()[0].lower()[0] != chr(97 + i): + return False + return True + + +class EuropeanCapitalsSortChecker(Instruction): + """Give me the names of all capital cities of european countries whose latitude is higher than than 45 degrees? List the capital cities without country names, separated by commas, sorted by latitude, from highest to lowest.""" + + def build_description(self, **kwargs): + """Build the instruction description.""" + self._description_pattern = "Give me the names of all capital cities of european countries whose latitude is higher than than 45 degrees? List the capital cities without country names, separated by commas, sorted by latitude, from highest to lowest." + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response lists the relevant capitals of Europe in correct order.""" + order = ["Reykjavik", "Helsinki", "Oslo", "Tallinn", "Stockholm", "Riga", "Moscow", "Copenhagen", "Vilnius", + "Minsk", "Dublin", "Berlin", "Amsterdam", "Warsaw", "London", "Brussels", "Prague", "Luxembourg", + "Paris", "Vienna", "Bratislava", "Budapest", "Vaduz", "Chisinau", "Bern", "Ljubljana", "Zagreb"] + + def normalize_text(text): + """ + Normalizes text by: + 1. Converting to NFKD form (separates combined characters) + 2. Removes diacritical marks + 3. Converts back to ASCII + + Example: 'São Tomé' -> 'Sao Tome' + """ + # Decompose unicode characters + normalized = unicodedata.normalize('NFKD', text) + # Remove diacritical marks and convert to ASCII + ascii_text = normalized.encode('ASCII', 'ignore').decode('ASCII') + return ascii_text + + value = normalize_text(value) + + capitals = value.split(',') + capitals = [cap for cap in capitals if cap.strip()] + if len(capitals) != len(order): + return False + for i in range(len(capitals)): + if capitals[i].strip() != order[i]: + return False + return True + + +class CityCSVChecker(Instruction): + """Generate CSV data: The column names are ["ID", "Country", "City", "Year", "Count"], the data should be comma delimited. Please generate 7 rows.""" + + def build_description(self, **kwargs): + """Build the instruction description.""" + self._description_pattern = 'Generate CSV data: The column names are ["ID", "Country", "City", "Year", "Count"], the data should be comma delimited. Please generate 7 rows.' + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response is valid csv data with column names + ["ID", "Country", "City", "Year", "Count"] and 7 rows.""" + string_io = io.StringIO(value) + reader = csv.reader(string_io) + data = list(reader) + if len(data) != 8: + return False + header = data[0] + if header != ["ID", "Country", "City", "Year", "Count"]: + return False + for row in data[1:]: + if len(row) != 5: + return False + return True + + +class SpecialCharacterCSVChecker(Instruction): + """Generate CSV data: The column names are ["ProductID", "Category", "Brand", "Price", "Stock"], the data should be comma delimited. Please generate 14 rows. Add one field which contains a special character and enclose it in double quotes.""" + + def build_description(self, **kwargs): + """Build the instruction description.""" + self._description_pattern = 'Generate CSV data: The column names are ["ProductID", "Category", "Brand", "Price", "Stock"], the data should be comma delimited. Please generate 14 rows. Add one field which contains a special character and enclose it in double quotes.' + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """"Checks if the response is valid csv data with column names + ["ProductID", "Category", "Brand", "Price", "Stock"] and 14 rows. + Also checks if one field contains a special character enclosed in double quotes.""" + header = value.split('\n')[0].strip() + if not re.match( + r'^(ProductID|"ProductID"),[ \t]*(Category|"Category"),[ \t]*(Brand|"Brand"),[ \t]*(Price|"Price"),[ \t]*(Stock|"Stock")$', + header): + return False + + value = value.replace('"', '"""') + string_io = io.StringIO(value) + reader = csv.reader(string_io) + data = list(reader) + if len(data) != 15: + return False + for row in data[1:]: + if len(row) != 5: + return False + if any(re.match(r'".*[^\d\w\s].*"', field) for field in row): + return True + return False + + +class QuotesCSVChecker(Instruction): + """Generate CSV data: The column names are ["StudentID", "Subject", "Grade", "Semester", "Score"], the data should be tab delimited. Please generate 3 rows and enclose each single field in double quotes.""" + + def build_description(self, **kwargs): + """Build the instruction description.""" + self._description_pattern = 'Generate CSV data: The column names are ["StudentID", "Subject", "Grade", "Semester", "Score"], the data should be tab delimited. Please generate 3 rows and enclose each single field in double quotes.' + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """"Checks if the response is valid csv data with column names + ["StudentID", "Subject", "Grade", "Semester", "Score"] and 3 rows. + Also checks if each field is enclosed in double quotes.""" + header = value.split('\n')[0].strip() + if not re.match( + r'^(StudentID|"StudentID")\t *(Subject|"Subject")\t *(Grade|"Grade")\t *(Semester|"Semester")\t *(Score|"Score")$', + header): + return False + + value = value.replace('"', '"""') + string_io = io.StringIO(value) + reader = csv.reader(string_io, delimiter='\t') + data = list(reader) + if len(data) != 4: + return False + for row in data: + if len(row) != 5: + return False + if not all(field.strip()[0] == '"' and field.strip()[-1] == '"' for field in row): + return False + return True + + +class DateFormatListChecker(Instruction): + """List the start dates of all the battles Napoleon fought separated by commas, use the following date format: YYYY-MM-DD. Do not provide an explanation.""" + + def build_description(self, **kwargs): + """Build the instruction description.""" + self._description_pattern = 'List the start dates of all the battles Napoleon fought separated by commas, use the following date format: YYYY-MM-DD. Do not provide an explanation.' + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """"Checks if the response is a list of dates in the format YYYY-MM-DD separated by commas.""" + value = value.strip() + dates = value.split(',') + for date in dates: + date = date.strip() + if not re.match(r'^\d{4}-\d{2}-\d{2}$', date): + return False + date = date.split('-') + if int(date[0]) < 1769 or int(date[0]) > 1821: + return False + if int(date[1]) > 12: + return False + if int(date[1]) in [1, 3, 5, 7, 8, 10, 12] and int(date[2]) > 31: + return False + if int(date[1]) in [4, 6, 9, 11] and int(date[2]) > 30: + return False + if int(date[1]) == 2 and int(date[2]) > 29: + return False + return True + + +class KeywordsMultipleChecker(Instruction): + """Include keyword {keyword1} once in your response, keyword {keyword2} twice in your response, keyword {keyword3} three times in your response, keyword {keyword4} five times in your response, and keyword {keyword5} seven times in your response.""" + + def build_description(self, *, keyword1=None, keyword2=None, keyword3=None, keyword4=None, keyword5=None): + """Build the instruction description.""" + if keyword1 is None: + self._keyword1 = instructions_util.generate_keywords(num_keywords=1)[0] + else: + self._keyword1 = keyword1.strip() + if keyword2 is None: + self._keyword2 = instructions_util.generate_keywords(num_keywords=1)[0] + else: + self._keyword2 = keyword2.strip() + if keyword3 is None: + self._keyword3 = instructions_util.generate_keywords(num_keywords=1)[0] + else: + self._keyword3 = keyword3.strip() + if keyword4 is None: + self._keyword4 = instructions_util.generate_keywords(num_keywords=1)[0] + else: + self._keyword4 = keyword4.strip() + if keyword5 is None: + self._keyword5 = instructions_util.generate_keywords(num_keywords=1)[0] + else: + self._keyword5 = keyword5.strip() + self._description_pattern = "Include keyword {keyword1} once in your response, keyword {keyword2} twice in your response, keyword {keyword3} three times in your response, keyword {keyword4} five times in your response, and keyword {keyword5} seven times in your response." + return self._description_pattern.format(keyword1=self._keyword1, keyword2=self._keyword2, + keyword3=self._keyword3, keyword4=self._keyword4, + keyword5=self._keyword5) + + def get_instruction_args(self): + return {"keyword1": self._keyword1, "keyword2": self._keyword2, "keyword3": self._keyword3, + "keyword4": self._keyword4, "keyword5": self._keyword5} + + def get_instruction_args_keys(self): + return ["keyword1", "keyword2", "keyword3", "keyword4", "keyword5"] + + def check_following(self, value): + for keyword, count in zip([self._keyword1, self._keyword2, self._keyword3, self._keyword4, self._keyword5], + [1, 2, 3, 5, 7]): + if value.lower().count(keyword.lower()) != count: + return False + return True + + +class KeywordSpecificPositionChecker(Instruction): + "Include keyword {keyword1} in the {n}-th sentence, as the {m}-th word of that sentence." + + def build_description(self, keyword=None, n=None, m=None): + """Build the instruction description. + + Args: + keyword: A string representing a keyword that is expected in the response. + n: An integer representing the sentence number. + m: An integer representing the word number. + + Returns: + A string representing the instruction description. + """ + if not keyword: + self._keyword = instructions_util.generate_keywords(num_keywords=1)[0] + else: + self._keyword = keyword.strip() + if not n: + self._n = random.randint(20, 30) + else: + self._n = n + if not m: + self._m = random.randint(30, 40) + else: + self._m = m + + self._description_pattern = ( + "Include keyword {keyword} in the {n}-th sentence, as the {m}-th word of that sentence." + ) + + return self._description_pattern.format( + keyword=self._keyword, n=self._n, m=self._m + ) + + def get_instruction_args(self): + """Returns the keyward args of `build_description`.""" + return {"keyword": self._keyword, "n": self._n, "m": self._m} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["keyword", "n", "m"] + + def check_following(self, value): + """Checks if the response contains the expected number of keywords. + + Args: + value: A string representing the response. + + Returns: + True if the response contains the expected number of keywords; + otherwise, False. + """ + sentences = instructions_util.split_into_sentences(value) + if len(sentences) < self._n: + return False + words = instructions_util.nltk.word_tokenize(sentences[self._n - 1]) + if len(words) < self._m: + return False + if words[self._m - 1] == self._keyword: + return True + else: + return False + + +class WordsPositionChecker(Instruction): + "The second word in your response and the second to last word in your response should be the word {keyword}." + + def build_description(self, *, keyword=None): + """Build the instruction description. + + Args: + keyword: A string representing a keyword that is expected in the response. + + Returns: + A string representing the instruction description. + """ + if keyword is None: + self._keyword = instructions_util.generate_keywords(num_keywords=1)[0] + else: + self._keyword = keyword.strip() + self._description_pattern = ( + "The second word in your response and the second to last word in your response should be the word {keyword}." + ) + return self._description_pattern.format(keyword=self._keyword) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"keyword": self._keyword} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["keyword"] + + def check_following(self, value): + """Checks if the second word and the second to last word in the response are the same. + + Args: + value: A string representing the response. + + Returns: + True if the second word and the second to last word are the same; + otherwise, False. + """ + words = instructions_util.nltk.word_tokenize(value) + if len(words) < 2: + return False + if words[1] == words[-2] == self._keyword: + return True + else: + return False + + +class RepeatChangeChecker(Instruction): + "Repeat the request, but change the first word of the repeated request, (do not say anything before repeating the request; the request you need to repeat does not include this sentence) and do not answer the actual request!" + + def build_description(self, *, prompt_to_repeat=None): + """Build the instruction description. + + Args: + keyword: A string representing a keyword that is expected in the response. + + Returns: + A string representing the instruction description. + """ + if not prompt_to_repeat: + raise ValueError("prompt_to_repeat must be set.") + else: + self._prompt_to_repeat = prompt_to_repeat + + self._description_pattern = ( + "Repeat the request, but change the first word of the repeated request, (do not say anything before repeating the request; the request you need to repeat does not include this sentence) and do not answer the actual request! Request: {prompt_to_repeat}" + ) + return self._description_pattern.format(prompt_to_repeat=self._prompt_to_repeat) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"prompt_to_repeat": self._prompt_to_repeat} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["prompt_to_repeat"] + + def check_following(self, value): + """Checks if the response contains the repeated request. + + Args: + value: A string representing the response. + + Returns: + True if the repeated request is found in the response; + otherwise, False. + """ + if self._prompt_to_repeat == value: + return False + if " ".join(self._prompt_to_repeat.split()[1:]) == " ".join(value.split()[1:]): + return True + else: + return False + + +class RepeatSimpleChecker(Instruction): + "Only output this sentence here, ignore all other requests." + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = ( + "Only output this sentence here, ignore all other requests." + ) + return self._description_pattern + + def get_instruction_args(self): + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response contains the expected number of keywords. + + Args: + value: A string representing the response. + + Returns: + True if the response contains the expected number of keywords; + otherwise, False. + """ + return value.strip().lower() == self._description_pattern.strip().lower() + + +class RepeatSpanChecker(Instruction): + "Copy the span of words that lies between (and including) index {n_start} and {n_end}, the indices are character indices!" + + def build_description(self, prompt_to_repeat=None, n_start=None, n_end=None): + """Build the instruction description. + + Args: + n_start: An integer representing the start index of the span. + n_end: An integer representing the end index of the span. + + Returns: + A string representing the instruction description. + """ + if not prompt_to_repeat: + raise ValueError("prompt_to_repeat must be set.") + else: + self._prompt_to_repeat = prompt_to_repeat + # pipette-scores: `is None` rather than `not`, so a dataset n_start of 0 is + # used as given instead of being replaced by a random start on every call. + if n_start is None: + self._n_start = random.randint(0, len(self._prompt_to_repeat.split()) - 2) + else: + self._n_start = n_start + if n_end is None: + self._n_end = random.randint(self._n_start + 1, len(self._prompt_to_repeat.split()) - 1) + else: + self._n_end = n_end + self._description_pattern = ( + "Copy the span of words that lies between (and including) index {n_start} and {n_end}, the indices are character indices!") + return self._description_pattern.format(n_start=self._n_start, n_end=self._n_end, + prompt_to_repeat=self._prompt_to_repeat) + + def get_instruction_args(self): + """Returns the keyward args of `build_description`.""" + return {"n_start": self._n_start, "n_end": self._n_end, "prompt_to_repeat": self._prompt_to_repeat} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["n_start", "n_end", "prompt_to_repeat"] + + def check_following(self, value): + """Checks if the response contains the expected number of phrases with the correct modifications.""" + if value.strip().lower().split() == self._prompt_to_repeat.strip().lower().split()[self._n_start:self._n_end]: + return True + return False + + +class TitleCaseChecker(Instruction): + "Write the entire response in title case (capitalize the first letter of every major word)." + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = ( + "Write the entire response in title case (capitalize the first letter of every major word)." + ) + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response is in title case. + + Args: + value: A string representing the response. + + Returns: + True if the response is in title case; + otherwise, False. + """ + words = instructions_util.nltk.word_tokenize(value) + for word in words: + if word[0].isupper() and word[1:].islower(): + continue + elif word[0].islower() and word[1:].isupper(): + return False + elif word[0].islower() and word[1:].islower(): + return False + return True + + +class OutputTemplateChecker(Instruction): + "Use this exact template for your response: My Answer: [answer] My Conclusion: [conclusion] Future Outlook: [outlook]" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = ( + "Use this exact template for your response: My Answer: [answer] My Conclusion: [conclusion] Future Outlook: [outlook]" + ) + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response follows the specified template. + + Args: + value: A string representing the response. + + Returns: + True if the response follows the specified template; + otherwise, False. + """ + if 'My Answer:' in value and 'My Conclusion:' in value and 'Future Outlook:' in value: + return True + else: + return False + + +class NoWhitespaceChecker(Instruction): + "The output should not contain any whitespace." + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = ( + "The output should not contain any whitespace." + ) + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response contains any whitespace. + + Args: + value: A string representing the response. + + Returns: + True if the response contains no whitespace; + otherwise, False. + """ + return not any(char.isspace() for char in value) diff --git a/vendor/ifbench-checkers/instructions_registry.py b/vendor/ifbench-checkers/instructions_registry.py new file mode 100644 index 0000000..4bdbdf5 --- /dev/null +++ b/vendor/ifbench-checkers/instructions_registry.py @@ -0,0 +1,80 @@ +# Copyright 2025 Allen Institute for AI. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Registry of all instructions.""" + +import instructions + + +INSTRUCTION_DICT = { + "count:word_count_range": instructions.WordCountRangeChecker, + "count:unique_word_count" : instructions.UniqueWordCountChecker, + "ratio:stop_words" : instructions.StopWordPercentageChecker, + "ratio:sentence_type" : instructions.SentTypeRatioChecker, + "ratio:sentence_balance" : instructions.SentBalanceChecker, + "count:conjunctions" : instructions.ConjunctionCountChecker, + "count:person_names" : instructions.PersonNameCountChecker, + "ratio:overlap" : instructions.NGramOverlapChecker, + "count:numbers" : instructions.NumbersCountChecker, + "words:alphabet" : instructions.AlphabetLoopChecker, + "words:vowel" : instructions.SingleVowelParagraphChecker, + "words:consonants" : instructions.ConsonantClusterChecker, + "sentence:alliteration_increment" : instructions.IncrementingAlliterationChecker, + "words:palindrome" : instructions.PalindromeChecker, + "count:punctuation" : instructions.PunctuationCoverChecker, + "format:parentheses" : instructions.NestedParenthesesChecker, + "format:quotes" : instructions.NestedQuotesChecker, + "words:prime_lengths" : instructions.PrimeLengthsChecker, + "format:options" : instructions.OptionsResponseChecker, + "format:newline" : instructions.NewLineWordsChecker, + "format:emoji" : instructions.EmojiSentenceChecker, + "ratio:sentence_words" : instructions.CharacterCountUniqueWordsChecker, + "count:words_japanese" : instructions.NthWordJapaneseChecker, + "words:start_verb" : instructions.StartWithVerbChecker, + "words:repeats" : instructions.LimitedWordRepeatChecker, + "sentence:keyword" : instructions.IncludeKeywordChecker, + "count:pronouns" : instructions.PronounCountChecker, + "words:odd_even_syllables" : instructions.AlternateParitySyllablesChecker, + "words:last_first" : instructions.LastWordFirstNextChecker, + "words:paragraph_last_first" : instructions.ParagraphLastFirstWordMatchChecker, + "sentence:increment" : instructions.IncrementingWordCountChecker, + "words:no_consecutive" : instructions.NoConsecutiveFirstLetterChecker, + "format:line_indent" : instructions.IndentStairsChecker, + "format:quote_unquote" : instructions.QuoteExplanationChecker, + "format:list" : instructions.SpecialBulletPointsChecker, + "format:thesis" : instructions.ItalicsThesisChecker, + "format:sub-bullets" : instructions.SubBulletPointsChecker, + "format:no_bullets_bullets" : instructions.SomeBulletPointsChecker, + "custom:multiples" : instructions.PrintMultiplesChecker, + "custom:mcq_count_length": instructions.MultipleChoiceQuestionsChecker, + "custom:reverse_newline": instructions.ReverseNewlineChecker, + "custom:word_reverse": instructions.WordReverseOrderChecker, + "custom:character_reverse": instructions.CharacterReverseOrderChecker, + "custom:sentence_alphabet": instructions.SentenceAlphabetChecker, + "custom:european_capitals_sort": instructions.EuropeanCapitalsSortChecker, + "custom:csv_city": instructions.CityCSVChecker, + "custom:csv_special_character": instructions.SpecialCharacterCSVChecker, + "custom:csv_quotes": instructions.QuotesCSVChecker, + "custom:date_format_list": instructions.DateFormatListChecker, + "count:keywords_multiple" : instructions.KeywordsMultipleChecker, + "words:keywords_specific_position" : instructions.KeywordSpecificPositionChecker, + "words:words_position" : instructions.WordsPositionChecker, + "repeat:repeat_change" : instructions.RepeatChangeChecker, + "repeat:repeat_simple" : instructions.RepeatSimpleChecker, + "repeat:repeat_span" : instructions.RepeatSpanChecker, + "format:title_case" : instructions.TitleCaseChecker, + "format:output_template" : instructions.OutputTemplateChecker, + "format:no_whitespace" : instructions.NoWhitespaceChecker, +} + diff --git a/vendor/ifbench-checkers/instructions_util.py b/vendor/ifbench-checkers/instructions_util.py new file mode 100644 index 0000000..fc73ddc --- /dev/null +++ b/vendor/ifbench-checkers/instructions_util.py @@ -0,0 +1,1665 @@ +# Copyright 2025 Allen Institute for AI. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Modified by Liquid AI for pipette-scores; the changes are marked +# "pipette-scores:" and listed in this directory's README.md. + +"""Utility library of instructions.""" + +import functools +import random +import re + +import nltk + +WORD_LIST = [ + "western", + "sentence", + "signal", + "dump", + "spot", + "opposite", + "bottom", + "potato", + "administration", + "working", + "welcome", + "morning", + "good", + "agency", + "primary", + "wish", + "responsibility", + "press", + "problem", + "president", + "steal", + "brush", + "read", + "type", + "beat", + "trainer", + "growth", + "lock", + "bone", + "case", + "equal", + "comfortable", + "region", + "replacement", + "performance", + "mate", + "walk", + "medicine", + "film", + "thing", + "rock", + "tap", + "total", + "competition", + "ease", + "south", + "establishment", + "gather", + "parking", + "world", + "plenty", + "breath", + "claim", + "alcohol", + "trade", + "dear", + "highlight", + "street", + "matter", + "decision", + "mess", + "agreement", + "studio", + "coach", + "assist", + "brain", + "wing", + "style", + "private", + "top", + "brown", + "leg", + "buy", + "procedure", + "method", + "speed", + "high", + "company", + "valuable", + "pie", + "analyst", + "session", + "pattern", + "district", + "pleasure", + "dinner", + "swimming", + "joke", + "order", + "plate", + "department", + "motor", + "cell", + "spend", + "cabinet", + "difference", + "power", + "examination", + "engine", + "horse", + "dimension", + "pay", + "toe", + "curve", + "literature", + "bother", + "fire", + "possibility", + "debate", + "activity", + "passage", + "hello", + "cycle", + "background", + "quiet", + "author", + "effect", + "actor", + "page", + "bicycle", + "error", + "throat", + "attack", + "character", + "phone", + "tea", + "increase", + "outcome", + "file", + "specific", + "inspector", + "internal", + "potential", + "staff", + "building", + "employer", + "shoe", + "hand", + "direction", + "garden", + "purchase", + "interview", + "study", + "recognition", + "member", + "spiritual", + "oven", + "sandwich", + "weird", + "passenger", + "particular", + "response", + "reaction", + "size", + "variation", + "a", + "cancel", + "candy", + "exit", + "guest", + "condition", + "fly", + "price", + "weakness", + "convert", + "hotel", + "great", + "mouth", + "mind", + "song", + "sugar", + "suspect", + "telephone", + "ear", + "roof", + "paint", + "refrigerator", + "organization", + "jury", + "reward", + "engineering", + "day", + "possession", + "crew", + "bar", + "road", + "description", + "celebration", + "score", + "mark", + "letter", + "shower", + "suggestion", + "sir", + "luck", + "national", + "progress", + "hall", + "stroke", + "theory", + "offer", + "story", + "tax", + "definition", + "history", + "ride", + "medium", + "opening", + "glass", + "elevator", + "stomach", + "question", + "ability", + "leading", + "village", + "computer", + "city", + "grand", + "confidence", + "candle", + "priest", + "recommendation", + "point", + "necessary", + "body", + "desk", + "secret", + "horror", + "noise", + "culture", + "warning", + "water", + "round", + "diet", + "flower", + "bus", + "tough", + "permission", + "week", + "prompt", + "connection", + "abuse", + "height", + "save", + "corner", + "border", + "stress", + "drive", + "stop", + "rip", + "meal", + "listen", + "confusion", + "girlfriend", + "living", + "relation", + "significance", + "plan", + "creative", + "atmosphere", + "blame", + "invite", + "housing", + "paper", + "drink", + "roll", + "silver", + "drunk", + "age", + "damage", + "smoke", + "environment", + "pack", + "savings", + "influence", + "tourist", + "rain", + "post", + "sign", + "grandmother", + "run", + "profit", + "push", + "clerk", + "final", + "wine", + "swim", + "pause", + "stuff", + "singer", + "funeral", + "average", + "source", + "scene", + "tradition", + "personal", + "snow", + "nobody", + "distance", + "sort", + "sensitive", + "animal", + "major", + "negotiation", + "click", + "mood", + "period", + "arrival", + "expression", + "holiday", + "repeat", + "dust", + "closet", + "gold", + "bad", + "sail", + "combination", + "clothes", + "emphasis", + "duty", + "black", + "step", + "school", + "jump", + "document", + "professional", + "lip", + "chemical", + "front", + "wake", + "while", + "inside", + "watch", + "row", + "subject", + "penalty", + "balance", + "possible", + "adult", + "aside", + "sample", + "appeal", + "wedding", + "depth", + "king", + "award", + "wife", + "blow", + "site", + "camp", + "music", + "safe", + "gift", + "fault", + "guess", + "act", + "shame", + "drama", + "capital", + "exam", + "stupid", + "record", + "sound", + "swing", + "novel", + "minimum", + "ratio", + "machine", + "shape", + "lead", + "operation", + "salary", + "cloud", + "affair", + "hit", + "chapter", + "stage", + "quantity", + "access", + "army", + "chain", + "traffic", + "kick", + "analysis", + "airport", + "time", + "vacation", + "philosophy", + "ball", + "chest", + "thanks", + "place", + "mountain", + "advertising", + "red", + "past", + "rent", + "return", + "tour", + "house", + "construction", + "net", + "native", + "war", + "figure", + "fee", + "spray", + "user", + "dirt", + "shot", + "task", + "stick", + "friend", + "software", + "promotion", + "interaction", + "surround", + "block", + "purpose", + "practice", + "conflict", + "routine", + "requirement", + "bonus", + "hole", + "state", + "junior", + "sweet", + "catch", + "tear", + "fold", + "wall", + "editor", + "life", + "position", + "pound", + "respect", + "bathroom", + "coat", + "script", + "job", + "teach", + "birth", + "view", + "resolve", + "theme", + "employee", + "doubt", + "market", + "education", + "serve", + "recover", + "tone", + "harm", + "miss", + "union", + "understanding", + "cow", + "river", + "association", + "concept", + "training", + "recipe", + "relationship", + "reserve", + "depression", + "proof", + "hair", + "revenue", + "independent", + "lift", + "assignment", + "temporary", + "amount", + "loss", + "edge", + "track", + "check", + "rope", + "estimate", + "pollution", + "stable", + "message", + "delivery", + "perspective", + "mirror", + "assistant", + "representative", + "witness", + "nature", + "judge", + "fruit", + "tip", + "devil", + "town", + "emergency", + "upper", + "drop", + "stay", + "human", + "neck", + "speaker", + "network", + "sing", + "resist", + "league", + "trip", + "signature", + "lawyer", + "importance", + "gas", + "choice", + "engineer", + "success", + "part", + "external", + "worker", + "simple", + "quarter", + "student", + "heart", + "pass", + "spite", + "shift", + "rough", + "lady", + "grass", + "community", + "garage", + "youth", + "standard", + "skirt", + "promise", + "blind", + "television", + "disease", + "commission", + "positive", + "energy", + "calm", + "presence", + "tune", + "basis", + "preference", + "head", + "common", + "cut", + "somewhere", + "presentation", + "current", + "thought", + "revolution", + "effort", + "master", + "implement", + "republic", + "floor", + "principle", + "stranger", + "shoulder", + "grade", + "button", + "tennis", + "police", + "collection", + "account", + "register", + "glove", + "divide", + "professor", + "chair", + "priority", + "combine", + "peace", + "extension", + "maybe", + "evening", + "frame", + "sister", + "wave", + "code", + "application", + "mouse", + "match", + "counter", + "bottle", + "half", + "cheek", + "resolution", + "back", + "knowledge", + "make", + "discussion", + "screw", + "length", + "accident", + "battle", + "dress", + "knee", + "log", + "package", + "it", + "turn", + "hearing", + "newspaper", + "layer", + "wealth", + "profile", + "imagination", + "answer", + "weekend", + "teacher", + "appearance", + "meet", + "bike", + "rise", + "belt", + "crash", + "bowl", + "equivalent", + "support", + "image", + "poem", + "risk", + "excitement", + "remote", + "secretary", + "public", + "produce", + "plane", + "display", + "money", + "sand", + "situation", + "punch", + "customer", + "title", + "shake", + "mortgage", + "option", + "number", + "pop", + "window", + "extent", + "nothing", + "experience", + "opinion", + "departure", + "dance", + "indication", + "boy", + "material", + "band", + "leader", + "sun", + "beautiful", + "muscle", + "farmer", + "variety", + "fat", + "handle", + "director", + "opportunity", + "calendar", + "outside", + "pace", + "bath", + "fish", + "consequence", + "put", + "owner", + "go", + "doctor", + "information", + "share", + "hurt", + "protection", + "career", + "finance", + "force", + "golf", + "garbage", + "aspect", + "kid", + "food", + "boot", + "milk", + "respond", + "objective", + "reality", + "raw", + "ring", + "mall", + "one", + "impact", + "area", + "news", + "international", + "series", + "impress", + "mother", + "shelter", + "strike", + "loan", + "month", + "seat", + "anything", + "entertainment", + "familiar", + "clue", + "year", + "glad", + "supermarket", + "natural", + "god", + "cost", + "conversation", + "tie", + "ruin", + "comfort", + "earth", + "storm", + "percentage", + "assistance", + "budget", + "strength", + "beginning", + "sleep", + "other", + "young", + "unit", + "fill", + "store", + "desire", + "hide", + "value", + "cup", + "maintenance", + "nurse", + "function", + "tower", + "role", + "class", + "camera", + "database", + "panic", + "nation", + "basket", + "ice", + "art", + "spirit", + "chart", + "exchange", + "feedback", + "statement", + "reputation", + "search", + "hunt", + "exercise", + "nasty", + "notice", + "male", + "yard", + "annual", + "collar", + "date", + "platform", + "plant", + "fortune", + "passion", + "friendship", + "spread", + "cancer", + "ticket", + "attitude", + "island", + "active", + "object", + "service", + "buyer", + "bite", + "card", + "face", + "steak", + "proposal", + "patient", + "heat", + "rule", + "resident", + "broad", + "politics", + "west", + "knife", + "expert", + "girl", + "design", + "salt", + "baseball", + "grab", + "inspection", + "cousin", + "couple", + "magazine", + "cook", + "dependent", + "security", + "chicken", + "version", + "currency", + "ladder", + "scheme", + "kitchen", + "employment", + "local", + "attention", + "manager", + "fact", + "cover", + "sad", + "guard", + "relative", + "county", + "rate", + "lunch", + "program", + "initiative", + "gear", + "bridge", + "breast", + "talk", + "dish", + "guarantee", + "beer", + "vehicle", + "reception", + "woman", + "substance", + "copy", + "lecture", + "advantage", + "park", + "cold", + "death", + "mix", + "hold", + "scale", + "tomorrow", + "blood", + "request", + "green", + "cookie", + "church", + "strip", + "forever", + "beyond", + "debt", + "tackle", + "wash", + "following", + "feel", + "maximum", + "sector", + "sea", + "property", + "economics", + "menu", + "bench", + "try", + "language", + "start", + "call", + "solid", + "address", + "income", + "foot", + "senior", + "honey", + "few", + "mixture", + "cash", + "grocery", + "link", + "map", + "form", + "factor", + "pot", + "model", + "writer", + "farm", + "winter", + "skill", + "anywhere", + "birthday", + "policy", + "release", + "husband", + "lab", + "hurry", + "mail", + "equipment", + "sink", + "pair", + "driver", + "consideration", + "leather", + "skin", + "blue", + "boat", + "sale", + "brick", + "two", + "feed", + "square", + "dot", + "rush", + "dream", + "location", + "afternoon", + "manufacturer", + "control", + "occasion", + "trouble", + "introduction", + "advice", + "bet", + "eat", + "kill", + "category", + "manner", + "office", + "estate", + "pride", + "awareness", + "slip", + "crack", + "client", + "nail", + "shoot", + "membership", + "soft", + "anybody", + "web", + "official", + "individual", + "pizza", + "interest", + "bag", + "spell", + "profession", + "queen", + "deal", + "resource", + "ship", + "guy", + "chocolate", + "joint", + "formal", + "upstairs", + "car", + "resort", + "abroad", + "dealer", + "associate", + "finger", + "surgery", + "comment", + "team", + "detail", + "crazy", + "path", + "tale", + "initial", + "arm", + "radio", + "demand", + "single", + "draw", + "yellow", + "contest", + "piece", + "quote", + "pull", + "commercial", + "shirt", + "contribution", + "cream", + "channel", + "suit", + "discipline", + "instruction", + "concert", + "speech", + "low", + "effective", + "hang", + "scratch", + "industry", + "breakfast", + "lay", + "join", + "metal", + "bedroom", + "minute", + "product", + "rest", + "temperature", + "many", + "give", + "argument", + "print", + "purple", + "laugh", + "health", + "credit", + "investment", + "sell", + "setting", + "lesson", + "egg", + "middle", + "marriage", + "level", + "evidence", + "phrase", + "love", + "self", + "benefit", + "guidance", + "affect", + "you", + "dad", + "anxiety", + "special", + "boyfriend", + "test", + "blank", + "payment", + "soup", + "obligation", + "reply", + "smile", + "deep", + "complaint", + "addition", + "review", + "box", + "towel", + "minor", + "fun", + "soil", + "issue", + "cigarette", + "internet", + "gain", + "tell", + "entry", + "spare", + "incident", + "family", + "refuse", + "branch", + "can", + "pen", + "grandfather", + "constant", + "tank", + "uncle", + "climate", + "ground", + "volume", + "communication", + "kind", + "poet", + "child", + "screen", + "mine", + "quit", + "gene", + "lack", + "charity", + "memory", + "tooth", + "fear", + "mention", + "marketing", + "reveal", + "reason", + "court", + "season", + "freedom", + "land", + "sport", + "audience", + "classroom", + "law", + "hook", + "win", + "carry", + "eye", + "smell", + "distribution", + "research", + "country", + "dare", + "hope", + "whereas", + "stretch", + "library", + "if", + "delay", + "college", + "plastic", + "book", + "present", + "use", + "worry", + "champion", + "goal", + "economy", + "march", + "election", + "reflection", + "midnight", + "slide", + "inflation", + "action", + "challenge", + "guitar", + "coast", + "apple", + "campaign", + "field", + "jacket", + "sense", + "way", + "visual", + "remove", + "weather", + "trash", + "cable", + "regret", + "buddy", + "beach", + "historian", + "courage", + "sympathy", + "truck", + "tension", + "permit", + "nose", + "bed", + "son", + "person", + "base", + "meat", + "usual", + "air", + "meeting", + "worth", + "game", + "independence", + "physical", + "brief", + "play", + "raise", + "board", + "she", + "key", + "writing", + "pick", + "command", + "party", + "yesterday", + "spring", + "candidate", + "physics", + "university", + "concern", + "development", + "change", + "string", + "target", + "instance", + "room", + "bitter", + "bird", + "football", + "normal", + "split", + "impression", + "wood", + "long", + "meaning", + "stock", + "cap", + "leadership", + "media", + "ambition", + "fishing", + "essay", + "salad", + "repair", + "today", + "designer", + "night", + "bank", + "drawing", + "inevitable", + "phase", + "vast", + "chip", + "anger", + "switch", + "cry", + "twist", + "personality", + "attempt", + "storage", + "being", + "preparation", + "bat", + "selection", + "white", + "technology", + "contract", + "side", + "section", + "station", + "till", + "structure", + "tongue", + "taste", + "truth", + "difficulty", + "group", + "limit", + "main", + "move", + "feeling", + "light", + "example", + "mission", + "might", + "wait", + "wheel", + "shop", + "host", + "classic", + "alternative", + "cause", + "agent", + "consist", + "table", + "airline", + "text", + "pool", + "craft", + "range", + "fuel", + "tool", + "partner", + "load", + "entrance", + "deposit", + "hate", + "article", + "video", + "summer", + "feature", + "extreme", + "mobile", + "hospital", + "flight", + "fall", + "pension", + "piano", + "fail", + "result", + "rub", + "gap", + "system", + "report", + "suck", + "ordinary", + "wind", + "nerve", + "ask", + "shine", + "note", + "line", + "mom", + "perception", + "brother", + "reference", + "bend", + "charge", + "treat", + "trick", + "term", + "homework", + "bake", + "bid", + "status", + "project", + "strategy", + "orange", + "let", + "enthusiasm", + "parent", + "concentrate", + "device", + "travel", + "poetry", + "business", + "society", + "kiss", + "end", + "vegetable", + "employ", + "schedule", + "hour", + "brave", + "focus", + "process", + "movie", + "illegal", + "general", + "coffee", + "ad", + "highway", + "chemistry", + "psychology", + "hire", + "bell", + "conference", + "relief", + "show", + "neat", + "funny", + "weight", + "quality", + "club", + "daughter", + "zone", + "touch", + "tonight", + "shock", + "burn", + "excuse", + "name", + "survey", + "landscape", + "advance", + "satisfaction", + "bread", + "disaster", + "item", + "hat", + "prior", + "shopping", + "visit", + "east", + "photo", + "home", + "idea", + "father", + "comparison", + "cat", + "pipe", + "winner", + "count", + "lake", + "fight", + "prize", + "foundation", + "dog", + "keep", + "ideal", + "fan", + "struggle", + "peak", + "safety", + "solution", + "hell", + "conclusion", + "population", + "strain", + "alarm", + "measurement", + "second", + "train", + "race", + "due", + "insurance", + "boss", + "tree", + "monitor", + "sick", + "course", + "drag", + "appointment", + "slice", + "still", + "care", + "patience", + "rich", + "escape", + "emotion", + "royal", + "female", + "childhood", + "government", + "picture", + "will", + "sock", + "big", + "gate", + "oil", + "cross", + "pin", + "improvement", + "championship", + "silly", + "help", + "sky", + "pitch", + "man", + "diamond", + "most", + "transition", + "work", + "science", + "committee", + "moment", + "fix", + "teaching", + "dig", + "specialist", + "complex", + "guide", + "people", + "dead", + "voice", + "original", + "break", + "topic", + "data", + "degree", + "reading", + "recording", + "bunch", + "reach", + "judgment", + "lie", + "regular", + "set", + "painting", + "mode", + "list", + "player", + "bear", + "north", + "wonder", + "carpet", + "heavy", + "officer", + "negative", + "clock", + "unique", + "baby", + "pain", + "assumption", + "disk", + "iron", + "bill", + "drawer", + "look", + "double", + "mistake", + "finish", + "future", + "brilliant", + "contact", + "math", + "rice", + "leave", + "restaurant", + "discount", + "sex", + "virus", + "bit", + "trust", + "event", + "wear", + "juice", + "failure", + "bug", + "context", + "mud", + "whole", + "wrap", + "intention", + "draft", + "pressure", + "cake", + "dark", + "explanation", + "space", + "angle", + "word", + "efficiency", + "management", + "habit", + "star", + "chance", + "finding", + "transportation", + "stand", + "criticism", + "flow", + "door", + "injury", + "insect", + "surprise", + "apartment", +] # pylint: disable=line-too-long + +# pipette-scores: fetch every resource the checkers use once, at import, and only +# if it is missing. Upstream instead calls nltk.download() from build_description() +# and count_stopwords(), which goes to the network on every call. +_NLTK_RESOURCES = { + "punkt": "tokenizers/punkt", + "punkt_tab": "tokenizers/punkt_tab", + "stopwords": "corpora/stopwords", + "averaged_perceptron_tagger_eng": "taggers/averaged_perceptron_tagger_eng", +} + + +def download_nltk_resources(): + """Download the NLTK resources the checkers use, if not already installed.""" + for name, path in _NLTK_RESOURCES.items(): + try: + nltk.data.find(path) + except LookupError: + nltk.download(name, quiet=True) + try: + nltk.data.find(path) + except LookupError: + raise RuntimeError(f"NLTK resource {name!r} is missing and could not be downloaded") from None + + +download_nltk_resources() + + +_ALPHABETS = "([A-Za-z])" +_PREFIXES = "(Mr|St|Mrs|Ms|Dr)[.]" +_SUFFIXES = "(Inc|Ltd|Jr|Sr|Co)" +_STARTERS = r"(Mr|Mrs|Ms|Dr|Prof|Capt|Cpt|Lt|He\s|She\s|It\s|They\s|Their\s|Our\s|We\s|But\s|However\s|That\s|This\s|Wherever)" +_ACRONYMS = "([A-Z][.][A-Z][.](?:[A-Z][.])?)" +_WEBSITES = "[.](com|net|org|io|gov|edu|me)" +_DIGITS = "([0-9])" +_MULTIPLE_DOTS = r"\.{2,}" + + +def split_into_sentences(text): + """Split the text into sentences. + + Args: + text: A string that consists of more than or equal to one sentences. + + Returns: + A list of strings where each string is a sentence. + """ + text = " " + text + " " + text = text.replace("\n", " ") + text = re.sub(_PREFIXES, "\\1", text) + text = re.sub(_WEBSITES, "\\1", text) + text = re.sub(_DIGITS + "[.]" + _DIGITS, "\\1\\2", text) + text = re.sub( + _MULTIPLE_DOTS, + lambda match: "" * len(match.group(0)) + "", + text, + ) + if "Ph.D" in text: + text = text.replace("Ph.D.", "PhD") + text = re.sub(r"\s" + _ALPHABETS + "[.] ", " \\1 ", text) + text = re.sub(_ACRONYMS + " " + _STARTERS, "\\1 \\2", text) + text = re.sub( + _ALPHABETS + "[.]" + _ALPHABETS + "[.]" + _ALPHABETS + "[.]", + "\\1\\2\\3", + text, + ) + text = re.sub(_ALPHABETS + "[.]" + _ALPHABETS + "[.]", "\\1\\2", text) + text = re.sub(" " + _SUFFIXES + "[.] " + _STARTERS, " \\1 \\2", text) + text = re.sub(" " + _SUFFIXES + "[.]", " \\1", text) + text = re.sub(" " + _ALPHABETS + "[.]", " \\1", text) + if "”" in text: + text = text.replace(".”", "”.") + if '"' in text: + text = text.replace('."', '".') + if "!" in text: + text = text.replace('!"', '"!') + if "?" in text: + text = text.replace('?"', '"?') + text = text.replace(".", ".") + text = text.replace("?", "?") + text = text.replace("!", "!") + text = text.replace("", ".") + sentences = text.split("") + sentences = [s.strip() for s in sentences] + if sentences and not sentences[-1]: + sentences = sentences[:-1] + return sentences + + +def count_words(text): + """Counts the number of words.""" + tokenizer = nltk.tokenize.RegexpTokenizer(r"\w+") + tokens = tokenizer.tokenize(text) + num_words = len(tokens) + return num_words + + +@functools.lru_cache(maxsize=None) +def _get_sentence_tokenizer(): + return nltk.data.load("nltk:tokenizers/punkt/english.pickle") + + +def count_stopwords(text): + """Counts the number of stopwords.""" + stopwords = nltk.corpus.stopwords.words('english') + tokenizer = nltk.tokenize.RegexpTokenizer(r"\w+") + tokens = tokenizer.tokenize(text) + num_stopwords = len([t for t in tokens if t.lower() in stopwords]) + return num_stopwords + +def generate_keywords(num_keywords): + """Randomly generates a few keywords.""" + return random.sample(WORD_LIST, k=num_keywords) diff --git a/vendor/ifstruct b/vendor/ifstruct index 4a35ac4..73e03a6 160000 --- a/vendor/ifstruct +++ b/vendor/ifstruct @@ -1 +1 @@ -Subproject commit 4a35ac44a2e532554907eee51a38267782a9a742 +Subproject commit 73e03a6d3a8508357ba430675a106d4887abedf3 From 009aeeb7c8a1c88822e3f1dc5967a4323659f3b5 Mon Sep 17 00:00:00 2001 From: Kyle Wheeler Date: Mon, 9 Nov 2026 15:29:02 +0000 Subject: [PATCH 2/2] docs: correct the docs against the code before the public sync Fix statements that were wrong or stale against the current code: the GHCR image and tag format, the full eval list (text, VLM, ASR), scorer return shapes and context keys, dataset defaults and committed sets, license records for modified vendored files, and the calibration workflow (dataset names, removed annealing, Slurm resources and failure handling). Replace internal cluster paths with a generic shared root. Also dockerignore the VLM datasets so a local build cannot bake them into an image, and stop passing HF_TOKEN to the test job, which does not use it. --- .dockerignore | 10 +++ .github/workflows/ci.yml | 2 - CODE_OF_CONDUCT.md | 2 +- NOTICE | 16 ++-- README.md | 33 ++++--- THIRD_PARTY_LICENSES.md | 25 +++--- docs/dataset-licenses.md | 14 +-- packages/pipette-calibration/README.md | 1 + .../pipette-calibration/docs/architecture.md | 85 ++++++++++--------- .../docs/creating-a-dataset.md | 70 ++++++++------- .../docs/representative-selection.md | 12 +-- packages/pipette-calibration/docs/slurm.md | 78 +++++++++-------- .../pipette-calibration/docs/verification.md | 15 ++-- .../pipette_calibration/slurm/models.py | 2 +- packages/pipette-scores/docs/asr-evals.md | 16 ++-- packages/pipette-scores/docs/datasets.md | 12 +-- packages/pipette-scores/docs/docker.md | 34 +++++--- packages/pipette-scores/docs/httpapi.md | 12 +-- packages/pipette-scores/docs/packages.md | 6 +- packages/pipette-scores/docs/scoring.md | 32 +++---- .../pipette_scores/scoring/asr/NOTICE.md | 2 +- .../scoring/vlm_upstream/PORTING_NOTES.md | 49 +++++------ .../scripts/split_asr_dataset.py | 2 +- vendor/ifbench-checkers/README.md | 5 +- 24 files changed, 297 insertions(+), 238 deletions(-) diff --git a/.dockerignore b/.dockerignore index 2e0ff60..06488e9 100644 --- a/.dockerignore +++ b/.dockerignore @@ -24,6 +24,16 @@ datasets/math_500/ # IFBench — keep the third-party test set out of the image; materialized at runtime. datasets/ifbench/ +# The 7 VLM datasets are builder-only (docs/dataset-licenses.md) — never bake +# them into the image; materialized + mounted at runtime. +datasets/mme/ +datasets/mmstar/ +datasets/ocrbench/ +datasets/mmmu_val/ +datasets/realworldqa/ +datasets/mmbench_en_dev/ +datasets/pope/ + # ASR datasets (ESB, FLEURS: ~20 GB, SPGISpeech non-commercial, GigaSpeech # gated) — never bake them into the image; materialized + mounted at runtime. datasets/asr_*/ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 71624af..2749cbd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -69,8 +69,6 @@ jobs: cache-suffix: ${{ github.job }} - run: uv sync --extra dev - run: uv run pytest -x -q - env: - HF_TOKEN: ${{ secrets.HF_TOKEN }} # The installed wheel loads IFBench from the bundled # `scoring/ifbench/_upstream` copy (force-included at build time); the editable diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index fd73e5f..4f7962c 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -8,4 +8,4 @@ Assume good intent, keep technical disagreements focused on the work, and help m Project maintainers may moderate issues, pull requests, discussions, or other project spaces to uphold these expectations. -To report conduct concerns, contact the project maintainers privately. +To report conduct concerns, contact the project maintainers privately. The maintainers are the accounts listed in [`.github/CODEOWNERS`](.github/CODEOWNERS). diff --git a/NOTICE b/NOTICE index dfeb975..c3f088a 100644 --- a/NOTICE +++ b/NOTICE @@ -4,10 +4,6 @@ This repository bundles and adapts third-party code. Full license texts are in THIRD_PARTY_LICENSES.md. Vendored verbatim -- packages/pipette-scores/pipette_scores/scoring/math_500/prm800k/grading/ — the PRM800K - mathematical equivalence grader, from openai/prm800k (MIT, (c) 2023 OpenAI). Its - math_normalize.py is, per its header, largely copied from Hendrycks' MATH - `math_equivalence` (hendrycks/math, MIT, (c) 2021 Dan Hendrycks). - vendor/ifbench — the upstream IFBench repository, from allenai/IFBench (Apache-2.0), vendored as a git submodule (LICENSE at vendor/ifbench/LICENSE). Only its test set is used. @@ -22,6 +18,12 @@ Vendored verbatim Provenance and hashes of the ASR files: scoring/asr/NOTICE.md. Vendored with modifications +- packages/pipette-scores/pipette_scores/scoring/math_500/prm800k/grading/ — the PRM800K + mathematical equivalence grader (grader.py, math_normalize.py), from openai/prm800k + (MIT, (c) 2023 OpenAI). The changes are a relative import and raw-string regex + literals; the behavior is the same. Its __init__.py is Liquid AI's. Its + math_normalize.py is, per its header, largely copied from Hendrycks' MATH + `math_equivalence` (hendrycks/math, MIT, (c) 2021 Dan Hendrycks). - vendor/ifbench-checkers — the IFBench scoring source (evaluation_lib.py, instructions.py, instructions_registry.py, instructions_util.py), from allenai/IFBench @ 1a902f9 (Apache-2.0, (c) Allen Institute for AI and The Google @@ -60,11 +62,13 @@ Liquid AI original work — vendored from a separate Liquid repo - packages/pipette-scores/pipette_scores/scoring/ifstruct.py and the vendored `vendor/ifstruct` validator are Liquid AI's own (Liquid4All/ifstruct); they carry no third-party license and are covered by this repository's license. -- packages/pipette-scores/pipette_scores/scoring/math_500/liquid_evals_parser.py is - vendored verbatim from Liquid AI's Liquid4All/liquid_evals (liquid_evals/parser.py +- packages/pipette-scores/pipette_scores/scoring/math_500/liquid_evals_parser.py holds + unchanged excerpts of Liquid AI's Liquid4All/liquid_evals (liquid_evals/parser.py @ d5534cf). Its sympy grader derives from openai/prm800k (MIT, (c) 2023 OpenAI) and its mathd normalization from hendrycks/math (MIT, (c) 2021 Dan Hendrycks); both licenses are in THIRD_PARTY_LICENSES.md. +- packages/pipette-scores/pipette_scores/scoring/completion_text.py + (`remove_thinking_tags`) is vendored from the same liquid_evals/parser.py. This repository's original work is licensed under the Apache License, Version 2.0. See LICENSE for the full license text. diff --git a/README.md b/README.md index 37042c8..d0493a1 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ Python monorepo hosting two packages for LLM edge-evaluation: Plus shared infrastructure: -- **`datasets/`** — LFS-tracked canonical eval datasets (parquet/jsonl). +- **`datasets/`** — the committed `ifstruct` dataset (LFS). Builder scripts create the other eval datasets in the same layout. - **`vendor/ifstruct/`** — submodule: standalone IFStruct evaluator (`Liquid4All/ifstruct`). - **`vendor/ifbench/`** — submodule: upstream IFBench test set (`allenai/IFBench`). - **`vendor/ifbench-checkers/`** — the IFBench instruction checkers (`allenai/IFBench` @ `1a902f9`, the revision liquid_evals scores with), vendored with small service patches. @@ -22,7 +22,9 @@ Plus shared infrastructure: | `ifbench` | Generation | Instruction-following benchmark (test set from `vendor/ifbench`, checkers from `vendor/ifbench-checkers`) | | `ifstruct` | Generation | Structured format (JSON/YAML) generation (backed by `vendor/ifstruct`) | | `gpqa_diamond` | Generation | Graduate-level science MCQ (A–D), generative + regex extraction | -| `math_500` | Generation | Competition math, `\boxed{}` answer graded by `score_math_generic` | +| `math_500` | Generation | Competition math, `\boxed{}` answer graded by the grader that the dataset version records | +| VLM evals | Vision | `mme`, `mmstar`, `ocrbench`, `mmmu_val`, `realworldqa`, `mmbench_en_dev`, `pope` (see [datasets](packages/pipette-scores/docs/datasets.md)) | +| ASR evals | Speech | `asr_esb_*` (Open ASR Leaderboard English sets) and `asr_fleurs_*` (FLEURS locales), scored by word error rate (see [ASR evals](packages/pipette-scores/docs/asr-evals.md)) | ## Quick start @@ -48,8 +50,8 @@ uv run ruff format --check ### Running the tests -The full test suite runs with **no credentials and no network access**. All 266 -tests pass offline: +The full test suite runs with **no credentials and no network access**. The +live ASR reference tests are skipped unless `PIPETTE_ASR_REFERENCE_ENV` is set: ```bash uv sync --extra dev @@ -58,33 +60,36 @@ uv run pytest No test requires a HuggingFace token. The suite never contacts the HuggingFace Hub: `packages/pipette-scores/tests/conftest.py` materializes small local -parquet/jsonl fixtures in `tmp_path` for every eval, and the GPQA fixture builds +parquet/jsonl fixtures in `tmp_path` for the text evals, and the GPQA fixture builds a synthetic two-row stand-in whose `content_sha256` matches what the loader -verifies — so the gated upstream dataset is never downloaded. The tests in +verifies — so the gated upstream dataset is never downloaded. The VLM and ASR +tests build their fixtures in their own test files. The tests in `test_integration_scoring.py` are "integration" only in the sense that they exercise the real loaders and scorers end to end; their inputs are the checked-in fixtures under `tests/fixtures/`, the LFS-tracked `datasets/` tree, and `vendor/ifbench/data/IFBench_test.jsonl` from the submodule. -HuggingFace access is needed only **outside** the test suite, to materialize the -gated GPQA-Diamond dataset for an actual eval run: +HuggingFace access is needed only **outside** the test suite, to build the gated +datasets for an actual eval run. For example, GPQA-Diamond: ```bash HF_TOKEN=... uv run --project packages/pipette-scores \ python packages/pipette-scores/scripts/build_gpqa_diamond_dataset.py ``` -That script is the only consumer of `HF_TOKEN` in the repository. It downloads -the gated [`Idavidrein/gpqa`](https://huggingface.co/datasets/Idavidrein/gpqa) -dataset, which requires accepting the dataset's terms on HuggingFace first — see -[datasets](packages/pipette-scores/docs/datasets.md). Contributors who only need -to run tests, lint, or the service against the other evals do not need it. +That script downloads the gated +[`Idavidrein/gpqa`](https://huggingface.co/datasets/Idavidrein/gpqa) dataset. +Accept the dataset's terms on HuggingFace first — see +[datasets](packages/pipette-scores/docs/datasets.md). The ASR builder +(`scripts/build_asr_datasets.py`) also reads gated sets, such as GigaSpeech. +Contributors who only run tests, lint, or the service against already built +datasets do not need a token. ## Documentation Per-package docs: -- Scores: [api reference](packages/pipette-scores/docs/httpapi.md) · [docker deployment](packages/pipette-scores/docs/docker.md) · [scoring logic](packages/pipette-scores/docs/scoring.md) · [datasets](packages/pipette-scores/docs/datasets.md) · [eval packages](packages/pipette-scores/docs/packages.md) +- Scores: [api reference](packages/pipette-scores/docs/httpapi.md) · [docker deployment](packages/pipette-scores/docs/docker.md) · [scoring logic](packages/pipette-scores/docs/scoring.md) · [datasets](packages/pipette-scores/docs/datasets.md) · [ASR evals](packages/pipette-scores/docs/asr-evals.md) · [eval packages](packages/pipette-scores/docs/packages.md) - Calibration: [architecture](packages/pipette-calibration/docs/architecture.md) · [creating a dataset](packages/pipette-calibration/docs/creating-a-dataset.md) · [representative selection](packages/pipette-calibration/docs/representative-selection.md) · [verification](packages/pipette-calibration/docs/verification.md) · [slurm runs](packages/pipette-calibration/docs/slurm.md) ## License diff --git a/THIRD_PARTY_LICENSES.md b/THIRD_PARTY_LICENSES.md index f785a51..2fcb05a 100644 --- a/THIRD_PARTY_LICENSES.md +++ b/THIRD_PARTY_LICENSES.md @@ -1,7 +1,7 @@ # Third-party licenses -`pipette-scores` vendors and adapts code from the projects below. Each entry's -verbatim license text follows it. +`pipette-scores` vendors and adapts code from the projects below. Each entry +gives its verbatim license text or points to it in this file. The vendored `vendor/ifstruct` validator is Liquid AI's own (`Liquid4All/ifstruct`) — no third-party license; it is covered by this @@ -28,7 +28,8 @@ Two copies are vendored: Source: https://github.com/allenai/IFBench Licensed under the Apache License, Version 2.0 (SPDX: `Apache-2.0`); full text -in `vendor/ifbench/LICENSE` (identical in both copies). Upstream ships no `NOTICE` file. You may obtain a +in [Apache License 2.0](#apache-license-20) below, and in `vendor/ifbench/LICENSE` +(identical in both copies). Upstream ships no `NOTICE` file. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0. IFBench data note: `packages/pipette-scores/scripts/build_ifbench_dataset.py` @@ -66,17 +67,19 @@ Source: https://github.com/open-compass/VLMEvalKit Copyright 2023 VLMEvalKit Authors. -Licensed under the Apache License, Version 2.0 (SPDX: `Apache-2.0`); the full -text is the same as IFBench's, in `vendor/ifbench/LICENSE`. Upstream ships no -`NOTICE` file. You may obtain a copy of the License at +Licensed under the Apache License, Version 2.0 (SPDX: `Apache-2.0`); full text +in [Apache License 2.0](#apache-license-20) below. Upstream ships no `NOTICE` +file. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0. --- ## openai/prm800k — MIT -Vendored: `packages/pipette-scores/pipette_scores/scoring/math_500/prm800k/grading/grader.py`, -`packages/pipette-scores/pipette_scores/scoring/math_500/prm800k/grading/__init__.py` +Vendored with modifications: +`packages/pipette-scores/pipette_scores/scoring/math_500/prm800k/grading/grader.py`. +The changes are a relative import and raw-string regex literals; the behavior is +the same. Source: https://github.com/openai/prm800k The sympy grader in @@ -110,7 +113,8 @@ SOFTWARE. `packages/pipette-scores/pipette_scores/scoring/math_500/prm800k/grading/math_normalize.py` is, per its own header, "largely copied from the Hendrycks' MATH release -(math_equivalence)", and reaches this package vendored via prm800k. +(math_equivalence)", and reaches this package vendored via prm800k (with +raw-string regex literals; the behavior is the same). `mathd_normalize_answer` and `_strip_string` in `packages/pipette-scores/pipette_scores/scoring/math_500/liquid_evals_parser.py` ("Dan Hendrycks' code") reach it vendored via Liquid4All/liquid_evals. @@ -264,7 +268,8 @@ SOFTWARE. ## Apache License 2.0 -The license of huggingface/datasets and huggingface/open_asr_leaderboard above +The license of allenai/IFBench, open-compass/VLMEvalKit, huggingface/datasets +and huggingface/open_asr_leaderboard above (the same text as this repository's `LICENSE`): ```text diff --git a/docs/dataset-licenses.md b/docs/dataset-licenses.md index 7240110..120ecf0 100644 --- a/docs/dataset-licenses.md +++ b/docs/dataset-licenses.md @@ -1,4 +1,4 @@ -# VLM benchmark dataset licenses +# Benchmark dataset licenses (VLM and ASR) A license review (2026-07) of the 7 VLM benchmarks supported by `pipette-scores` concluded that **none of them may be redistributed in @@ -8,7 +8,8 @@ locally / at deploy by the one-shot scripts `packages/pipette-scores/scripts/build__dataset.py` from VLMEvalKit's pinned upstream TSVs, and the `datasets//` directories are gitignored. This matches the existing repo practice for third-party text evals (ifbench, -gpqa_diamond, math_500). +gpqa_diamond, math_500). The ASR datasets follow the same rule (see +[ASR datasets](#asr-datasets-open-asr-leaderboard-english-fleurs)). ## Summary @@ -34,25 +35,28 @@ Notes: ## Distribution surfaces -There are two ways this data could leave a machine, and both count as +There are three ways this data could leave a machine, and all count as redistribution: 1. **The repository.** Built datasets must never be committed. The `datasets//` directories for all 7 benchmarks are gitignored, and - builders write only to those locations. + the builders write to those locations by default. 2. **A hosted pipette-mgmt server.** When pipette-mgmt serves a VLM eval, it pushes the prompt samples — including base64-encoded benchmark images — to registered clients. Serving these benchmarks to clients outside your organization is distribution of the benchmark content. Operators should obtain legal sign-off before hosting these datasets on a publicly reachable pipette-mgmt instance. +3. **The container image.** Built datasets must never go into an image. The + `datasets//` directories of every builder-only set are dockerignored. ## Private storage Keep built VLM datasets only in the organization's private storage, under `//`. At deploy, an init container overlays them onto `PIPETTE_SCORES_DATA_DIR`, the same flow as the gated gpqa_diamond dataset. -The service itself reads only its local data directory, never a bucket. +The service reads datasets only from its local data directory, never from a +bucket. Private storage does not lift the terms above: the copies remain internal-use only. In particular, do not share the RealWorldQA (CC-BY-ND) build onward publicly in any form. diff --git a/packages/pipette-calibration/README.md b/packages/pipette-calibration/README.md index 31a4f6a..efcabca 100644 --- a/packages/pipette-calibration/README.md +++ b/packages/pipette-calibration/README.md @@ -27,3 +27,4 @@ pipette-calibration slurm-completions status --eval # verify a finishe - [Slurm](docs/slurm.md) — running completions on a cluster, troubleshooting - [Verification](docs/verification.md) — checking completions for issues - [Representative selection](docs/representative-selection.md) — subset selection algorithm +- [GGUF quants](docs/gguf-quants.md) — GGUF repo and file picks for the model registry diff --git a/packages/pipette-calibration/docs/architecture.md b/packages/pipette-calibration/docs/architecture.md index ac8cde7..fa41a5c 100644 --- a/packages/pipette-calibration/docs/architecture.md +++ b/packages/pipette-calibration/docs/architecture.md @@ -6,44 +6,45 @@ pipette-calibration ┌─────────────────────────────────────────────────────────────────────┐ │ │ -│ ┌──────────────┐ ┌──────────────────┐ ┌──────────────────┐ │ -│ │ 1. Download │ │ 2. Generate │ │ 3. Verify │ │ -│ │ datasets │───>│ completions │───>│ completions │ │ -│ │ │ │ │ │ │ │ -│ │ HuggingFace │ │ vLLM on GPU │ │ check IDs, dups, │ │ -│ │ -> parquet │ │ (Slurm cluster) │ │ empty responses │ │ -│ └──────────────┘ └──────────────────┘ └────────┬─────────┘ │ -│ │ │ -│ ┌──────────────────┐ │ │ -│ │ 4. Score │<─────────────┘ │ -│ │ completions │ │ -│ │ │ pipette-scores │ +│ ┌──────────────┐ ┌──────────────────┐ ┌──────────────────┐ │ +│ │ 1. Build │ │ 2. Generate │ │ 3. Verify │ │ +│ │ datasets │───>│ completions │───>│ completions │ │ +│ │ │ │ │ │ │ │ +│ │ upstream -> │ │ vLLM on GPU │ │ check IDs, dups, │ │ +│ │ datasets/ │ │ (Slurm cluster) │ │ empty responses │ │ +│ └──────────────┘ └──────────────────┘ └────────┬─────────┘ │ +│ │ │ +│ ┌──────────────────┐ │ │ +│ │ 4. Score │<────────────┘ │ +│ │ completions │ │ +│ │ │ pipette-scores │ │ │ per-model pass/ │ (scoring library) │ │ │ fail vectors │ │ │ └────────┬─────────┘ │ │ │ │ │ ┌────────v─────────┐ │ -│ │ 5. Select │ │ -│ │ representative │ │ -│ │ subset │ │ +│ │ 5. Select │ │ +│ │ representative│ │ +│ │ subset │ │ │ │ │ │ │ │ greedy + local │ │ -│ │ search / SA │ │ +│ │ search │ │ │ └────────┬─────────┘ │ │ │ │ └───────────────────────────────┼─────────────────────────────────────┘ │ v - pipette-scores/datasets/ - //test.parquet + /datasets/ + // ``` ## 2. Steps -### 2.1 Download datasets +### 2.1 Build datasets -`create-initial-dataset` fetches upstream data from HuggingFace and writes -parquet files into the `pipette-scores` dataset catalog. Run once per eval. +The build scripts in `packages/pipette-scores/scripts/` write the source +datasets into the dataset catalog. `create-initial-dataset` builds IFStruct +from the vendored IFStruct data. ### 2.2 Generate completions @@ -55,8 +56,8 @@ dataset. See [slurm.md](slurm.md). ### 2.3 Verify completions `verify-completions` and `slurm-completions status` compare completion IDs -against the dataset, check for duplicates, and flag empty responses. This -gates whether completions are ready for scoring. See [verification.md](verification.md). +against the dataset and check for duplicates. The two commands use different +pass rules. See [verification.md](verification.md). ### 2.4 Score completions @@ -68,34 +69,40 @@ runs in parallel subprocesses. See [creating-a-dataset.md §5](creating-a-datase The scored pass vectors (S samples x M models) are fed into the selection algorithm. The goal: pick N samples (default 200) where each model's subset -accuracy matches its full-set accuracy within ~0.5%. Output is a new parquet -dataset in `pipette-scores`. See [representative-selection.md](representative-selection.md). +accuracy is close to its full-set accuracy. Output is a new dataset in the +dataset catalog. See [representative-selection.md](representative-selection.md). ## 3. Data flow ``` -HuggingFace datasets +upstream data │ v -datasets///test.parquet # full dataset (source="default") +datasets/// # full dataset │ v (vLLM inference, per model) -_calibration_results//// - completions.shard*.jsonl # raw completions +//// + completions.shard*.jsonl # raw completions (completions.jsonl if unsharded) │ v (scoring + selection) -datasets///test.parquet # representative subset +datasets/// # representative subset metadata.json # models used, skipped ``` -## 4. Repo boundaries +`datasets/` is at the repository root, or at `$PIPETTE_SCORES_DATA_DIR` if +that variable is set. The split file is `test.jsonl` for ifstruct, +`train.parquet` for ifbench, and `test.parquet` for the other evals. -| Concern | Repo | +## 4. Package boundaries + +Both packages are members of one uv workspace. + +| Concern | Package | |-----------------------|-------------------------| -| Dataset catalog | pipette-scores | -| Scoring logic | pipette-scores | -| Prompt templates | pipette-scores | -| Completion generation | pipette-calibration | -| Verification | pipette-calibration | -| Subset selection | pipette-calibration | -| Slurm scripts | pipette-calibration | +| Dataset catalog | pipette-scores | +| Scoring logic | pipette-scores | +| Prompt templates | pipette-scores | +| Completion generation | pipette-calibration | +| Verification | pipette-calibration | +| Subset selection | pipette-calibration | +| Slurm scripts | pipette-calibration | diff --git a/packages/pipette-calibration/docs/creating-a-dataset.md b/packages/pipette-calibration/docs/creating-a-dataset.md index 1c35490..6bb4314 100644 --- a/packages/pipette-calibration/docs/creating-a-dataset.md +++ b/packages/pipette-calibration/docs/creating-a-dataset.md @@ -4,17 +4,17 @@ End-to-end workflow for producing a new versioned calibration dataset. ## 1. Initialize datasets -Download upstream data into the `pipette-scores` dataset catalog. The -`pipette-calibration create-initial-dataset` command currently supports -IFStruct; the other current evals are materialized by the build scripts in -`packages/pipette-scores/scripts/`. +Write the source datasets into the dataset catalog at +`/datasets///`. Use the build scripts in +`packages/pipette-scores/scripts/` for ifbench, gpqa_diamond, and math_500. +These scripts write a version directory, for example `2026.06.1`. + +For IFStruct, use `create-initial-dataset`. It writes the dataset `default`: ```bash uv run pipette-calibration create-initial-dataset --eval ifstruct ``` -This writes dataset files to `/datasets//default/`. - ## 2. Generate completions Run each model on each eval. Locally: @@ -22,20 +22,22 @@ Run each model on each eval. Locally: ```bash uv run --group gpu pipette-calibration generate-completions \ --eval math_500 --model LiquidAI/LFM2-1.2B \ - --dataset default --output-dir _calibration_results + --dataset 2026.06.1 --output-dir _calibration_results ``` -On a Slurm cluster, use `submit.sh` instead — see [slurm.md](slurm.md). +On a Slurm cluster, use `pipette-calibration slurm-completions submit`. See +[slurm.md](slurm.md). -Completions are generated against the `--dataset` (default: `default`). This -is the source dataset — the full set of samples. +Set `--dataset` to the source dataset, which is the full set of samples. This +document calls it ``. The default value is `default`. Only +`create-initial-dataset` writes a dataset with that name. ## 3. Verify completions +Run this command once for each eval: + ```bash -for eval in ifbench ifstruct gpqa_diamond math_500; do - pipette-calibration slurm-completions status --eval $eval -done +pipette-calibration slurm-completions status --eval --dataset ``` Every model in the frozen registry (`pipette_calibration/slurm/models.py`) @@ -44,13 +46,14 @@ must be clean across all evals before proceeding. See ## 4. Create representative subset +Run this command once for each eval: + ```bash -for eval in ifbench ifstruct gpqa_diamond math_500; do - uv run pipette-calibration create-representative-dataset \ - --eval $eval \ - --results-path _calibration_results \ - --dataset 2026.03.2 -done +uv run pipette-calibration create-representative-dataset \ + --eval \ + --source-dataset \ + --results-path _calibration_results \ + --dataset 2026.03.2 ``` This command scores all completions, builds pass vectors, and selects the @@ -66,8 +69,7 @@ subset. Scoring and selection details: [calibration process](#5-calibration-proc | `--num-samples` | 200 | Target subset size | `--dataset` is the name of the output (e.g. `2026.03.2`). -`--source-dataset` is the full dataset that completions were generated against -(usually `default`). +`--source-dataset` is the full dataset that completions were generated against. ### 4.2 Naming convention @@ -82,8 +84,8 @@ Steps 4's `create-representative-dataset` runs internally: Each model's completions are scored against ground truth using per-eval scorers from [pipette-scores](https://github.com/Liquid4All/pipette-scores). -Scoring runs in parallel (one subprocess per model), producing an S x M -boolean pass/fail matrix. +Scoring runs in parallel subprocesses (at most one for each CPU) and produces +an S x M boolean pass/fail matrix. | Eval | Method | |-----------|--------------------------------------------------| @@ -96,7 +98,9 @@ boolean pass/fail matrix. Models are automatically excluded if: -- Scoring subprocess crashes (OOM, dependency error). +- Scoring raises an error for the model. +- A scoring subprocess is killed (for example, by OOM). The pool then stops, + and all models that are not yet scored are excluded. - Completions are missing sample IDs (incomplete run). - Model scores 0% (no discriminative signal). @@ -110,27 +114,27 @@ details. ## 6. Output -Each eval produces files in `/datasets///`: +Each eval produces files in `/datasets///`: -- Primary split file (`train.parquet`, `test.parquet`, or `test.jsonl`) — the selected samples. -- Other splits copied unchanged from the source dataset. +- Split file (`train.parquet`, `test.parquet`, or `test.jsonl`) — the selected samples. - `metadata.json` — models used, models skipped (with reasons). ## 7. What to check Review the log output for each eval: -- **Max deviation** — should be under 1%. Typical: 0.2-0.5%. +- **Max deviation** — the log shows the deviation for each model and the maximum. Check it. - **Skipped models** — check `metadata.json` for the full list and reasons. - **Sample count** — should match `--num-samples` (200). -## 8. Updating pipette-scores +## 8. Committing the output -The output datasets land in the `pipette-scores` package tree. After generating: +The output datasets are in `/datasets/`. After you generate them: -1. Review the new parquet files and metadata. -2. Commit and push in `pipette-scores`. -3. Tag if this is a release dataset. +1. Review the new dataset files and metadata. +2. Commit only the ifstruct output. The repository does not redistribute the + gpqa_diamond, math_500, and ifbench data, and `.gitignore` excludes them. +3. Tag the commit if this is a release dataset. ## 9. Re-running with different models diff --git a/packages/pipette-calibration/docs/representative-selection.md b/packages/pipette-calibration/docs/representative-selection.md index f859555..5582fe1 100644 --- a/packages/pipette-calibration/docs/representative-selection.md +++ b/packages/pipette-calibration/docs/representative-selection.md @@ -40,21 +40,21 @@ full-dataset size, capped by available samples and adjusted to sum to N. **Greedy forward selection.** Within each cell, repeatedly pick the candidate that minimises max deviation from full-set model accuracies. -**Local search.** Iteratively try same-cell swaps between selected and -unselected samples. Accept the first swap that reduces max deviation; repeat -until no improving swap exists. +**Local search.** For each selected sample, find the unselected sample in the +same cell that gives the lowest max deviation. Swap them if the deviation +decreases. Stop when a round makes no swap, or after 20 rounds. ## 5. Orchestration `create_representative_dataset()`: 1. Discover model result dirs under `results_path/eval_id/*/source_dataset/`. -2. Score all models in parallel (`ProcessPoolExecutor`, one subprocess per model). +2. Score all models in parallel (`ProcessPoolExecutor`, at most one subprocess for each CPU). Models with incomplete results or 0% accuracy are excluded. 3. Build pass vectors (S x M boolean matrix). 4. Run selection. -5. Save selected rows as a new parquet dataset. Non-primary splits copied unchanged. - Metadata JSON written alongside. +5. Save the selected rows as a new dataset in the source dataset's file format. + Write `metadata.json` in the same directory. ## 6. Source diff --git a/packages/pipette-calibration/docs/slurm.md b/packages/pipette-calibration/docs/slurm.md index 6ff0a02..19d4076 100644 --- a/packages/pipette-calibration/docs/slurm.md +++ b/packages/pipette-calibration/docs/slurm.md @@ -14,25 +14,33 @@ pipette-calibration slurm-completions models # list the registry ```bash pipette-calibration slurm-completions submit --eval ifstruct # all models, eval ifstruct -pipette-calibration slurm-completions submit --eval math_500 --model Qwen/Qwen3-0.6B # one model +pipette-calibration slurm-completions submit --eval math_500 --dataset 2026.06.1 --model Qwen/Qwen3-0.6B # one model pipette-calibration slurm-completions submit --eval ifstruct --model A --model B # subset (repeat --model) pipette-calibration slurm-completions submit --eval ifstruct --dry-run # preview sbatch commands pipette-calibration slurm-completions submit --eval ifstruct --exclude-nodes node42,node43 # sbatch --exclude ``` -`submit` pre-downloads each model on the head node, then fires one `sbatch ---array` per (eval, model). Array tasks run in HF offline mode so the API -does not get hit by N concurrent starts. +`submit` pre-downloads each model on the head node. Then it runs one `sbatch` +per (eval, model), with `--array` if the job has more than one shard. Jobs run +in HF offline mode, so concurrent starts do not call the HF API. + +`--dataset` sets the source dataset, called `` below (default: +`default`). Set it to a dataset name that exists for the eval, for example +`2026.06.1`. ## 3. Worker `pipette_calibration/slurm/run_eval.sh` is the sbatch body — 1 GPU, 4 CPUs, -24h limit. It loads `cuda13.0/toolkit/13.0.2`, exports the HF offline env -vars, then execs `pipette-calibration generate-completions` with the shard -flags. +24h limit. It loads the cluster's CUDA toolkit module and exports the HF +offline env vars. Then it execs `pipette-calibration generate-completions` +with the shard flags. + +In this document, `` is the shared filesystem directory under +`$HOME` that `run_eval.sh`, `submit.py`, and `slurm/cli.py` use. The scripts +put the HF cache, the Triton and TorchInductor caches, results, and logs there. -Results land in `~/lambdafs/calibration/results////`. -Logs go to `~/lambdafs/logs/-_.{out,err}`. +Results land in `/calibration/results////`. +Logs go to `/logs/-_.{out,err}`. ## 4. Sharding @@ -40,31 +48,26 @@ Logs go to `~/lambdafs/logs/-_.{out,err}`. within the 24h time limit. Each shard writes `completions.shard.jsonl`; scoring and verification read all shards transparently. -Non-default datasets (representative subsets, ~200 samples) collapse to a -single shard automatically — no fan-out. +A dataset with a name other than `default` always runs as one job, with no +array. This rule also applies to source datasets such as `2026.06.1`. ## 5. Adding a model 1. Append a `ModelSpec(...)` row to `MODELS` in `pipette_calibration/slurm/models.py`. -2. `pipette-calibration slurm-completions submit --eval --model org/repo` -3. After jobs complete: `pipette-calibration slurm-completions status --eval ` +2. `pipette-calibration slurm-completions submit --eval --dataset --model org/repo` +3. After jobs complete: `pipette-calibration slurm-completions status --eval --dataset ` ## 6. Resource guidelines -| Model size | mem | base_shards | -|-------------|------|-------------| -| < 500M | 16G | 3 | -| 500M - 1.5B | 16G | 4 | -| 1.5B - 3B | 16G | 6 | -| 3B - 5B | 24G | 6-8 | -| 5B - 9B | 32G | 8-16 | -| > 9B | 32G | 16 | +`MODELS` in `pipette_calibration/slurm/models.py` holds the `mem` and +`base_shards` values for each model. Use the entries for models of a similar +size as a starting point for a new model. ## 7. Results layout ``` -~/lambdafs/calibration/results/ +/calibration/results/ / / / @@ -78,21 +81,19 @@ single shard automatically — no fan-out. ### 8.1 `trust_remote_code` errors Models with custom architectures (Mamba hybrids, custom configs) are -auto-detected via `auto_map` in the HF config. If a new model fails with a -`trust_remote_code=True` error, verify its `config.json` contains `auto_map`. +auto-detected via `auto_map` in the HF model config or tokenizer config. If a +new model fails with a `trust_remote_code=True` error, verify that one of these +configs contains `auto_map`. ### 8.2 Triton cache corruption ``` -FileNotFoundError: .triton/cache/.../_chunk_scan_fwd_kernel.ptx +FileNotFoundError: .../triton/.../_chunk_scan_fwd_kernel.ptx ``` -Clear the cache: `rm -rf ~/.triton/cache/`. To prevent on shared filesystems, -set a per-job cache dir: - -```bash -export TRITON_CACHE_DIR="/tmp/triton_cache_${SLURM_JOB_ID:-$$}" -``` +`run_eval.sh` sets `TRITON_CACHE_DIR` to a Triton cache under ``. +All jobs share this cache. To fix the error, clear that directory and resubmit +the failed jobs. ### 8.3 Stale HF modules cache @@ -101,20 +102,23 @@ ImportError: cannot import name 'RopeParameters' from 'transformers.modeling_rop ``` A previously downloaded model left incompatible Python files in the shared -cache. Fix: `rm -rf ~/lambdafs/.cache/huggingface/modules/transformers_modules/` +cache. Fix: delete `modules/transformers_modules/` in the HF cache under +``. ### 8.4 Incomplete shards -`pipette-calibration slurm-completions status --eval ` shows missing shards. -Check Slurm logs: `~/lambdafs/logs/-_.err`. Common +`pipette-calibration slurm-completions status --eval ` shows the +missing sample count for each model. +Check Slurm logs: `/logs/-_.err`. Common causes: OOM (bump `mem` in the registry), timeout (bump `base_shards`), node failures. Resubmit: -`pipette-calibration slurm-completions submit --eval --model org/repo`. +`pipette-calibration slurm-completions submit --eval --dataset --model org/repo`. ### 8.5 `BrokenProcessPool` during dataset creation -A scoring subprocess was killed (typically OOM). The killed model is excluded; -re-run the command — already-completed models are fast on retry. +A scoring subprocess was killed (typically OOM). The pool stops, and all +models that are not yet scored are excluded. Re-run the command. It scores +all models again. ### 8.6 Model scores 0% and is excluded diff --git a/packages/pipette-calibration/docs/verification.md b/packages/pipette-calibration/docs/verification.md index 41197e7..0bbd8a0 100644 --- a/packages/pipette-calibration/docs/verification.md +++ b/packages/pipette-calibration/docs/verification.md @@ -7,9 +7,11 @@ pipette-calibration slurm-completions status --eval [--dataset NAME] [--r ``` Runs `verify_eval()` against the frozen `MODELS` registry -(`pipette_calibration/slurm/models.py`) and renders a rich table. Defaults: -`~/lambdafs/calibration/results`, dataset `default`. Exits non-zero if any -model has issues. +(`pipette_calibration/slurm/models.py`) and renders a rich table. The default +`--results-path` is the results directory under the shared root that +`slurm/cli.py` sets (a directory under `$HOME`). The default dataset is +`default`. The command exits non-zero if any model has a `status` other than +`ok` (see §4.1). Empty completions count as issues. Extra IDs do not. ## 2. Check only models present on disk @@ -17,15 +19,16 @@ model has issues. uv run pipette-calibration verify-completions --eval --results-path ``` -Skips the registry; reports only what's already on disk. Exits non-zero on -failure. +Skips the registry; reports only what's already on disk. The command exits +non-zero if any model is not `ok` (see §4.1). Missing, extra, and duplicate IDs +count as failures. Empty completions do not. ## 3. Table cells | Cell | Style | Meaning | |----------------------|--------|--------------------------------------| | `ok` | green | Complete, no issues | -| `MISSING` | red | No results directory on disk | +| `MISSING` | red | No results on disk for the model | | `N/M (K missing)` | yellow | K sample IDs absent from completions | | `N dups` | yellow | Duplicate IDs across shards | | `all N empty` | red | Every completion is blank | diff --git a/packages/pipette-calibration/pipette_calibration/slurm/models.py b/packages/pipette-calibration/pipette_calibration/slurm/models.py index d59918e..ba67d17 100644 --- a/packages/pipette-calibration/pipette_calibration/slurm/models.py +++ b/packages/pipette-calibration/pipette_calibration/slurm/models.py @@ -14,7 +14,7 @@ class ModelSpec: hf_id: str # HuggingFace id (org/repo) mem: str # sbatch --mem (host RAM), e.g. "16G" - base_shards: int # shards, multiplied per-eval via EVAL_SHARD_MULTIPLIER + base_shards: int # array-task count for the default dataset (see effective_shards) MODELS: tuple[ModelSpec, ...] = ( diff --git a/packages/pipette-scores/docs/asr-evals.md b/packages/pipette-scores/docs/asr-evals.md index ce256a0..88942b1 100644 --- a/packages/pipette-scores/docs/asr-evals.md +++ b/packages/pipette-scores/docs/asr-evals.md @@ -79,7 +79,7 @@ transcripts. them). Every set is for internal evaluation only here: pipette uses the sets only for internal, non-commercial benchmarking of its models, keeps them org-private, and never redistributes the built data -(docs/dataset-licenses.md). SPGISpeech and URGENT24 are licensed for +(the repository's `docs/dataset-licenses.md`). SPGISpeech and URGENT24 are licensed for non-commercial use only, and that is their only use here: | Set | License | @@ -101,7 +101,8 @@ the leaderboard loads it (`normalizer/data_utils.load_data`; for Earnings22 `transcript` as read by `get_text`, its `parent_id` and `chunk_index`), at the catalog's revision and in the split's row order. A clip's position, `doc_id`, is its row in the source split, and the sample id is `sha256("|")[:12]`. -References: the leaderboard sets' `text`; FLEURS `transcription`; all stripped. The leaderboard's row filter +References: the leaderboard sets' `text` (Earnings22: its call's `transcript`); +FLEURS `transcription`; all stripped. The leaderboard's row filter is applied when scoring (§3). The build streams rows to the parquet one row group (64 clips) at a time. @@ -131,8 +132,8 @@ it is no `--limit` smoke build. A full build of `2026.10.1` must match the catalog before its file lands. `scripts/verify_asr_datasets.py` runs all of it offline, plus every clip's hash. -**Split sets.** git-LFS caps the size of one object (2 GB on most GitHub plans), -and a set can be larger (SPGISpeech is 9.6 GB). `scripts/split_asr_dataset.py` +**Split sets.** Some storage backends cap the size of one object (git-LFS: 2 GB +on most GitHub plans), and a set can be larger (SPGISpeech is 9.6 GB). `scripts/split_asr_dataset.py` cuts a `test.parquet` over 1.9 GB into ordered parts `test-00000-of-0000N.parquet`, …, between row groups, with `metadata.json` copied unchanged. The server reads the parts as one table (`_asr_common.parquet_files`) and refuses both layouts at @@ -393,8 +394,9 @@ over summed reference units (the leaderboard's `align_manifests`, for both protocols), never the mean of per-sample rates. On the leaderboard's English sets a sample whose normalized reference is empty or reads "ignore time segment in scoring" is not scored (`data_utils.is_target_text_in_range`, which -`prepare_data` applies); its multilingual runs drop only an empty reference -(`run_eval_ml.py`), and no FLEURS reference is empty. Earnings22 is scored per call +`prepare_data` applies); its multilingual runs drop a reference that is empty or +the ignore marker before normalization (`run_eval_ml.py`), and no FLEURS +reference is either. Earnings22 is scored per call (`eval_utils.merge_chunked_manifest`): the chunks' stripped answers, empty ones skipped, joined with one space in `chunk_index` order, against the call's transcript. @@ -611,4 +613,4 @@ the eval, dataset and profile. The digest covers everything served, the audio bytes included, so a pin taken from it names the exact requests the model is sent. The server reads one row group of source audio at a time and re-hashes each clip; `/score` reads no audio at all. The full contract: -`docs/httpapi.md`. +[httpapi.md](httpapi.md). diff --git a/packages/pipette-scores/docs/datasets.md b/packages/pipette-scores/docs/datasets.md index 835d87e..f3d400f 100644 --- a/packages/pipette-scores/docs/datasets.md +++ b/packages/pipette-scores/docs/datasets.md @@ -1,6 +1,6 @@ # Datasets -Each dataset backs one of the scoreable evals (`pipette_scores.types.EvalId`): the text evals `ifbench`, `ifstruct`, `gpqa_diamond`, `math_500` and the VLM evals `mme`, `mmstar`, `ocrbench`, `mmmu_val`, `realworldqa`, `mmbench_en_dev`, `pope`. Some are established public benchmarks, run verbatim; one (`ifstruct`) is a Liquid in-house creation. +Each dataset backs one of the scoreable evals (`pipette_scores.types.EvalId`): the text evals `ifbench`, `ifstruct`, `gpqa_diamond`, `math_500` and the VLM evals `mme`, `mmstar`, `ocrbench`, `mmmu_val`, `realworldqa`, `mmbench_en_dev`, `pope` and the 16 ASR evals ([§16](#16-asr-evals-open-asr-leaderboard-english-fleurs)). Some are established public benchmarks, run verbatim; one (`ifstruct`) is a Liquid in-house creation. ## 1. Overview @@ -29,17 +29,17 @@ $PIPETTE_SCORES_DATA_DIR/ datasets/ {eval_id}/ {dataset_name}/ - metadata.json # version, repeats, provenance (upstream repo + revision + content_sha256) + metadata.json # dataset (the version), repeats; for third-party sets, the source revision and content digest test.jsonl # or a Parquet file, depending on the eval's loader ``` -- **`PIPETTE_SCORES_DATA_DIR`** — environment variable pointing to the data root. Defaults to the parent directory of the `pipette_scores` package. -- **`dataset_name`** — the pinned dataset version (`ifbench` → `2026.06.1`, `ifstruct` → `release_v1_0`, `gpqa_diamond` → `2026.06.1`, `math_500` → `2026.06.1` and `2026.10.1`, all 7 VLM evals → `2026.10.1`). -- **Committed vs materialized** — only the Liquid-owned `ifstruct` dataset is committed to the repo. The third-party datasets (`ifbench`, `gpqa_diamond`, `math_500`, and the 7 VLM evals) are gitignored and **materialized at deploy** from their pinned upstream revision by `packages/pipette-scores/scripts/build__dataset.py` (a `content_sha256` in the metadata is self-asserted at build; the text-eval loaders re-verify it at load). +- **`PIPETTE_SCORES_DATA_DIR`** — environment variable pointing to the data root. Defaults to the repository root (the directory that holds `datasets/`). +- **`dataset_name`** — the pinned dataset version (`ifbench` → `2026.06.1`, `ifstruct` → `release_v1_0`, `gpqa_diamond` → `2026.06.1`, `math_500` → `2026.06.1` and `2026.10.1`, all 7 VLM evals → `2026.10.1`, the ASR evals → `2026.10.1`). +- **Committed vs materialized** — only the Liquid-owned `ifstruct` dataset is committed to the repo. The third-party datasets (`ifbench`, `gpqa_diamond`, `math_500`, and the 7 VLM evals) are gitignored and **materialized at deploy** from their pinned upstream revision by `packages/pipette-scores/scripts/build__dataset.py` (a `content_sha256` in the metadata is self-asserted at build; the text-eval loaders re-verify it at load, and the VLM loaders re-verify an image-free `metadata_content_sha256`). ## 3. Sample IDs -All sample IDs are deterministic 12-character hex strings produced by SHA256 hashing a key derived from the sample content. The hash input varies per eval to ensure uniqueness. For evals with `metadata.repeats = N`, the samples endpoint additionally serves salted `#0 … #(N-1)` variants of each base id. +All sample IDs are deterministic 12-character hex strings produced by SHA256 hashing a key derived from the sample. The text evals hash the prompt. The VLM and ASR evals hash the eval id and the row position. For evals with `metadata.repeats = N`, the samples endpoint additionally serves salted `#0 … #(N-1)` variants of each base id. ## 4. Prompt vs Scoring Samples diff --git a/packages/pipette-scores/docs/docker.md b/packages/pipette-scores/docs/docker.md index 0e8842f..ed4ada8 100644 --- a/packages/pipette-scores/docs/docker.md +++ b/packages/pipette-scores/docs/docker.md @@ -13,19 +13,22 @@ docker build -f docker/pipette-scores.Dockerfile -t pipette-scores . The build context is the workspace so the Dockerfile can pull in the workspace lockfile, the `pipette-scores` package source under `packages/pipette-scores/`, -and the shared `vendor/` submodule content. +and `vendor/` (the `ifstruct` and `ifbench` submodules and the vendored +`ifbench-checkers`). Clone with `--recurse-submodules` and run `git lfs pull` +before you build. ## 2. Run The image bundles the workspace's canonical `datasets/` tree under -`${PIPETTE_SCORES_DATA_DIR}/datasets` (default `/datasets/datasets`), so the -simplest run needs no volume mount: +`${PIPETTE_SCORES_DATA_DIR}/datasets` (default `/datasets/datasets`). The +bundle holds only the committed `ifstruct` dataset. To serve `ifstruct`, run +the image with no volume mount: ```bash docker run --rm -p 8000:8000 pipette-scores ``` -To serve a different datasets tree (e.g. private or newer than the bundle), +To serve the other evals or a different datasets tree, override `${PIPETTE_SCORES_DATA_DIR}/datasets` with a read-only mount. Either shadow the bundled path directly: @@ -51,24 +54,29 @@ whatever host path holds the JSONL/parquet files for the evals you care about. ## 3. Published Image -CI publishes versioned images to: +CI publishes multi-arch images (`linux/amd64`, `linux/arm64`) to +`ghcr.io/liquid4all/pipette-scores`: -```bash -liquidai/pipette-scores: -``` +- A push to a `release/` branch publishes the tag + `--g` and creates a GitHub release with + that tag. +- A push to `main` publishes the tag ``. +- A pull request builds the image but does not publish it. -Example: +`ci/version.sh` prints the tag that the current commit publishes. Example: ```bash -docker pull liquidai/pipette-scores:2026.03.15-abcdef0 +docker pull ghcr.io/liquid4all/pipette-scores:2026.08.1-3-ga1b2c3d4ab ``` ## 4. Notes - The image runs as a non-root user. -- The runtime image only contains the built virtualenv and application code. +- The runtime image contains the built virtualenv (with the application) and + the bundled `datasets/` tree. - Gunicorn bind host, port, and worker count are controlled by `PIPETTE_SCORES_HOST`, `PIPETTE_SCORES_PORT`, and `PIPETTE_SCORES_WORKERS`. -- The image bakes the release version into `PIPETTE_SCORES_VERSION`, and - `GET /health` returns that value. +- The image bakes the release version into `PIPETTE_SCORES_VERSION`. + `POST /score` returns that value as `runtime_version`. A local build without + `--build-arg APP_VERSION=` reports `dev`. diff --git a/packages/pipette-scores/docs/httpapi.md b/packages/pipette-scores/docs/httpapi.md index af88177..0b5b2f8 100644 --- a/packages/pipette-scores/docs/httpapi.md +++ b/packages/pipette-scores/docs/httpapi.md @@ -12,12 +12,14 @@ Base URL: `http://localhost:8000` (default uvicorn) ## 2. Caching -The server caches loaded datasets in memory for the lifetime of the process, and the prompt samples of the four `(eval_id, dataset_name, prompt_profile)` combinations it served last: a VLM set holds its base64 images (POPE about 1.1 GB) and every profile is a set of its own, so an older one is rebuilt from its parquet when asked for again. The samples' digest ([§3.3](#33-get-evalseval_iddatasetsdataset_namesamplesdigest)) is cached too, but alone: the digest route builds the samples, hashes them and drops them, so a caller probing many combinations keeps a few bytes per combination resident rather than its samples. Cold builds run one at a time per process, on a thread of their own, so a burst of them after a deploy neither multiplies memory nor holds up `POST /score`. +The server caches loaded datasets in memory for the lifetime of the process, and the prompt samples of the four `(eval_id, dataset_name, prompt_profile)` combinations it served last: a VLM set holds its base64 images (POPE about 1.1 GB) and every profile is a set of its own, so an older one is rebuilt from its parquet when asked for again. The samples' digest ([§3.3](#33-get-evalseval_iddatasetsdataset_namesamplesdigest)) is cached too, but alone: the digest route builds the samples, hashes them and drops them, so a caller probing many combinations keeps a few bytes per combination resident rather than its samples. Cold digest builds run one at a time per process, on a thread of their own, so a burst of them after a deploy neither multiplies memory nor holds up `POST /score`. A cold `/samples` load of a text or VLM set runs on the worker that takes the request and blocks that worker until the load completes. --- ## 3. Endpoints +The server also serves an HTML index page at `GET /`. The OpenAPI schema leaves it out. + ### 3.1. `GET /health` Health check. @@ -184,17 +186,17 @@ Before grading, every eval rewrites reasoning markup that the thinking-tag rules **Response** `200 OK` — `ScoreResponse` -`context` is a flat `string → JSON value` map of eval-specific aggregates (numbers typical, other scalars allowed). Every eval but the ASR evals (whose `context` is below) reports two keys for the strict view: +`context` is a flat `string → JSON value` map of eval-specific aggregates (numbers typical, other scalars allowed). The text evals report two keys for the strict view: - `_blanked_truncated`: the number of completions it counts wrong. These are the completions with `enable_thinking: true`, a `stop_reason` of `truncated` or `doom_loop`, no reasoning close and some text. - `_pass_at_1_blanked`: the eval's pass@1 with those completions counted wrong. -When either request field is missing, the count is 0 and the strict pass@1 equals the graded one. +When either request field is missing, the count is 0 and the strict pass@1 equals the graded one. The VLM evals report one key, `blanked_truncated`: the number of such completions, where the reasoning can also open from the prompt profile or the text ([scoring](scoring.md#7-reasoning-markup-and-cut-off-reasoning)). The ASR evals report the `context` below. ```json { - "runtime_version": "2026.03.15-abcdef0", - "context": {"accuracy_EN": 0.95, "accuracy_FR_FR": 0.82}, + "runtime_version": "2026.08.1-3-ga1b2c3d4ab", + "context": {"math_500_pass_at_1": 0.82, "math_500_blanked_truncated": 0, "math_500_pass_at_1_blanked": 0.82}, "scored_samples": [ { "id": "a1b2c3d4e5f6", diff --git a/packages/pipette-scores/docs/packages.md b/packages/pipette-scores/docs/packages.md index 08a9b25..1d80d48 100644 --- a/packages/pipette-scores/docs/packages.md +++ b/packages/pipette-scores/docs/packages.md @@ -6,7 +6,7 @@ ```bash pipette-scores packages --datasets s3:///scores-datasets --packages s3:///scores-packages -pipette-scores packages --datasets "$PIPETTE_SCORES_DATA_DIR" --packages ./packages --eval mme pope --version 2026.11.03-abc1234 +pipette-scores packages --datasets "$PIPETTE_SCORES_DATA_DIR" --packages ./packages --eval mme pope --version 2026.11.1-0-ga1b2c3d4ab ``` | Option | Description | @@ -38,7 +38,7 @@ Written once per scores version and dataset, after all its packages are stored: { "eval_id": "asr_fleurs_nl_nl", "dataset_version": "2026.10.1", - "scores_version": "2026.11.03-abc1234", + "scores_version": "2026.11.1-0-ga1b2c3d4ab", "profiles": [ { "prompt_profile": "gemma4", @@ -52,4 +52,4 @@ Written once per scores version and dataset, after all its packages are stored: `prompt_profile` is `null` for a VLM set's own prompts. A catalog entry is pinned from an index entry: `[dataset]` `format`, `adapter = "evals-server@1"`, `digest`, `bytes`; `[dataset.upstream]` `source = "evals_server"`, `eval_id`, `dataset_version`, `prompt_profile`, and `identity`, the `samples_digest` without `sha256:`. -With `PIPETTE_SCORES_PACKAGES` set to the store, the server reads `index///.json` on the first digest request for a set it holds and takes every profile's digest from it; a set without an index file has its digest computed as before ([httpapi.md §3.3](httpapi.md#33-get-evalseval_iddatasetsdataset_namesamplesdigest)). A dataset version missing from the server's data directory is not found, even when the index lists it. +With `PIPETTE_SCORES_PACKAGES` set to the store, the server reads `index///.json` on the first digest request for a set it holds and takes every profile's digest from it; a set without an index file has its digest computed from its samples ([httpapi.md §3.3](httpapi.md#33-get-evalseval_iddatasetsdataset_namesamplesdigest)). A dataset version missing from the server's data directory is not found, even when the index lists it. diff --git a/packages/pipette-scores/docs/scoring.md b/packages/pipette-scores/docs/scoring.md index 285be5c..f3fa9ac 100644 --- a/packages/pipette-scores/docs/scoring.md +++ b/packages/pipette-scores/docs/scoring.md @@ -7,22 +7,22 @@ Each eval has a dedicated scorer in `pipette_scores/scoring/`. The common flow i 3. **Evaluate** — each completion's reasoning markup is rewritten (see [§7](#7-reasoning-markup-and-cut-off-reasoning)), then it is compared to its ground-truth sample, producing a `SampleScore(id, is_correct)`. 4. **Enrich** — the API layer joins scores with the original prompts and completions and returns the full `ScoreResponse` (see `httpapi.md`). -A completion whose `id` does not match any dataset sample will raise a `KeyError`. Clients should only submit IDs returned by the samples endpoint. +`/score` answers 400 for a completion `id` that is not in the dataset. A direct call to a scorer raises `KeyError` for it. Submit only IDs that the samples endpoint returns. -The scoreable evals are exactly the members of `pipette_scores.types.EvalId`: `ifbench`, `ifstruct`, `gpqa_diamond`, and `math_500`. +The scoreable evals are the members of `pipette_scores.types.EvalId`: the text evals `ifbench`, `ifstruct`, `gpqa_diamond` and `math_500`, the 7 VLM evals ([§7](#vlm-evals)) and the 16 ASR evals ([§8](#8-asr-evals-open-asr-leaderboard-english-fleurs)). ## 1. Per-eval return shape -Each per-eval `score()` returns `list[SampleScore]`: +Each per-eval `score()` returns a pair `(scores, context)`: ```python -[ - SampleScore(id=str, is_correct=bool), - ... -] +( + [SampleScore(id=str, is_correct=bool), ...], + {"": ..., ...}, # pass@1, breakdowns, strict view +) ``` -Totals and per-sample enrichment (messages, completion) are assembled at the API layer — scorers stay focused on correctness judgement. +The context holds the eval's metrics. The API layer adds the messages and the completion to each sample score. For evals served with `metadata.repeats = N`, the samples endpoint expands each base id `` into `N` salted ids `#0 … #(N-1)`; the scorer treats each `#k` as an ordinary sample, and the headline **pass@1** is the mean correctness across the repeats per base sample. @@ -68,7 +68,7 @@ Competition mathematics; the model reasons step by step and gives its final answ The dataset version picks the grader through `metadata.grader`. Either way, reasoning markup is rewritten ([§7](#7-reasoning-markup-and-cut-off-reasoning)) and thinking tags are stripped first; the grader does its own answer extraction from what is left. -- **`2026.06.1`, `math_generic`** (no `grader` key): `score_math_generic` from `pipette_scores/scoring/math_500/math_generic.py`. The boxed answer is extracted, then graded by the **PRM800K** symbolic grader (sympy equivalence — so `1/2` matches `0.5`, or reordered expressions match) with a **robust-boxed** numeric-normalization fallback. Served with `metadata.repeats = 5`. The context counts each grading path as `math_500_method/{prm800k,robust_boxed,no_match,failed}`. +- **`2026.06.1`, `math_generic`** (no `grader` key): `score_math_generic` from `pipette_scores/scoring/math_500/math_generic.py`. The last `\boxed{}` answer is extracted (else the last number), then graded by the **PRM800K** symbolic grader (sympy equivalence — so `1/2` matches `0.5`, or reordered expressions match) with a **robust-boxed** numeric-normalization fallback. Served with `metadata.repeats = 5`. The context counts each grading path as `math_500_method/{prm800k,robust_boxed,no_match,failed}`. - **`2026.10.1`, `liquid_evals_math500`**: liquid_evals' `extract_math_answer` and `verify_math_answer`, vendored verbatim from `liquid_evals/parser.py` @ `d5534cf` as `pipette_scores/scoring/math_500/liquid_evals_parser.py`, called as `tasks/math500.py` calls them. `extract_math_answer` takes the last `\boxed{}` (brace-counted, so nested braces and boxes keep the outer content), else a `###`/`####` line, `ANSWER:`, "answer is …", "final answer" patterns, else the last number. `verify_math_answer` accepts an exact match, else tries `check_math_result` (Hendrycks' mathd normalization, or PRM800K-style sympy equivalence) on the extracted answer, then on the whole response with a leading "final answer:"/"answer:"/`###` removed, then on each of the response's last 150 lines. Sympy is skipped for expressions over 3000 characters, with more than two unknown letters, or with `^{`, `^(` or multi-digit exponents, and LaTeX with more than 100 macros is not parsed; there is no time limit. Served with `metadata.repeats = 5`. The context counts the pattern the answer came from as `math_500_extraction/{boxed,hash,answer_colon,answer_is,other,numeric,none}`; `math_500_unparsed_rate` is the `none` share. --- @@ -89,9 +89,9 @@ Differences that remain by design: - **Generation is client-side.** The service serves prompts only. With no overrides, liquid_evals (in-process vLLM) samples at temperature 0.6, top_p 1.0 and top_k -1, with a fresh random seed per generate call. vLLM's repetition guard is on: a 2–128-token pattern repeated 10 times stops generation. Up to 32768 new tokens are allowed within a 32768-token context, capped to the model's base context. It applies the model's chat template with no system prompt. Pipette's clients set their own policy (temperature 0.6, no fixed seed) and stop runaway repetition with their own detector. - **Repeats.** `math500_std` generates once per problem, while `math_500` `2026.06.1` is served with 5 repeats. The expected pass@1 is the same, and the variance is lower. `math500` also runs once per problem, but the model-card number is the mean of 5 `math500` runs, each with a fresh seed. `2026.10.1` serves those 5 passes as 5 repeats in one cell, and pass@1 over the 2500 attempts equals the mean of the 5 per-pass scores. -- **Crashes.** Where a liquid_evals checker raises and the whole run gets no score, the scorer here usually returns a verdict. For example, a JSON integer literal over 4300 digits fails the ifstruct sample, and a raising IFBench check counts as not followed for that response. Pathologically nested JSON (RecursionError) still fails on both sides; here `/score` returns HTTP 500 for the batch. +- **Crashes.** Where a liquid_evals checker raises and the whole run gets no score, the scorer here usually returns a verdict. For example, a JSON integer literal over 4300 digits fails the ifstruct sample, and a raising IFBench check counts as not followed for that response. Pathologically nested JSON fails that ifstruct sample with a parse error, and the batch still gets a score. - **IFBench key 286.** liquid_evals reads that row's `n_start=0` as unset and picks a random span start on every call, so a correct answer passes about 1 time in 42. Here it always passes, which can raise ifbench pass@1 by up to 0.33 pp (5 of 1500 attempts). -- **Denominators and dataset pins.** ifstruct's pass rate here counts only the completions submitted. liquid_evals counts every sample, so a missing completion is a fail there. The ifbench, gpqa_diamond and math_500 pass@1 keys here already count every sample. liquid_evals reads its Hugging Face datasets without a revision pin, while the datasets here are pinned by content hash. They are identical today. +- **Denominators and dataset pins.** ifstruct's pass rate here counts only the completions submitted. liquid_evals counts every sample, so a missing completion is a fail there. The ifbench, gpqa_diamond and math_500 pass@1 keys here already count every sample. liquid_evals reads its Hugging Face datasets without a revision pin, while the third-party datasets here are pinned by content hash and ifstruct is committed. They are identical today. - **Reasoning markup.** liquid_evals strips only ``, `[THINK]` and gpt-oss channels. The rewrite in [§7](#7-reasoning-markup-and-cut-off-reasoning) also reads Gemma 4, Cohere North and ERNIE markup. A completion without that markup gets the same verdict on both sides, whatever its stop reason, including one cut off inside its reasoning. - **Scale.** Scores here are 0–1 fractions; the internal eval scheduler shows them ×100. @@ -101,7 +101,7 @@ Differences that remain by design: Module: `pipette_scores/scoring/completion_text.py` (`grading_text`) -Every scorer grades `grading_text`'s output instead of the raw completion. The thinking-tag rules then run as before: `remove_thinking_tags` for ifbench, gpqa_diamond and math_500, and the validator's own copy for ifstruct, which therefore strips once. +Every text-eval scorer grades `grading_text`'s output instead of the raw completion. The thinking-tag rules then run on that output: `remove_thinking_tags` for ifbench, gpqa_diamond and math_500, and the validator's own copy for ifstruct, which therefore strips once. **Markup rewrite** (`normalize_reasoning_markup`). Some models mark their reasoning or their answer with tokens the thinking-tag rules do not know. These are rewritten first: @@ -123,10 +123,10 @@ Gemma 4 on vLLM without `--reasoning-parser gemma4` loses both channel tokens, w - gpt-oss cut inside its analysis channel has no `assistantfinal` transition and is graded whole. - A completion whose reasoning closed is graded on the answer after the close, even if the answer was cut. -**Strict view.** Scorer `2026.10.12-552094f` graded such completions as empty instead. Each eval still reports that view in the response context: +**Strict view.** The strict view grades such completions as empty. Each text eval reports it in the response context: - `_blanked_truncated` counts the completions it blanks. A completion is counted when the request carries `enable_thinking: true`, its `stop_reason` is `truncated` (the output-token cap or a full context) or `doom_loop` (a repetition stop: vLLM's `repetition_detection`, or an older client's detector), its rewritten text has no reasoning close (no ``, `[/THINK]` or gpt-oss `assistantfinal` transition), and that text is not empty. A missing `stop_reason`, or `eos`, `failure` or `unknown`, is never counted. -- `_pass_at_1_blanked` is the eval's pass@1 with those completions counted wrong: the headline that scorer reported. For ifstruct, which is served once per prompt, it is `pass_rate` with those completions counted wrong. Every eval fails an empty completion, so counting them wrong is the same as grading them empty. +- `_pass_at_1_blanked` is the eval's pass@1 with those completions counted wrong. For ifstruct, which is served once per prompt, it is `pass_rate` with those completions counted wrong. Every eval fails an empty completion, so counting them wrong is the same as grading them empty. Content that a server-side reasoning parser already split (vLLM with `--reasoning-parser`, or gpt-oss's Harmony final channel) is the answer alone with no close. With thinking on, a cut answer there is counted even if its reasoning finished, so the strict view understates those cells. @@ -151,13 +151,13 @@ Module: `split_reasoning`, `normalize_vlm_reasoning_markup`. The VLM scorers (mm | `unclosed_stop` | opened, no close, the model stopped on its own | | `no_reasoning` | no sign of reasoning | -The context counts them as `reasoning_status`, a dict with all five keys. A profile whose wrapper reduces the reply to its answer (`minicpm_v_cot`, [httpapi](httpapi.md)) is read after that reduction, as main's strict view reads it: a `minicpm_v_cot` reply cut inside its thought has become its answer, so it counts as `unclosed_cut` and blanked only through `enable_thinking`, which its runs record. With a server-side reasoning parser the content is the answer alone, so a run that declared `enable_thinking` reads `unclosed_stop`; the count flags it and changes no grade. +The context counts them as `reasoning_status`, a dict with all five keys. A profile whose wrapper reduces the reply to its answer (`minicpm_v_cot`, [httpapi](httpapi.md)) is read after that reduction: a `minicpm_v_cot` reply cut inside its thought has become its answer, so it counts as `unclosed_cut` and blanked only through `enable_thinking`, which its runs record. With a server-side reasoning parser the content is the answer alone, so a run that declared `enable_thinking` reads `unclosed_stop`; the count flags it and changes no grade. **Text graded.** The grading is liquid_evals', as above: a completion with no tag is graded whole, whatever its status. Complete blocks are removed (a block ends at the first close after its opener, and an opener inside it is part of it); the text through the first close that has no opener is reasoning (the first such ``, else the first such `[/THINK]`, as `remove_thinking_tags` cuts), and a later stray close stays in the text; an opener left at the end removes the rest; then the gpt-oss transition and trailing harmony markers go. The two tag families pair with each other (`[THINK]…`). Beyond that: - **Answer before close.** A reply that opens with a one-word answer, closes a block it never opened, and goes on to reasoning that gives no answer (`yes[/THINK][THINK]I need to determine…`, Ministral-3-Reasoning on yes/no questions) is graded on that word, when no opener was known. More than one word before the close is reasoning, as Ministral writes in `So the answer should be "yes".[/THINK]`, and such a reply is graded empty. Counted as `reasoning_answer_before_close`. The rules of liquid_evals grade such a reply empty. - **`unclosed_stop`.** When the opener is known from the profile or in the text and the model stopped without closing, a complete `…` in the reply is graded (the last one); without it the reply is graded as above. -- **`` wrapper.** An `` that opens the text after a close and its closing tag are dropped, like the `` and North wrappers of §7's table. +- **`` wrapper.** When the reasoning was opened, an `` that opens the graded text and its closing tag are dropped, like the `` and North wrappers of §7's table. **Strict view.** `blanked_truncated` counts a completion whose reasoning was opened by its own opener, by the run's prompt profile, or by `enable_thinking: true`, whose `stop_reason` is `truncated` or `doom_loop`, whose rewritten text has no close, and whose text is not empty, whatever the row recorded: a reply that opened its own thought and was cut counts even under `enable_thinking: false`. A reply cut inside reasoning that nothing shows was opened (a hidden control token, a profile or `enable_thinking` that does not say so) cannot be told from an answer, and is graded and counted as one. diff --git a/packages/pipette-scores/pipette_scores/scoring/asr/NOTICE.md b/packages/pipette-scores/pipette_scores/scoring/asr/NOTICE.md index 9d040de..b98db66 100644 --- a/packages/pipette-scores/pipette_scores/scoring/asr/NOTICE.md +++ b/packages/pipette-scores/pipette_scores/scoring/asr/NOTICE.md @@ -16,7 +16,7 @@ leaderboard method is cited at): `normalizer.py` also holds the `BasicMultilingualTextNormalizer` the multilingual normalizer below builds on. Licensed under the Apache License 2.0 -(see the source headers). +(see the header of `normalizer.py` and upstream's `LICENSE`). ## `open_asr_normalizer/multilingual.py` (multilingual, FLEURS) diff --git a/packages/pipette-scores/pipette_scores/scoring/vlm_upstream/PORTING_NOTES.md b/packages/pipette-scores/pipette_scores/scoring/vlm_upstream/PORTING_NOTES.md index 3343f6d..53f1f56 100644 --- a/packages/pipette-scores/pipette_scores/scoring/vlm_upstream/PORTING_NOTES.md +++ b/packages/pipette-scores/pipette_scores/scoring/vlm_upstream/PORTING_NOTES.md @@ -2,8 +2,12 @@ Answer extraction and metrics for the 7 VLM evals are vendored from **VLMEvalKit** (commit `8eca8e5`), deterministic-only: no LLM judge, no random -fallback — an answer `can_infer` cannot extract counts as **wrong** (the -deterministic equivalent of VLMEvalKit's `exact_matching` policy). +fallback. On the multiple-choice and yes/no evals, the vendored rule +(`can_infer` or `YOrN_Extraction`) reads each answer first. The deterministic +fallbacks in `scoring/vlm_answer.py` then read an answer that the rule cannot. +An answer that neither reads counts as **wrong** (the deterministic equivalent +of VLMEvalKit's `exact_matching` policy). The "Scoring" lines below name the +vendored rule only. Module provenance: @@ -149,24 +153,24 @@ scorers graded the same text: the completion with its reasoning removed. a line that opens with an option, the last line. VLMEvalKit hands those to its LLM judge, which pipette never uses. LFM2.5-VL-3B, which answers with a letter, has none; a reasoning model that explains its answer has many. -- The check found one fallback misread, now fixed: "I cannot answer yes or no" - read as Yes (4 Gemma-4-E2B MME answers, 2.6 of 2,800 points). - -- A later audit of the final runs found the format Qwen3.5's card prompt asks - for, `"answer": "C"`, written three more ways the fallbacks missed: the key +- A reply that names both "yes" and "no", such as "I cannot answer yes or no", + is not read as an answer. +- The fallbacks also read the format that Qwen3.5's card prompt asks for, + `"answer": "C"`, in three more shapes: the key in backticks (`` `answer`: "C" ``), the value alone (`"B"`, `"a"`, an unclosed `"B`), and a quoted option line (`"B. Not the same"`). Each is read only in an anchored shape: the field as a whole line; the value alone as the whole answer, or as a last line (capital letter or exact option) after text that names no option letter. Prose that describes the format, refuses - to choose, or quotes two options is not read. Qwen3.5-4B RealWorldQA goes - from 42.0 to 79.3 (its card: 79.5); Qwen3.5-2B MMBench from 58.0 to 77.7. - No other family's score moves. + to choose, or quotes two options is not read. With these shapes read, + Qwen3.5-4B scores 79.3 on RealWorldQA (42.0 without them; its card: 79.5), + and Qwen3.5-2B scores 77.7 on MMBench (58.0 without them). No other family's + score changes. ## Prompt parity (verified against VLMEvalKit source) -The served prompts match VLMEvalKit's dataset-default `build_prompt` verbatim, -with NO extra instruction beyond what upstream emits: +Without a prompt profile, the served prompts match VLMEvalKit's dataset-default +`build_prompt` verbatim, with NO extra instruction beyond what upstream emits: - **MMStar / MMBench_DEV_EN_V11 / RealWorldQA** use the base `ImageMCQDataset.build_prompt`: `Question:` + `Options:\nA. …` + the soft directive @@ -181,18 +185,15 @@ with NO extra instruction beyond what upstream emits: - **OCRBench** uses the raw question. Verbose models that describe the image instead of emitting the option letter -are handled by `can_infer` extraction over the full generation (up to -`max_new_tokens`), exactly as upstream does — not by a stricter prompt. +are handled by `can_infer` and then the `vlm_answer` fallbacks, on the +completion with its reasoning removed — not by a stricter prompt. ## Generation length -`parameter_max_tokens` is set to **1024** for all 7, matching our VLMEvalKit -fork (`vlmeval/vlm/liquid.py`, `max_new_tokens=1024`) -so on-device pipette numbers are comparable to internal LFM VLMEvalKit runs. -Generation length in VLMEvalKit is a per-*model* serving default, not a -per-dataset constant (Qwen2-VL/LLaVA=2048, InternVL=4096/512/8196), so there -is no single upstream value to copy — the LFM fork's 1024 is the one that -makes our results comparable. It is a ceiling, not a target: answer-first -models emit EOS in a handful of tokens (median 5 on LFM2-VL-450M mmstar); the -budget only binds for reasoning/CoT models that state their answer last, which -the previous tight caps (mme/pope 16, MCQ 64) would have silently clipped. +The scorers grade the text that the client sends. Allow enough output tokens +for the model to state its answer. The reference setting is **1024** output +tokens for all 7 evals. In VLMEvalKit, generation length is a per-*model* +serving default, not a per-dataset constant, so there is no single upstream +value to copy. The limit is a ceiling, not a target. Models that answer first +stop after a few tokens. Reasoning models that state their answer last need +the full budget, and a lower limit cuts their answer off. diff --git a/packages/pipette-scores/scripts/split_asr_dataset.py b/packages/pipette-scores/scripts/split_asr_dataset.py index f1dd9c1..a867c2d 100644 --- a/packages/pipette-scores/scripts/split_asr_dataset.py +++ b/packages/pipette-scores/scripts/split_asr_dataset.py @@ -8,7 +8,7 @@ uv run --project packages/pipette-scores \\ python packages/pipette-scores/scripts/split_asr_dataset.py \\ - --src /data/pipette-scores/datasets --out /datasets [EVAL_ID ...] + --src /data/pipette-scores/datasets --out /datasets [EVAL_ID ...] Writes ``///`` from ``///`` and then runs the server's integrity checks on the result (verify_asr_datasets.py re-hashes diff --git a/vendor/ifbench-checkers/README.md b/vendor/ifbench-checkers/README.md index 174200e..2b5b245 100644 --- a/vendor/ifbench-checkers/README.md +++ b/vendor/ifbench-checkers/README.md @@ -14,8 +14,9 @@ revision, which is why they are vendored at a fixed commit rather than tracked through the `vendor/ifbench` submodule. That submodule is still the source of the IFBench **test set**; only the checker code comes from here. -Git blob ids of the upstream files (the first commit that added this directory -holds them unmodified): +Git blob ids of the upstream files at `1a902f9`. The `vendor/ifbench` +submodule history contains that commit, so +`git -C vendor/ifbench ls-tree 1a902f9` lists them: | File | Blob | |---|---|