diff --git a/.agents/skills/run-piika-nanoknow/SKILL.md b/.agents/skills/run-piika-nanoknow/SKILL.md new file mode 100644 index 0000000..3934db5 --- /dev/null +++ b/.agents/skills/run-piika-nanoknow/SKILL.md @@ -0,0 +1,44 @@ +--- +name: run-piika-nanoknow +description: Install, run, evaluate, and report NanoKnow NQ-Open or NanoKnow SQuAD through Piika using the ClimbMix-supported query sets and climbmix-400b backend. Use for NanoKnow runs with normal retrieval or supplied-document bundles, multi-answer ground-truth preparation, smoke tests, full runs, and side-by-side experimental comparisons. +--- + +# Run Piika NanoKnow + +Run NanoKnow through generic installed manifests and adapters. Keep NanoKnow-specific paths and commands in this skill; do not add dataset-specific imports to Piika's TypeScript benchmark registry. + +## Select the condition + +Choose one dataset: + +- `nanoknow-nq`: aliases `nq`, `nq-open`; 3,021 ClimbMix-supported queries. +- `nanoknow-squad`: alias `squad`; 9,071 ClimbMix-supported queries. + +Choose one evidence condition: + +- Retrieval: start from the question and search `climbmix-400b`. +- Supplied documents: inject grouped documents per query, while retaining the backend for narrowly targeted gap filling. + +Read `references/end-to-end.md` completely before executing either condition. + +## Guardrails + +1. Work from the Piika repository root. +2. Inspect the required query, qrels, source-answer, and ground-truth paths. +3. Install the selected asset manifest with the generic installer; use `--dry-run` first. +4. Keep the retrieval and supplied-document conditions in distinct output directories. +5. Use the same model, thinking level, timeout, query population, and backend across a comparison. +6. Run a small `--limit` smoke test before a full dataset. +7. Verify `benchmark_manifest_snapshot.json`, `run_setup.json`, processed-query count, and failure count before evaluation. +8. Never treat missing backend access or provider usage limits as benchmark results. + +## Expected outputs + +For each condition, produce: + +- Native per-query run JSON. +- Retrieval evaluation from the benchmark qrels. +- Gold-answer judge evaluation using all acceptable answers. +- A Markdown report. + +When comparing supplied documents against retrieval-only runs, record the supplied bundle path and hash outside ignored data directories if long-term provenance is required. diff --git a/.agents/skills/run-piika-nanoknow/agents/openai.yaml b/.agents/skills/run-piika-nanoknow/agents/openai.yaml new file mode 100644 index 0000000..07ffedc --- /dev/null +++ b/.agents/skills/run-piika-nanoknow/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Run Piika NanoKnow" + short_description: "Run NanoKnow with or without supplied docs" + default_prompt: "Use $run-piika-nanoknow to run and evaluate a NanoKnow benchmark end to end." diff --git a/.agents/skills/run-piika-nanoknow/assets/nanoknow-nq.benchmark.json b/.agents/skills/run-piika-nanoknow/assets/nanoknow-nq.benchmark.json new file mode 100644 index 0000000..b6147b0 --- /dev/null +++ b/.agents/skills/run-piika-nanoknow/assets/nanoknow-nq.benchmark.json @@ -0,0 +1,25 @@ +{ + "id": "nanoknow-nq", + "aliases": ["nq", "nq-open"], + "displayName": "NanoKnow NQ-Open (ClimbMix)", + "datasetId": "nanoknow-nq", + "piSearchPromptVariant": "plain_minimal", + "defaultQuerySetId": "climbmix-supported", + "defaultQueryPath": "data/nanoknow-nq/queries/climbmix-supported.tsv", + "querySets": { + "climbmix-supported": "data/nanoknow-nq/queries/climbmix-supported.tsv" + }, + "defaultQrelsPath": "data/nanoknow-nq/qrels/climbmix-supported.txt", + "defaultGroundTruthPath": "data/nanoknow-nq/ground-truth/climbmix-supported.jsonl", + "defaultIndexPath": "climbmix-400b", + "managedPresets": {}, + "setup": { "steps": {} }, + "retrievalEvaluation": { + "runFileBackend": "internal", + "runDirBackend": "internal" + }, + "judgeEvaluation": { + "supportedModes": ["gold-answer", "reference-free"], + "defaultMode": "gold-answer" + } +} diff --git a/.agents/skills/run-piika-nanoknow/assets/nanoknow-squad.benchmark.json b/.agents/skills/run-piika-nanoknow/assets/nanoknow-squad.benchmark.json new file mode 100644 index 0000000..afddee2 --- /dev/null +++ b/.agents/skills/run-piika-nanoknow/assets/nanoknow-squad.benchmark.json @@ -0,0 +1,25 @@ +{ + "id": "nanoknow-squad", + "aliases": ["squad"], + "displayName": "NanoKnow SQuAD (ClimbMix)", + "datasetId": "nanoknow-squad", + "piSearchPromptVariant": "plain_minimal", + "defaultQuerySetId": "climbmix-supported", + "defaultQueryPath": "data/nanoknow-squad/queries/climbmix-supported.tsv", + "querySets": { + "climbmix-supported": "data/nanoknow-squad/queries/climbmix-supported.tsv" + }, + "defaultQrelsPath": "data/nanoknow-squad/qrels/climbmix-supported.txt", + "defaultGroundTruthPath": "data/nanoknow-squad/ground-truth/climbmix-supported.jsonl", + "defaultIndexPath": "climbmix-400b", + "managedPresets": {}, + "setup": { "steps": {} }, + "retrievalEvaluation": { + "runFileBackend": "internal", + "runDirBackend": "internal" + }, + "judgeEvaluation": { + "supportedModes": ["gold-answer", "reference-free"], + "defaultMode": "gold-answer" + } +} diff --git a/.agents/skills/run-piika-nanoknow/references/end-to-end.md b/.agents/skills/run-piika-nanoknow/references/end-to-end.md new file mode 100644 index 0000000..72fbae2 --- /dev/null +++ b/.agents/skills/run-piika-nanoknow/references/end-to-end.md @@ -0,0 +1,253 @@ +# NanoKnow end-to-end workflows + +## 1. Select NQ or SQuAD + +Set these values consistently. + +NQ-Open: + +```bash +NANOKNOW_BENCHMARK=nanoknow-nq +NANOKNOW_DATA_DIR=data/nanoknow-nq +NANOKNOW_MANIFEST=.agents/skills/run-piika-nanoknow/assets/nanoknow-nq.benchmark.json +``` + +SQuAD: + +```bash +NANOKNOW_BENCHMARK=nanoknow-squad +NANOKNOW_DATA_DIR=data/nanoknow-squad +NANOKNOW_MANIFEST=.agents/skills/run-piika-nanoknow/assets/nanoknow-squad.benchmark.json +``` + +The commands below assume the selected values remain in the current shell. + +## 2. Verify and prepare assets + +Required files: + +```bash +test -f "$NANOKNOW_DATA_DIR/queries/climbmix-supported.tsv" +test -f "$NANOKNOW_DATA_DIR/qrels/climbmix-supported.txt" +test -f "$NANOKNOW_DATA_DIR/source/answers.jsonl" +``` + +Prepare judge-compatible multi-answer ground truth: + +```bash +npm run adapt:multi-answer-ground-truth -- \ + --queries "$NANOKNOW_DATA_DIR/queries/climbmix-supported.tsv" \ + --answers "$NANOKNOW_DATA_DIR/source/answers.jsonl" \ + --output "$NANOKNOW_DATA_DIR/ground-truth/climbmix-supported.jsonl" \ + --id-field qid \ + --answers-field answer +``` + +Install the dataset manifest: + +```bash +npm run install:benchmark-manifest -- \ + --manifest "$NANOKNOW_MANIFEST" \ + --dry-run + +npm run install:benchmark-manifest -- \ + --manifest "$NANOKNOW_MANIFEST" +``` + +Confirm discovery: + +```bash +npm run run:benchmark:query-set -- --help +``` + +The selected benchmark must appear with query set `climbmix-supported`. + +## 3. Configure the ClimbMix backend + +Use the Pyserini REST service that exposes `climbmix-400b`: + +```bash +export PYSERINI_REST_BASE_URL=http://127.0.0.1:8080 +export PYSERINI_REST_INDEX=climbmix-400b +``` + +If the service requires a bearer token: + +```bash +export PYSERINI_API_TOKEN= +export PYSERINI_REST_TOKEN_ENV=PYSERINI_API_TOKEN +``` + +Do not put the token in a benchmark manifest or committed command file. Confirm service reachability before running thousands of queries. + +## 4A. Retrieval-only condition + +Choose an output directory: + +```bash +NANOKNOW_RUN_DIR="runs/${NANOKNOW_BENCHMARK}-retrieval" +``` + +Inspect the plan: + +```bash +npm run run:benchmark:query-set -- \ + --dry-run \ + --benchmark "$NANOKNOW_BENCHMARK" \ + --query-set climbmix-supported \ + --output-dir "$NANOKNOW_RUN_DIR" \ + --output-mode answer \ + --tool-interface pyserini-rest-2tool +``` + +Run a smoke test: + +```bash +npx tsx src/orchestration/run_pi_benchmark.ts \ + --benchmark "$NANOKNOW_BENCHMARK" \ + --querySet climbmix-supported \ + --outputDir "$NANOKNOW_RUN_DIR-smoke" \ + --model openai-codex/gpt-5.4-mini \ + --thinking medium \ + --limit 3 \ + --toolInterface pyserini-rest-2tool +``` + +Run the complete query set: + +```bash +npm run run:benchmark:query-set -- \ + --benchmark "$NANOKNOW_BENCHMARK" \ + --query-set climbmix-supported \ + --output-dir "$NANOKNOW_RUN_DIR" \ + --model openai-codex/gpt-5.4-mini \ + --thinking medium \ + --timeout-seconds 900 \ + --output-mode answer \ + --tool-interface pyserini-rest-2tool +``` + +## 4B. Supplied-document condition + +Set the supplied bundle path and a distinct output directory: + +```bash +NANOKNOW_SUPPLIED_DOC_BUNDLE=/absolute/path/to/supplied-docs.jsonl +NANOKNOW_RUN_DIR="runs/${NANOKNOW_BENCHMARK}-supplied-docs" +test -f "$NANOKNOW_SUPPLIED_DOC_BUNDLE" +``` + +Each JSONL row must have this shape: + +```json +{ + "qid": "query-id", + "question": "Question text", + "groups": [ + { + "docs": [ + { + "doc_id": "document-id", + "cited_snippet": "The source-provided supporting excerpt", + "full_text": "The complete supplied document text" + } + ] + } + ] +} +``` + +Requirements: + +- Include exactly the intended experimental query population. +- Use qids matching NanoKnow qrels and ground truth. +- Preserve document groups and stable document ids. +- Do not include gold answers as document fields or prompt text. +- Treat empty `cited_snippet` or `full_text` as data-quality issues even though the runner has fallbacks. + +Inspect the plan: + +```bash +npm run run:benchmark:query-set -- \ + --dry-run \ + --benchmark "$NANOKNOW_BENCHMARK" \ + --query-set climbmix-supported \ + --supplied-doc-bundle "$NANOKNOW_SUPPLIED_DOC_BUNDLE" \ + --output-dir "$NANOKNOW_RUN_DIR" \ + --output-mode answer \ + --tool-interface pyserini-rest-2tool +``` + +Run a three-query smoke test through the low-level runner: + +```bash +npx tsx src/orchestration/run_pi_benchmark.ts \ + --benchmark "$NANOKNOW_BENCHMARK" \ + --querySet climbmix-supported \ + --supplied-doc-bundle "$NANOKNOW_SUPPLIED_DOC_BUNDLE" \ + --outputDir "$NANOKNOW_RUN_DIR-smoke" \ + --model openai-codex/gpt-5.4-mini \ + --thinking medium \ + --limit 3 \ + --toolInterface pyserini-rest-2tool +``` + +Run the complete supplied-document population: + +```bash +npm run run:benchmark:query-set -- \ + --benchmark "$NANOKNOW_BENCHMARK" \ + --query-set climbmix-supported \ + --supplied-doc-bundle "$NANOKNOW_SUPPLIED_DOC_BUNDLE" \ + --output-dir "$NANOKNOW_RUN_DIR" \ + --model openai-codex/gpt-5.4-mini \ + --thinking medium \ + --timeout-seconds 900 \ + --output-mode answer \ + --tool-interface pyserini-rest-2tool +``` + +Supplied docids should appear in each run's metadata and surfaced, previewed, and agent-visible docid views. + +## 5. Evaluate and report either condition + +Retrieval evaluation: + +```bash +RUN_DIR="$NANOKNOW_RUN_DIR" npm run evaluate:retrieval +``` + +Gold-answer judge: + +```bash +INPUT_DIR="$NANOKNOW_RUN_DIR" npm run evaluate:run -- \ + --judge-mode gold-answer +``` + +Summary and report: + +```bash +RUN_DIR="$NANOKNOW_RUN_DIR" npm run summarize:run +RUN_DIR="$NANOKNOW_RUN_DIR" npm run report:run +``` + +## 6. Comparison checks + +Before comparing the two conditions, confirm: + +```bash +test -f "$NANOKNOW_RUN_DIR/benchmark_manifest_snapshot.json" +test -f "$NANOKNOW_RUN_DIR/run_setup.json" +``` + +Inspect these fields: + +- benchmark and query-set ids; +- model and thinking level; +- tool interface and backend kind; +- timeout and total query count; +- supplied bundle path for the supplied condition; +- failed, timed-out, and completed query counts; +- input hashes in the benchmark snapshot. + +Do not compare aggregate accuracy or retrieval metrics until both conditions contain the intended identical qid population. diff --git a/.agents/skills/use-piika/SKILL.md b/.agents/skills/use-piika/SKILL.md new file mode 100644 index 0000000..664cc91 --- /dev/null +++ b/.agents/skills/use-piika/SKILL.md @@ -0,0 +1,57 @@ +--- +name: use-piika +description: Configure, run, evaluate, compare, summarize, and report Piika research and retrieval benchmarks. Use for Piika package setup; built-in, prebuilt, or custom benchmark manifests; local Anserini or remote Pyserini backends; single, shared, or sharded runs; answer, ranked-list, or combined outputs; supplied-document bundles; and reproducible evaluation workflows. +--- + +# Use Piika + +Operate Piika through its package entrypoints and benchmark manifests. Prefer dry runs and manifest-backed defaults so run artifacts preserve the actual experimental condition. + +## Establish context + +1. Work from the repository root containing `package.json`. +2. Inspect `git status --short --branch`; do not disturb unrelated work. +3. Read `README.md` for the current high-level workflow. +4. Read `docs/cli.md` when using the packaged `piika` CLI or prebuilt indexes. +5. Read `references/workflows.md` for the command and option matrix. +6. Use `--help` and `--dry-run` before unfamiliar or expensive operations. + +## Choose the benchmark source + +- Use a built-in benchmark when `npm run run:benchmark:query-set -- --help` lists it. +- Use `npm run prebuilt -- setup ...` for a Castorini prebuilt index/topics/qrels combination. +- Use `npm run install:benchmark-manifest -- --manifest ` for any other reusable benchmark. Validate first with `--dry-run`. +- Use explicit `--query-file`, `--qrels`, and `--index-path` only for ad hoc experiments where generic benchmark identity in the run manifest is acceptable. + +Never add a dataset-specific TypeScript registry entry when an installed manifest expresses the same configuration. + +## Choose the backend and topology + +- Prefer the default direct `pyserini-rest-2tool` interface for remote Pyserini and ordinary local runs. +- Use `pi-serini-3tool` only to reproduce cached search-result browsing behavior. +- Use the single-process runner for smoke tests and small query sets. +- Use shared BM25 when multiple queries should reuse one local backend. +- Use sharded shared BM25 for parallel production runs. + +Confirm that the configured index is reachable before launching a long run. A remote index name is not a local filesystem path. + +## Choose outputs + +- Use `answer` for question answering. +- Use `ranked_list` for TREC-style document rankings. +- Use `answer+ranked_list` to produce both from one research pass. +- Set `--ranked-list-depth` as a maximum. +- Set `--ranked-list-count` only when an exact list length is an intentional experimental constraint. +- Add `--supplied-doc-bundle` when each query includes preselected evidence. Ensure bundle qids align with qrels and ground truth. + +## Execute and verify + +1. Print the launch plan with `--dry-run`. +2. Run a small `--limit` smoke test when using a new model, backend, or dataset. +3. Launch the intended full topology. +4. Run retrieval evaluation when qrels exist. +5. Run judge evaluation in `gold-answer` or `reference-free` mode as appropriate. +6. Generate a report from the same run directory. +7. Inspect the run's `benchmark_manifest_snapshot.json` and `run_setup.json` before comparing results. + +Treat provider usage limits, missing indexes, judge nonzero exits, malformed ground truth, and qid mismatches as failed setup—not valid benchmark outcomes. diff --git a/.agents/skills/use-piika/agents/openai.yaml b/.agents/skills/use-piika/agents/openai.yaml new file mode 100644 index 0000000..0f10e4c --- /dev/null +++ b/.agents/skills/use-piika/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Use Piika" + short_description: "Run, evaluate, and report Piika benchmarks" + default_prompt: "Use $use-piika to configure and run a Piika benchmark end to end." diff --git a/.agents/skills/use-piika/references/workflows.md b/.agents/skills/use-piika/references/workflows.md new file mode 100644 index 0000000..d493c5d --- /dev/null +++ b/.agents/skills/use-piika/references/workflows.md @@ -0,0 +1,239 @@ +# Piika workflow reference + +## Entry points + +```bash +npm run run:benchmark:query-set -- --help +npm run run:benchmark:query-set:shared-bm25 -- --help +npm run run:benchmark:query-set:sharded-shared-bm25 -- --help +npm run evaluate:retrieval -- --help +npm run evaluate:run -- --help +npm run summarize:run -- --help +npm run report:run -- --help +``` + +The packaged CLI exposes the same orchestration: + +```bash +npx piika --help +npx piika benchmarks +``` + +## Dependency and quality checks + +```bash +npm install +npm run typecheck +npm test +npm run check +``` + +Do not reinstall dependencies when `node_modules` is already usable unless the lockfile changed or a command reports a missing dependency. + +## Benchmark setup choices + +Built-in setup: + +```bash +npm run setup:benchmark -- --benchmark benchmark-template +npm run setup:benchmark -- --benchmark msmarco-v1-passage +``` + +Castorini prebuilt setup: + +```bash +npm run prebuilt -- indexes +npm run prebuilt -- topics +npm run prebuilt -- qrels +npm run prebuilt -- setup --topics --qrels --dry-run +npm run prebuilt -- setup --topics --qrels +``` + +Custom installed manifest: + +```bash +npm run install:benchmark-manifest -- \ + --manifest path/to/benchmark.json \ + --dry-run + +npm run install:benchmark-manifest -- \ + --manifest path/to/benchmark.json +``` + +The default destination is `data/prebuilt//benchmark.json`. Set `PIIKA_BENCHMARKS_DIR` or pass `--root` to use another installed-manifest root. Installation is idempotent for identical manifests and refuses differing replacements unless `--force` is explicit. + +Required manifest shape: + +```json +{ + "id": "custom-benchmark", + "aliases": [], + "displayName": "Custom Benchmark", + "datasetId": "custom-benchmark", + "piSearchPromptVariant": "plain_minimal", + "defaultQuerySetId": "test", + "defaultQueryPath": "data/custom/queries.tsv", + "querySets": { + "test": "data/custom/queries.tsv" + }, + "defaultQrelsPath": "data/custom/qrels.txt", + "defaultGroundTruthPath": "data/custom/ground-truth.jsonl", + "defaultIndexPath": "indexes/custom", + "managedPresets": {}, + "setup": { "steps": {} }, + "retrievalEvaluation": { + "runFileBackend": "internal", + "runDirBackend": "internal" + }, + "judgeEvaluation": { + "supportedModes": ["gold-answer", "reference-free"], + "defaultMode": "gold-answer" + } +} +``` + +## Backends + +Remote Pyserini REST: + +```bash +PYSERINI_REST_BASE_URL=http://host:port \ +PYSERINI_REST_INDEX= \ +npm run run:benchmark:query-set -- \ + --benchmark \ + --tool-interface pyserini-rest-2tool +``` + +If authentication is required, set the token variable and `PYSERINI_REST_TOKEN_ENV` according to the service configuration. Do not write tokens into manifests, commands committed to source, or run metadata. + +Local single-process BM25: + +```bash +npm run run:benchmark:query-set -- \ + --benchmark \ + --index-path indexes/ +``` + +Shared local BM25: + +```bash +npm run run:benchmark:query-set:shared-bm25 -- \ + --benchmark +``` + +Sharded shared local BM25: + +```bash +npm run run:benchmark:query-set:sharded-shared-bm25 -- \ + --benchmark \ + --shard-count 4 +``` + +## Output modes + +Answer: + +```bash +--output-mode answer +``` + +Ranked list: + +```bash +--output-mode ranked_list --ranked-list-depth 1000 +``` + +Answer and ranking: + +```bash +--output-mode answer+ranked_list --ranked-list-depth 30 +``` + +Exact count: + +```bash +--ranked-list-depth 30 --ranked-list-count 30 +``` + +## Supplied documents + +Pass a JSONL file: + +```bash +--supplied-doc-bundle path/to/supplied-docs.jsonl +``` + +Each row must contain: + +```json +{ + "qid": "q1", + "question": "Question text", + "groups": [ + { + "docs": [ + { + "doc_id": "d1", + "cited_snippet": "Relevant excerpt", + "full_text": "Full document text" + } + ] + } + ] +} +``` + +Supplied docids are recorded in metadata and count as surfaced, previewed, and agent-visible evidence. The agent may still use the configured search backend for gaps. + +## Evaluation and reporting + +Retrieval evaluation: + +```bash +RUN_DIR=runs/ npm run evaluate:retrieval +``` + +Gold-answer judge: + +```bash +INPUT_DIR=runs/ npm run evaluate:run -- \ + --judge-mode gold-answer +``` + +Explicit gold and qrels overrides: + +```bash +INPUT_DIR=runs/ npm run evaluate:run -- \ + --judge-mode gold-answer \ + --ground-truth path/to/ground-truth.jsonl \ + --qrel-evidence path/to/qrels.txt +``` + +Reference-free judge: + +```bash +INPUT_DIR=runs/ npm run evaluate:run -- \ + --judge-mode reference-free +``` + +Summarize and report: + +```bash +RUN_DIR=runs/ npm run summarize:run +RUN_DIR=runs/ npm run report:run +``` + +## Multi-answer ground truth + +Convert a source JSONL containing an id and an array of acceptable answers: + +```bash +npm run adapt:multi-answer-ground-truth -- \ + --queries path/to/queries.tsv \ + --answers path/to/source-answers.jsonl \ + --output path/to/ground-truth.jsonl \ + --id-field qid \ + --answers-field answer +``` + +The output stores the acceptable alternatives in the judge-compatible ground-truth format. The judge treats a match to any alternative as correct. diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c661b0..47f895e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Added + +- Added generic supplied document bundles for benchmark runs via `--supplied-doc-bundle` or `SUPPLIED_DOC_BUNDLE`. Bundles inject per-query grouped evidence into the agent prompt, persist `metadata.supplied_docids`, and count supplied documents in surfaced, previewed, and agent-visible retrieval views. + ### Changed - Updated the repository branding from `pi-serini` to `piika`, including the new logo and successor note while preserving historical release notes under their original name. diff --git a/README.md b/README.md index 8bf9143..5b3228c 100644 --- a/README.md +++ b/README.md @@ -95,6 +95,41 @@ classic TREC topic readers are normalized directly; piika reports a clear error Anserini topic readers that do not yet have a lossless converter. Index downloads are atomic, checksum-verified, and retried against the mirrors published in the upstream catalog. +### Install a custom benchmark manifest + +Datasets that only need different queries, qrels, ground truth, index paths, aliases, or evaluation +defaults do not need a TypeScript registry entry. Define a `BenchmarkDefinition` as JSON and install +it into Piika's dynamic manifest directory: + +```bash +npm run install:benchmark-manifest -- \ + --manifest path/to/benchmark.json \ + --dry-run + +npm run install:benchmark-manifest -- \ + --manifest path/to/benchmark.json +``` + +The installer validates the manifest and writes +`data/prebuilt//benchmark.json`. Reinstalling identical content is a no-op; replacing +different content requires an explicit `--force`. Set `PIIKA_BENCHMARKS_DIR` or pass `--root` to +use a different manifest directory. + +For datasets with multiple acceptable answers per question, adapt source JSONL into Piika's +ground-truth format: + +```bash +npm run adapt:multi-answer-ground-truth -- \ + --queries data/custom/queries.tsv \ + --answers data/custom/source/answers.jsonl \ + --output data/custom/ground-truth/answers.jsonl \ + --id-field qid \ + --answers-field answer +``` + +The source id and answers field names are configurable. Answer arrays are deduplicated and judged +so that matching any acceptable alternative counts as correct. + ### 1. Set up benchmark assets BrowseComp-Plus base assets: @@ -164,6 +199,17 @@ MODEL=openai-codex/gpt-5.4-mini \ npm run run:benchmark:query-set ``` +To inject supplied evidence directly into each query prompt, pass a JSONL bundle with one row per query: + +```bash +BENCHMARK=benchmark-template \ +QUERY_SET=test \ +MODEL=openai-codex/gpt-5.4-mini \ +npm run run:benchmark:query-set -- --supplied-doc-bundle data/my-run/supplied-docs.jsonl +``` + +Rows use `{ "qid", "question", "groups": [{ "docs": [{ "doc_id", "cited_snippet", "full_text" }] }] }`. Supplied document ids are recorded in run metadata and counted in surfaced, previewed, and agent-visible retrieval views. See [docs/running-benchmarks.md](docs/running-benchmarks.md#supplied-document-bundles) for details. + Benchmark launches default to the direct `pyserini-rest-2tool` interface: `search` returns ranked hits directly and `read_document` opens a selected document. This interface is also used when the configured backend is local Anserini BM25; the name describes the two-tool contract, not @@ -384,6 +430,7 @@ Legacy shell scripts under `scripts/` still work, but they are compatibility shi - `src/legacy/` — compatibility-only TypeScript entrypoints that are still intentionally preserved for historical low-level contracts - `src/runtime/` — shared runtime primitives such as prompt construction, artifact-path helpers, and isolated agent-dir handling - `src/benchmarks/` — typed benchmark definitions, registry helpers, run-manifest snapshot logic +- `src/adapters/` — reusable import and ground-truth normalization adapters - `src/wrappers/` — downstream summarize/eval/report wrapper entrypoints and precedence helpers - `src/operator/` — monitor, supervisor, TUI, and benchctl operator surfaces - `src/evaluation/` — retrieval and judge evaluation backends plus metric helpers @@ -398,6 +445,7 @@ Legacy shell scripts under `scripts/` still work, but they are compatibility shi - `runs/` — benchmark run outputs - `evals/` — evaluation outputs - `notes/` — local notes and experiment writeups +- `.agents/skills/` — versioned Codex skills for general Piika and benchmark-specific workflows ## Read more @@ -407,6 +455,8 @@ Legacy shell scripts under `scripts/` still work, but they are compatibility shi - [Evaluation semantics](docs/evaluation.md) - [Reproducibility](docs/reproducibility.md) - [Adding a benchmark](docs/adding-a-benchmark.md) +- [General Piika agent skill](.agents/skills/use-piika/SKILL.md) +- [NanoKnow agent skill](.agents/skills/run-piika-nanoknow/SKILL.md) - [BM25 backend interface](docs/bm25-extension-interface.md) - Released Run on BrowseComp-Plus (Canary to prevent leakage: `piserini-a-minimal-search-agent`) - [piika w/ DeepSeek V4 Flash](https://huggingface.co/datasets/ricky42613/piserini_bcp_deepseekv4_flash) diff --git a/docs/adding-a-benchmark.md b/docs/adding-a-benchmark.md index 85d62a5..0bb073f 100644 --- a/docs/adding-a-benchmark.md +++ b/docs/adding-a-benchmark.md @@ -1,7 +1,9 @@ # Adding a benchmark -This repo is benchmark-manifest-driven. -Adding a new benchmark should primarily mean registering one typed benchmark definition and implementing any benchmark-specific setup scripts, not editing a dozen unrelated entrypoints. +This repo is benchmark-manifest-driven. Most datasets should be added as dynamically installed +JSON manifests, without changing the TypeScript registry. Add a built-in typed definition only +when the benchmark is a permanent package feature with code-owned setup, managed presets, or +semantics that cannot be expressed declaratively. Related docs: @@ -25,7 +27,12 @@ The architecture intentionally separates: - prompt construction - artifact-path helpers - isolated agent-dir handling -- benchmark-specific concerns under `src/benchmarks/` plus benchmark-scoped setup scripts +- reusable benchmark metadata in installed `benchmark.json` files + - dataset/query files + - qrels and optional secondary qrels + - optional ground truth + - evaluation modes and index paths +- package-owned benchmark concerns under `src/benchmarks/` plus benchmark-scoped setup scripts - dataset/query files - qrels and optional secondary qrels - optional ground truth @@ -34,9 +41,39 @@ The architecture intentionally separates: That separation is what keeps multi-benchmark support maintainable. -## 1. Add the benchmark definition +## 1. Choose installed or built-in integration + +Use an installed manifest when the benchmark only needs: + +- ids, aliases, and display metadata +- query-set paths +- qrels and optional ground-truth paths +- an index path or remote index name +- existing retrieval and judge evaluation modes + +Install it with: + +```bash +npm run install:benchmark-manifest -- \ + --manifest path/to/benchmark.json \ + --dry-run + +npm run install:benchmark-manifest -- \ + --manifest path/to/benchmark.json +``` + +The default destination is `data/prebuilt//benchmark.json`. Installed manifests are +automatically visible to launch, evaluation, reporting, and benchmark discovery. Installation is +idempotent for identical content and protects differing existing content unless `--force` is +explicit. + +Use a built-in TypeScript definition only when the package must ship benchmark-specific code, +setup steps, managed presets, or genuinely distinct evaluation semantics. + +## 2. Define benchmark metadata -Create a new file under: +For an installed benchmark, create a JSON object matching `BenchmarkDefinition`. For a built-in +benchmark, create a TypeScript file under: - `src/benchmarks/.ts` @@ -73,15 +110,16 @@ keep defaults benchmark-scoped and declarative here instead of spreading them ac Use query-set-specific overrides when different query sets need different qrels, as with MSMARCO `dl19` vs `dl20`. -## 2. Register the benchmark +## 3. Register built-in benchmarks only Update: - `src/benchmarks/registry.ts` -Import the new benchmark and add it to `BENCHMARKS`. +Import the new benchmark and add it to `BUILTIN_BENCHMARKS`. -Once registered, these generic helpers will automatically understand it: +Skip this step for installed manifests. Once a built-in definition is registered, these generic +helpers will automatically understand it: - `getBenchmarkDefinition()` - `resolveBenchmarkConfig()` @@ -89,7 +127,7 @@ Once registered, these generic helpers will automatically understand it: - `resolveManagedPreset()` - `resolveBenchmarkSetupStep()` -## 3. Add benchmark-scoped setup scripts +## 4. Add benchmark-scoped setup scripts when necessary Add benchmark-specific setup implementations under: @@ -106,7 +144,7 @@ setup internals are often dataset-specific, but setup dispatch should still be s The current architectural decision is to keep benchmark setup implementations as benchmark-scoped subprocess boundaries unless a benchmark's setup becomes simple enough to be genuinely generic. In other words, `src/orchestration/setup_benchmark_entry.ts` owns the operator-facing control plane, while `scripts/benchmarks//...` owns the benchmark-specific bootstrap details. -## 4. Decide evaluation semantics +## 5. Decide evaluation semantics The generic retrieval/report pipeline assumes path defaults can come from the benchmark manifest, but not every benchmark necessarily has the same semantics. @@ -127,7 +165,7 @@ Do not invent fake ground-truth compatibility just to reuse the gold-answer path If the benchmark requires semantic differences instead of just different paths, add benchmark-specific evaluation adapters deliberately rather than smuggling differences through ad hoc conditionals. The current mechanism is `BenchmarkDefinition.retrievalEvaluation`, which lets a benchmark choose an internal TypeScript backend or a `trec_eval` run-file backend. Both backends now write normalized retrieval-summary artifacts under `evals/retrieval//...`. -## 5. Decide managed presets +## 6. Decide managed presets If the benchmark needs operator-facing managed presets for `benchctl`, define them in: @@ -143,7 +181,7 @@ Each preset can define: Use registry-defined presets for benchmark-specific ergonomics, but keep generic launch entrypoints benchmark-agnostic. -## 6. Add or stage local assets +## 7. Add or stage local assets Typical local asset layout should remain benchmark-scoped: @@ -154,7 +192,7 @@ Typical local asset layout should remain benchmark-scoped: Avoid reusing BrowseComp-Plus paths or names in a new benchmark unless the assets are genuinely shared. -## 7. Verify run-manifest behavior +## 8. Verify run-manifest behavior Every run should emit: @@ -174,9 +212,10 @@ Verify that downstream tools resolve your benchmark correctly from those artifac This is critical because reproducibility now depends on artifact-local benchmark metadata, not only mutable repo defaults. -## 8. Add tests +## 9. Add tests -At minimum, add or update tests for: +For an installed manifest, test validation, installation, discovery, and launcher dry-run +resolution. For a built-in benchmark, add or update tests for: - registry lookup - query-set resolution @@ -187,11 +226,12 @@ At minimum, add or update tests for: Good starting files: +- `tests/installed_benchmark_manifest.test.ts` - `tests/benchmarks/registry.test.ts` - `tests/launcher_wrappers.test.ts` - `tests/report_run_markdown.test.ts` -## 9. Document the benchmark +## 10. Document the benchmark Update operator-facing docs: @@ -214,28 +254,28 @@ Document: Use this as the short version: -1. Add `src/benchmarks/.ts` -2. Register it in `src/benchmarks/registry.ts` -3. Add `scripts/benchmarks//...` setup scripts -4. Add local dataset/index path conventions under `data/` and `indexes/` -5. Verify run-manifest snapshot output -6. Verify `run_setup.json` output for new runs -7. Verify summarize/eval/report behavior -8. Add tests -9. Update docs +1. Prefer a JSON `BenchmarkDefinition` and install it with `install:benchmark-manifest` +2. Add local dataset/index path conventions under `data/` and `indexes/` +3. Verify benchmark discovery and launcher dry-run output +4. Verify run-manifest snapshot and `run_setup.json` output +5. Verify summarize/eval/report behavior +6. Add a built-in TypeScript definition only if package-owned behavior requires it +7. Add tests and update docs ## What not to do Do not: - hardcode new benchmark defaults into many separate CLIs +- add a built-in registry entry for a path-only dataset that an installed manifest can express - add fallback legacy dual-mode logic unless there is a real compatibility requirement - hide benchmark-specific evaluation semantics inside generic path helpers - make shell wrappers the only supported path for a new benchmark The intended steady state is: -- benchmark metadata lives in typed registry entries +- reusable benchmark metadata lives in validated installed manifests +- package-owned benchmark metadata lives in typed registry entries - active Node-first entrypoints live under `src/orchestration/` - compatibility-only TypeScript entrypoints live under `src/legacy/` - shared runtime helpers live under `src/runtime/` diff --git a/docs/cli.md b/docs/cli.md index dd057a2..42f3a18 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -115,6 +115,40 @@ manifests are loaded dynamically by `piika benchmarks`, `piika run`, evaluation, adding an upstream index or query set does not require a source-code change. Set `PIIKA_BENCHMARKS_DIR` to use a different installed-manifest directory. +### Custom installed manifests + +The same dynamic loader supports non-Castorini datasets. Create a JSON file matching +`BenchmarkDefinition`, then validate and install it with: + +```bash +npm run install:benchmark-manifest -- \ + --manifest path/to/benchmark.json \ + --dry-run + +npm run install:benchmark-manifest -- \ + --manifest path/to/benchmark.json +``` + +The default destination is `data/prebuilt//benchmark.json`. Use `--root ` or +`PIIKA_BENCHMARKS_DIR` for a different manifest root. Identical reinstallation is idempotent. +Different existing content is protected unless `--force` is passed explicitly. + +Use an installed manifest instead of a source-code registry entry when a dataset differs only in +paths, aliases, query sets, evaluation defaults, or index configuration. Reserve built-in +TypeScript definitions for benchmarks shipped as permanent package features with code-owned setup +or semantics. + +For source answer JSONL that contains arrays of acceptable alternatives, normalize it with: + +```bash +npm run adapt:multi-answer-ground-truth -- \ + --queries path/to/queries.tsv \ + --answers path/to/source-answers.jsonl \ + --output path/to/ground-truth.jsonl \ + --id-field qid \ + --answers-field answer +``` + ## Setup Set up benchmark assets with: diff --git a/docs/evaluation.md b/docs/evaluation.md index c96228b..130387c 100644 --- a/docs/evaluation.md +++ b/docs/evaluation.md @@ -119,6 +119,27 @@ Judge evaluation supports two explicit modes: The benchmark manifest declares which modes are supported and which mode is the benchmark default. +### Multiple acceptable gold answers + +Gold-answer ground truth may encode multiple acceptable alternatives as a JSON array serialized in +the `answer` field. The judge is instructed to mark a response correct when its final answer +matches any alternative. + +Normalize source JSONL containing answer arrays with: + +```bash +npm run adapt:multi-answer-ground-truth -- \ + --queries path/to/queries.tsv \ + --answers path/to/source-answers.jsonl \ + --output path/to/ground-truth.jsonl \ + --id-field qid \ + --answers-field answer +``` + +The adapter joins rows by query id, rejects missing or duplicate records, trims and deduplicates +string alternatives, and writes the standard `query_id`, `query`, and `answer` ground-truth +fields. The source id and answer-array field names are configurable. + ### Gold-answer mode The judge sees: diff --git a/docs/running-benchmarks.md b/docs/running-benchmarks.md index ceab16c..a27573f 100644 --- a/docs/running-benchmarks.md +++ b/docs/running-benchmarks.md @@ -141,6 +141,60 @@ MODEL=openai-codex/gpt-5.4-mini \ npm run run:benchmark:query-set ``` +### Supplied document bundles + +Use a supplied document bundle when a run should inject known evidence directly into the agent prompt before it uses search tools. The runner still loads the normal benchmark query and qrels paths for identity and retrieval evaluation, but the question text and supplied documents come from the bundle rows. + +Pass the bundle either as a flag: + +```bash +BENCHMARK=benchmark-template \ +QUERY_SET=test \ +MODEL=openai-codex/gpt-5.4-mini \ +npm run run:benchmark:query-set -- --supplied-doc-bundle data/my-run/supplied-docs.jsonl +``` + +Or as an environment variable: + +```bash +SUPPLIED_DOC_BUNDLE=data/my-run/supplied-docs.jsonl \ +BENCHMARK=benchmark-template \ +QUERY_SET=test \ +MODEL=openai-codex/gpt-5.4-mini \ +npm run run:benchmark:query-set +``` + +The bundle is JSONL, one query per line. The example below is formatted for readability; in the file, write each row as one JSON object on a single line: + +```json +{ + "qid": "1", + "question": "Which city hosted the event?", + "groups": [ + { + "docs": [ + { + "doc_id": "doc-1", + "cited_snippet": "The event took place in Lisbon.", + "full_text": "Full document text..." + } + ] + } + ] +} +``` + +Each row requires: + +- `qid`: query id used for the per-query run artifact +- `question`: question sent to the agent +- `groups`: ordered document groups, each with a `docs` array +- `doc_id`: document id recorded in run metadata and retrieval views +- `cited_snippet`: short evidence snippet shown before the full text +- `full_text`: supplied document text shown in the prompt + +When a bundle is present, the prompt tells the agent to answer from supplied documents first and use a small optional search budget only when the supplied evidence leaves a concrete gap. Supplied `doc_id` values are recorded in `metadata.supplied_docids` and counted in `surfaced_docids`, `previewed_docids`, and `agent_docids` so retrieval summaries treat injected evidence as visible evidence, not hidden oracle state. + ### Shared BM25 daemon Use this when multiple benchmark workers should reuse one BM25 server: diff --git a/package.json b/package.json index e2fe22b..dc1d800 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,7 @@ "piika": "./bin/piika.js" }, "files": [ + ".agents", "bin", "src", "scripts", @@ -65,6 +66,8 @@ "evaluate:retrieval": "npx tsx src/wrappers/evaluate_retrieval_entry.ts", "report:run": "npx tsx src/wrappers/report_run_markdown_entry.ts", "adapt:search-jsonl-run": "npx tsx src/adapters/import_search_jsonl_run.ts", + "adapt:multi-answer-ground-truth": "npx tsx src/adapters/prepare_multi_answer_ground_truth.ts", + "install:benchmark-manifest": "npx tsx src/benchmarks/install_manifest_entry.ts", "tune:bm25": "npx tsx src/orchestration/tune_bm25_entry.ts", "compare:bm25": "npx tsx src/evaluation/compare_bm25_runs.ts", "bench": "npx tsx src/operator/benchctl.ts", diff --git a/src/adapters/prepare_multi_answer_ground_truth.ts b/src/adapters/prepare_multi_answer_ground_truth.ts new file mode 100644 index 0000000..9a59195 --- /dev/null +++ b/src/adapters/prepare_multi_answer_ground_truth.ts @@ -0,0 +1,137 @@ +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { pathToFileURL } from "node:url"; + +export type MultiAnswerGroundTruthOptions = { + queriesPath: string; + answersPath: string; + outputPath: string; + idField?: string; + answersField?: string; +}; + +type ParsedArgs = MultiAnswerGroundTruthOptions & { help?: boolean }; + +function parseArgs(argv: string[]): ParsedArgs { + const args: Partial = {}; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === "--help" || arg === "-h") return { ...args, help: true } as ParsedArgs; + const next = argv[index + 1]; + if (!next) throw new Error(`${arg} requires a value`); + if (arg === "--queries") args.queriesPath = next; + else if (arg === "--answers") args.answersPath = next; + else if (arg === "--output") args.outputPath = next; + else if (arg === "--id-field") args.idField = next; + else if (arg === "--answers-field") args.answersField = next; + else throw new Error(`Unknown argument: ${arg}`); + index += 1; + } + if (!args.queriesPath || !args.answersPath || !args.outputPath) { + throw new Error("--queries, --answers, and --output are required"); + } + return args as ParsedArgs; +} + +function readAcceptableAnswers( + answersPath: string, + idField: string, + answersField: string, +): Map { + const answers = new Map(); + for (const [index, line] of readFileSync(resolve(answersPath), "utf8") + .split(/\r?\n/u) + .entries()) { + if (!line.trim()) continue; + const row = JSON.parse(line) as Record; + const rawId = row[idField]; + const rawAnswers = row[answersField]; + if ((typeof rawId !== "string" && typeof rawId !== "number") || !Array.isArray(rawAnswers)) { + throw new Error( + `Invalid answer row ${index + 1}: expected ${idField} as string/number and ${answersField} as an array`, + ); + } + const queryId = String(rawId); + if (answers.has(queryId)) { + throw new Error(`Duplicate answer row for query ${queryId}`); + } + const acceptable = Array.from( + new Set( + rawAnswers + .filter((answer): answer is string => typeof answer === "string") + .map((answer) => answer.trim()) + .filter(Boolean), + ), + ); + if (acceptable.length === 0) { + throw new Error(`No acceptable answers found for query ${queryId}`); + } + answers.set(queryId, acceptable); + } + return answers; +} + +export function prepareMultiAnswerGroundTruth(options: MultiAnswerGroundTruthOptions): { + outputPath: string; + rowCount: number; +} { + const idField = options.idField?.trim() || "qid"; + const answersField = options.answersField?.trim() || "answer"; + const answers = readAcceptableAnswers(options.answersPath, idField, answersField); + const seenQueries = new Set(); + const outputRows: string[] = []; + + for (const [index, line] of readFileSync(resolve(options.queriesPath), "utf8") + .split(/\r?\n/u) + .entries()) { + if (!line.trim()) continue; + const [queryId, ...questionParts] = line.split("\t"); + if (!queryId || questionParts.length === 0) { + throw new Error(`Invalid query row ${index + 1}: expected query_idquestion`); + } + if (seenQueries.has(queryId)) { + throw new Error(`Duplicate query row for query ${queryId}`); + } + seenQueries.add(queryId); + const acceptable = answers.get(queryId); + if (!acceptable) throw new Error(`No acceptable answers found for query ${queryId}`); + outputRows.push( + JSON.stringify({ + query_id: queryId, + query: questionParts.join("\t"), + answer: JSON.stringify(acceptable), + }), + ); + } + + const outputPath = resolve(options.outputPath); + mkdirSync(dirname(outputPath), { recursive: true }); + writeFileSync(outputPath, `${outputRows.join("\n")}\n`, "utf8"); + return { outputPath, rowCount: outputRows.length }; +} + +function printHelp(): void { + console.log(`Usage: npm run adapt:multi-answer-ground-truth -- --queries --answers --output [options] + +Options: + --queries TSV rows in query_idquestion format + --answers Source JSONL containing an id and an array of acceptable answers + --output Piika ground-truth JSONL destination + --id-field Source JSONL id field (default: qid) + --answers-field Source JSONL answers field (default: answer) +`); +} + +function main(): void { + const args = parseArgs(process.argv.slice(2)); + if (args.help) { + printHelp(); + return; + } + const result = prepareMultiAnswerGroundTruth(args); + console.log(`Wrote ${result.rowCount} ground-truth rows to ${result.outputPath}`); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main(); +} diff --git a/src/benchmarks/install_manifest_entry.ts b/src/benchmarks/install_manifest_entry.ts new file mode 100644 index 0000000..52dc72e --- /dev/null +++ b/src/benchmarks/install_manifest_entry.ts @@ -0,0 +1,73 @@ +import { pathToFileURL } from "node:url"; + +import { installBenchmarkManifest } from "./installed"; + +type Args = { + manifestPath?: string; + root?: string; + dryRun: boolean; + force: boolean; +}; + +function parseArgs(argv: string[]): Args { + const args: Args = { dryRun: false, force: false }; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + const next = argv[index + 1]; + switch (arg) { + case "--manifest": + if (!next) throw new Error(`${arg} requires a value`); + args.manifestPath = next; + index += 1; + break; + case "--root": + if (!next) throw new Error(`${arg} requires a value`); + args.root = next; + index += 1; + break; + case "--dry-run": + case "--dryRun": + args.dryRun = true; + break; + case "--force": + args.force = true; + break; + case "--help": + case "-h": + printHelp(); + process.exit(0); + default: + throw new Error(`Unknown argument: ${arg}`); + } + } + if (!args.manifestPath) throw new Error("--manifest is required"); + return args; +} + +function printHelp(): void { + console.log(`Usage: npm run install:benchmark-manifest -- --manifest [options] + +Options: + --manifest BenchmarkDefinition JSON file to validate and install + --root Installed-manifest root (default: PIIKA_BENCHMARKS_DIR or data/prebuilt) + --dry-run Validate and print the destination without writing it + --force Replace an existing manifest with different contents +`); +} + +function main(): void { + const args = parseArgs(process.argv.slice(2)); + const result = installBenchmarkManifest({ + sourcePath: args.manifestPath as string, + root: args.root, + dryRun: args.dryRun, + force: args.force, + }); + console.log(`BENCHMARK=${result.benchmark.id}`); + console.log(`MANIFEST_PATH=${result.targetPath}`); + console.log(`WRITTEN=${result.written ? "1" : "0"}`); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main(); +} diff --git a/src/benchmarks/installed.ts b/src/benchmarks/installed.ts index 89d5d8d..9fe6475 100644 --- a/src/benchmarks/installed.ts +++ b/src/benchmarks/installed.ts @@ -1,5 +1,5 @@ -import { existsSync, readFileSync, readdirSync } from "node:fs"; -import { resolve } from "node:path"; +import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs"; +import { dirname, isAbsolute, relative, resolve } from "node:path"; import type { BenchmarkDefinition } from "./types"; function isRecord(value: unknown): value is Record { @@ -19,7 +19,7 @@ export function getInstalledBenchmarkRoot(): string { return resolve(process.env.PIIKA_BENCHMARKS_DIR?.trim() || "data/prebuilt"); } -function parseInstalledBenchmark(path: string): BenchmarkDefinition { +export function loadInstalledBenchmarkManifest(path: string): BenchmarkDefinition { let value: unknown; try { value = JSON.parse(readFileSync(path, "utf8")); @@ -34,6 +34,7 @@ function parseInstalledBenchmark(path: string): BenchmarkDefinition { const manifest = value as Partial; if ( !isNonEmptyString(manifest.id) || + !/^[a-z0-9][a-z0-9._-]*$/u.test(manifest.id) || !isNonEmptyString(manifest.displayName) || !isNonEmptyString(manifest.datasetId) || manifest.piSearchPromptVariant !== "plain_minimal" || @@ -68,6 +69,44 @@ export function loadInstalledBenchmarks(root = getInstalledBenchmarkRoot()): Ben .filter((entry) => entry.isDirectory()) .map((entry) => resolve(root, entry.name, "benchmark.json")) .filter(existsSync) - .map(parseInstalledBenchmark) + .map(loadInstalledBenchmarkManifest) .sort((left, right) => left.id.localeCompare(right.id)); } + +export function resolveInstalledBenchmarkManifestPath( + manifest: BenchmarkDefinition, + root = getInstalledBenchmarkRoot(), +): string { + const resolvedRoot = resolve(root); + const target = resolve(resolvedRoot, manifest.id, "benchmark.json"); + const relativeTarget = relative(resolvedRoot, target); + if (!relativeTarget || relativeTarget.startsWith("..") || isAbsolute(relativeTarget)) { + throw new Error(`Installed benchmark id escapes the manifest root: ${manifest.id}`); + } + return target; +} + +export function installBenchmarkManifest(options: { + sourcePath: string; + root?: string; + dryRun?: boolean; + force?: boolean; +}): { benchmark: BenchmarkDefinition; targetPath: string; written: boolean } { + const sourcePath = resolve(options.sourcePath); + const benchmark = loadInstalledBenchmarkManifest(sourcePath); + const targetPath = resolveInstalledBenchmarkManifestPath(benchmark, options.root); + const serialized = `${JSON.stringify(benchmark, null, 2)}\n`; + if (existsSync(targetPath) && readFileSync(targetPath, "utf8") === serialized) { + return { benchmark, targetPath, written: false }; + } + if (existsSync(targetPath) && !options.force) { + throw new Error( + `Installed benchmark manifest already exists with different contents: ${targetPath}. Pass --force to replace it.`, + ); + } + if (!options.dryRun) { + mkdirSync(dirname(targetPath), { recursive: true }); + writeFileSync(targetPath, serialized, "utf8"); + } + return { benchmark, targetPath, written: !options.dryRun }; +} diff --git a/src/evaluation/evaluate_run_with_pi.ts b/src/evaluation/evaluate_run_with_pi.ts index c28000d..83163d9 100644 --- a/src/evaluation/evaluate_run_with_pi.ts +++ b/src/evaluation/evaluate_run_with_pi.ts @@ -891,6 +891,14 @@ async function main() { timeoutSeconds: args.timeoutSeconds, isolatedAgentDir, }); + if (phase.exitCode !== 0 && !phase.timedOut) { + const stderrTail = phase.stderr.trim().split(/\r?\n/).slice(-20).join("\n"); + throw new Error( + `pi judge exited with status ${phase.exitCode ?? "null"} for query_id=${queryId} before producing a trusted eval record.${ + stderrTail ? `\nStderr tail:\n${stderrTail}` : "" + }`, + ); + } const judgeResponseText = getFinalAssistantText(phase.events); const judgeResult = phase.timedOut ? { diff --git a/src/evaluation/judge_prompt.ts b/src/evaluation/judge_prompt.ts index 995c09b..3125e13 100644 --- a/src/evaluation/judge_prompt.ts +++ b/src/evaluation/judge_prompt.ts @@ -47,6 +47,7 @@ export function createJudgePrompt(input: JudgePromptInput): string { "Do not use outside knowledge.", "Focus only on whether the response's final answer matches the correct answer.", "Allow harmless wording differences, equivalent formatting, and added correct detail.", + "The correct-answer field may be a JSON array of acceptable alternatives; matching any one alternative is correct.", "For numerical answers, allow small formatting differences and obvious equivalent forms.", "If the response does not contain a final answer you can extract, set extracted_final_answer to null and correct to false.", "", diff --git a/src/orchestration/benchmark_query_set_launch.ts b/src/orchestration/benchmark_query_set_launch.ts index 2b57bfd..f47ca0d 100644 --- a/src/orchestration/benchmark_query_set_launch.ts +++ b/src/orchestration/benchmark_query_set_launch.ts @@ -31,6 +31,7 @@ export type BenchmarkQuerySetLaunchArgs = { queryPath?: string; qrelsPath?: string; indexPath?: string; + suppliedDocBundlePath?: string; outputMode?: string; toolInterface?: string; rankedListDepth?: number; @@ -50,6 +51,7 @@ export type BenchmarkQuerySetLaunchPlan = { queryPath: string; qrelsPath: string; indexPath: string; + suppliedDocBundlePath?: string; outputMode: string; outputModes: PiSearchOutputModes; toolInterface: PiSearchToolInterface; @@ -182,6 +184,7 @@ export function resolveBenchmarkQuerySetLaunchPlan( queryPath: config.queryPath, qrelsPath: config.qrelsPath, indexPath: config.indexPath, + suppliedDocBundlePath: args.suppliedDocBundlePath ?? readEnv("SUPPLIED_DOC_BUNDLE"), outputMode: formatPiSearchOutputModes(outputModes), outputModes, toolInterface, @@ -210,6 +213,7 @@ export function buildBenchmarkQuerySetLaunchEnv( EXTENSION: plan.extensionPath, PI_BM25_INDEX_PATH: plan.indexPath, PROMPT_VARIANT: plan.piSearchPromptVariant, + ...(plan.suppliedDocBundlePath ? { SUPPLIED_DOC_BUNDLE: plan.suppliedDocBundlePath } : {}), PI_SEARCH_TOOL_INTERFACE: plan.toolInterface, OUTPUT_MODE: plan.outputMode, RANKED_LIST_DEPTH: String(plan.rankedListDepth), @@ -218,7 +222,7 @@ export function buildBenchmarkQuerySetLaunchEnv( } export function buildRunPiBenchmarkCommand(plan: BenchmarkQuerySetLaunchPlan): string[] { - return buildTsxCommand("src/orchestration/run_pi_benchmark.ts", [ + const command = buildTsxCommand("src/orchestration/run_pi_benchmark.ts", [ "--benchmark", plan.benchmarkId, "--querySet", @@ -247,6 +251,10 @@ export function buildRunPiBenchmarkCommand(plan: BenchmarkQuerySetLaunchPlan): s String(plan.rankedListDepth), ...(plan.rankedListCount ? ["--rankedListCount", String(plan.rankedListCount)] : []), ]); + if (plan.suppliedDocBundlePath) { + command.push("--supplied-doc-bundle", plan.suppliedDocBundlePath); + } + return command; } export function printBenchmarkQuerySetLaunchPlan(plan: BenchmarkQuerySetLaunchPlan): void { @@ -267,4 +275,7 @@ export function printBenchmarkQuerySetLaunchPlan(plan: BenchmarkQuerySetLaunchPl console.log(`OUTPUT_DIR=${plan.outputDir}`); console.log(`TIMEOUT_SECONDS=${plan.timeoutSeconds}`); console.log(`INDEX_PATH=${plan.indexPath}`); + if (plan.suppliedDocBundlePath) { + console.log(`SUPPLIED_DOC_BUNDLE=${plan.suppliedDocBundlePath}`); + } } diff --git a/src/orchestration/query_set.ts b/src/orchestration/query_set.ts index 93c0b05..98bc3fe 100644 --- a/src/orchestration/query_set.ts +++ b/src/orchestration/query_set.ts @@ -121,6 +121,11 @@ function parseArgs(argv: string[]): Args { args.indexPath = next; index += 1; break; + case "--supplied-doc-bundle": + if (!next) throw new Error(`${arg} requires a value`); + args.suppliedDocBundlePath = next; + index += 1; + break; case "--dryRun": case "--dry-run": args.dryRun = true; @@ -166,6 +171,7 @@ Options: --query-file Explicit override; wins over benchmark defaults --qrels Explicit override; wins over benchmark defaults --index-path Explicit override; wins over benchmark defaults + --supplied-doc-bundle JSONL bundle of supplied documents grouped per query --dry-run Benchmarks: diff --git a/src/orchestration/run_pi_benchmark.ts b/src/orchestration/run_pi_benchmark.ts index a456cee..47dac2f 100644 --- a/src/orchestration/run_pi_benchmark.ts +++ b/src/orchestration/run_pi_benchmark.ts @@ -13,6 +13,7 @@ import { tmpdir } from "node:os"; import { attachJsonlLineReader } from "../runtime/jsonl"; import { buildAnseriniBm25TcpExtensionConfig, + buildPyseriniRestExtensionConfig, parsePiSearchExtensionConfig, type PiSearchExtensionConfig, } from "../pi-search/config"; @@ -24,6 +25,7 @@ import { import { startBm25ServerTcp } from "../search-providers/anserini/bm25_server_process"; import { prepareIsolatedAgentDir } from "../runtime/pi_agent_dir"; import { + formatPiSearchPromptWithSuppliedContext, formatPiSearchOutputModes, formatPiSearchPrompt, hasPiSearchOutputMode, @@ -70,6 +72,7 @@ type BenchmarkRun = { search_backend_kind?: string; bm25_search_tool_mode?: string; bm25_render_excerpts?: string; + supplied_docids?: string[]; }; query_id: string; tool_call_counts: Record; @@ -115,6 +118,7 @@ type PersistedRunSetup = { model?: string; queryFile?: string; qrelsFile?: string; + suppliedDocBundle?: string; shardCount?: string; totalQueries?: string; timeoutSeconds?: string; @@ -144,6 +148,30 @@ type RunPiOptions = { extraEnv?: Record; }; +type QueryWithSuppliedContext = { + queryId: string; + query: string; + suppliedContext?: string; + suppliedDocids?: string[]; +}; + +type SuppliedDocBundleDoc = { + doc_id?: unknown; + cited_snippet?: unknown; + full_text?: unknown; +}; + +type SuppliedDocBundleGroup = { + docs?: unknown; +}; + +type SuppliedDocBundleRow = { + qid?: unknown; + question?: unknown; + groups?: unknown; + all_doc_ids?: unknown; +}; + type Bm25RpcConnection = { env: Record; endpoint: { @@ -198,8 +226,8 @@ const DEFAULT_BENCHMARK_ID = getDefaultBenchmarkId(); const DEFAULT_INDEX_PATH = resolveBenchmarkConfig({ benchmarkId: DEFAULT_BENCHMARK_ID }).indexPath; function getExternalBm25RpcConnection(): Bm25RpcConnection | null { - const host = process.env.PI_BM25_RPC_HOST?.trim(); - const rawPort = process.env.PI_BM25_RPC_PORT?.trim(); + const host = readEnv("PI_BM25_RPC_HOST"); + const rawPort = readEnv("PI_BM25_RPC_PORT"); if (!host && !rawPort) { return null; } @@ -224,6 +252,51 @@ function getExternalBm25RpcConnection(): Bm25RpcConnection | null { }; } +function readEnv(name: string): string | undefined { + const value = process.env[name]?.trim(); + return value ? value : undefined; +} + +function parseOptionalIntegerEnv(name: string): number | undefined { + const value = readEnv(name); + if (!value) return undefined; + const parsed = Number.parseInt(value, 10); + if (!Number.isFinite(parsed) || parsed <= 0) { + throw new Error(`${name} must be a positive integer; received ${value}`); + } + return parsed; +} + +function parsePyseriniRestReadMode(value: string | undefined): "full" | "paginated" | undefined { + if (!value) return undefined; + if (value === "full" || value === "paginated") return value; + throw new Error(`PYSERINI_REST_READ_MODE must be "full" or "paginated"; received ${value}`); +} + +function getPyseriniRestEnvShortcutConfig(): PiSearchExtensionConfig | null { + const baseUrl = readEnv("PYSERINI_REST_BASE_URL"); + const index = readEnv("PYSERINI_REST_INDEX"); + if (!baseUrl && !index) { + return null; + } + if (!baseUrl || !index) { + throw new Error( + "PYSERINI_REST_BASE_URL and PYSERINI_REST_INDEX must both be set to use the Pyserini REST backend shortcut.", + ); + } + + const tokenEnv = + readEnv("PYSERINI_REST_TOKEN_ENV") ?? + (readEnv("PYSERINI_API_TOKEN") ? "PYSERINI_API_TOKEN" : undefined); + return buildPyseriniRestExtensionConfig({ + baseUrl, + index, + tokenEnv, + searchMaxDocLength: parseOptionalIntegerEnv("PYSERINI_REST_SEARCH_MAX_DOC_LENGTH"), + readMode: parsePyseriniRestReadMode(readEnv("PYSERINI_REST_READ_MODE") ?? "paginated"), + }); +} + async function getBm25RpcConnection(cwd: string): Promise { const external = getExternalBm25RpcConnection(); if (external) { @@ -269,6 +342,19 @@ async function getSearchBackendConnection(cwd: string): Promise { + // External backends are owned by the caller. + }, + }; + } const bm25 = await getBm25RpcConnection(cwd); const config = buildAnseriniBm25TcpExtensionConfig({ @@ -303,6 +389,7 @@ function parseArgs(argv: string[]) { pi: "pi", limit: "0", timeoutSeconds: "900", + suppliedDocBundle: process.env.SUPPLIED_DOC_BUNDLE?.trim() || "", toolInterface: process.env.PI_SEARCH_TOOL_INTERFACE?.trim() || DEFAULT_PI_SEARCH_TOOL_INTERFACE, outputMode: process.env.OUTPUT_MODE?.trim() || "answer", rankedListDepth: process.env.RANKED_LIST_DEPTH?.trim() || String(DEFAULT_RANKED_LIST_DEPTH), @@ -312,22 +399,23 @@ function parseArgs(argv: string[]) { for (let i = 0; i < argv.length; i += 1) { const arg = argv[i]; if (!arg.startsWith("--")) continue; - const rawKey = arg.slice(2).replace(/-([a-z])/g, (_, letter: string) => letter.toUpperCase()); + const rawKey = arg.slice(2); + const normalizedKey = rawKey.replace(/-([a-z])/g, (_, letter: string) => letter.toUpperCase()); const key = - rawKey === "queryFile" + normalizedKey === "queryFile" ? "query" - : rawKey === "outputDir" + : normalizedKey === "outputDir" ? "outputDir" - : rawKey === "promptVariant" + : normalizedKey === "promptVariant" ? "promptVariant" - : rawKey === "timeoutSeconds" + : normalizedKey === "timeoutSeconds" ? "timeoutSeconds" - : rawKey === "indexPath" + : normalizedKey === "indexPath" ? "indexPath" - : rawKey; + : normalizedKey; const value = argv[i + 1]; if (!value || value.startsWith("--")) { - throw new Error(`Missing value for --${key}`); + throw new Error(`Missing value for --${rawKey}`); } out[key] = value; i += 1; @@ -377,6 +465,7 @@ function parseArgs(argv: string[]) { limit: Number.parseInt(out.limit, 10), timeoutSeconds: Number.parseInt(out.timeoutSeconds, 10), piSearchPromptVariant, + suppliedDocBundlePath: out.suppliedDocBundle ? resolve(out.suppliedDocBundle) : undefined, outputMode: formatPiSearchOutputModes(outputModes), outputModes, rankedListDepth, @@ -385,7 +474,7 @@ function parseArgs(argv: string[]) { }; } -function readQueries(tsvPath: string): Array<{ queryId: string; query: string }> { +function readQueries(tsvPath: string): QueryWithSuppliedContext[] { const text = readFileSync(tsvPath, "utf8"); return text .split(/\r?\n/) @@ -402,6 +491,78 @@ function readQueries(tsvPath: string): Array<{ queryId: string; query: string }> }); } +function assertString(value: unknown, label: string, lineNumber: number): string { + if (typeof value !== "string" || value.trim().length === 0) { + throw new Error( + `Invalid supplied document bundle line ${lineNumber}: ${label} must be a non-empty string`, + ); + } + return value; +} + +function parseSuppliedDocBundleRow(rawLine: string, lineNumber: number): QueryWithSuppliedContext { + const parsed = JSON.parse(rawLine) as SuppliedDocBundleRow; + const queryId = assertString(parsed.qid, "qid", lineNumber); + const query = assertString(parsed.question, "question", lineNumber); + if (!Array.isArray(parsed.groups)) { + throw new Error(`Invalid supplied document bundle line ${lineNumber}: groups must be an array`); + } + + const suppliedDocids = new Set(); + const sections: string[] = [ + "Provided documents:", + "These documents are grouped by document group. Treat them as supplied evidence, but do not assume they are sufficient or perfectly complete. You may use the search tools to look up additional evidence when the supplied documents leave a gap.", + ]; + + for (const [groupIndex, rawGroup] of parsed.groups.entries()) { + const group = rawGroup as SuppliedDocBundleGroup; + if (!Array.isArray(group.docs)) { + throw new Error( + `Invalid supplied document bundle line ${lineNumber}: groups[${groupIndex}].docs must be an array`, + ); + } + sections.push(``, `Document group ${groupIndex + 1}:`); + for (const [docIndex, rawDoc] of group.docs.entries()) { + const doc = rawDoc as SuppliedDocBundleDoc; + const docid = assertString( + doc.doc_id, + `groups[${groupIndex}].docs[${docIndex}].doc_id`, + lineNumber, + ); + const fullText = + typeof doc.full_text === "string" && doc.full_text.trim() + ? doc.full_text.trim() + : "(full_text was empty in the supplied document bundle)"; + const citedSnippet = + typeof doc.cited_snippet === "string" && doc.cited_snippet.trim() + ? doc.cited_snippet.trim() + : "(no cited snippet provided)"; + suppliedDocids.add(docid); + sections.push( + `Doc ${docIndex + 1} docid=${docid}`, + `Cited snippet: ${citedSnippet}`, + `Full text:`, + fullText, + ); + } + } + + return { + queryId, + query, + suppliedContext: sections.join("\n"), + suppliedDocids: Array.from(suppliedDocids), + }; +} + +function readSuppliedDocBundleQueries(jsonlPath: string): QueryWithSuppliedContext[] { + const text = readFileSync(jsonlPath, "utf8"); + return text + .split(/\r?\n/) + .filter(Boolean) + .map((line, index) => parseSuppliedDocBundleRow(line, index + 1)); +} + function formatPrompt( query: string, variant: PiSearchPromptVariant, @@ -409,13 +570,17 @@ function formatPrompt( toolInterface: PiSearchToolInterface, rankedListDepth: number, rankedListCount?: number, + suppliedContext?: string, ): string { - return formatPiSearchPrompt(query, variant, { + const options = { outputModes, toolInterface, rankedListDepth, rankedListCount, - }); + }; + return suppliedContext + ? formatPiSearchPromptWithSuppliedContext(query, suppliedContext, options) + : formatPiSearchPrompt(query, variant, options); } function readEvidenceQrels(path: string): EvidenceQrels { @@ -1005,6 +1170,7 @@ function finalizeRun( exitCode: number | null, timedOut: boolean, elapsedSeconds: number, + suppliedDocids: string[] = [], ): BenchmarkRun { const finalizedResults = [...normalizedResults]; @@ -1036,7 +1202,11 @@ function finalizeRun( const citedDocids = state.finalAssistantText ? extractCitationsFromText(state.finalAssistantText) : []; - const agentDocids = Array.from(new Set([...state.openedDocids, ...citedDocids])); + const agentDocids = Array.from( + new Set([...suppliedDocids, ...state.openedDocids, ...citedDocids]), + ); + const surfacedDocids = Array.from(new Set([...suppliedDocids, ...state.surfacedDocids])); + const previewedDocids = Array.from(new Set([...suppliedDocids, ...state.previewedDocids])); const parsedRankedList = hasPiSearchOutputMode(outputModes, "ranked_list") && state.finalAssistantText ? parseRankedDocidsFromAssistantText(state.finalAssistantText) @@ -1068,13 +1238,14 @@ function finalizeRun( : undefined, tool_interface: toolInterface, search_backend_kind: searchBackendKind, + supplied_docids: suppliedDocids, }, query_id: queryId, tool_call_counts: state.toolCallCounts, status, completion_source: completionSource, - surfaced_docids: Array.from(state.surfacedDocids), - previewed_docids: Array.from(state.previewedDocids), + surfaced_docids: surfacedDocids, + previewed_docids: previewedDocids, agent_docids: agentDocids, opened_docids: Array.from(state.openedDocids), cited_docids: citedDocids, @@ -1122,11 +1293,7 @@ function appendBenchmarkProgressEvent(event: BenchmarkProgressEvent): void { } function resolveEnvValue(name: string, fallback?: string): string | undefined { - const value = process.env[name]?.trim(); - if (value) { - return value; - } - return fallback; + return readEnv(name) ?? fallback; } function buildPersistedRunSetup(args: { @@ -1134,6 +1301,7 @@ function buildPersistedRunSetup(args: { model: string; queryPath: string; qrelsPath: string; + suppliedDocBundlePath?: string; totalQueries: number; timeoutSeconds: number; indexPath: string; @@ -1149,6 +1317,7 @@ function buildPersistedRunSetup(args: { model: args.model, queryFile: args.queryPath, qrelsFile: args.qrelsPath, + suppliedDocBundle: args.suppliedDocBundlePath, shardCount: resolveEnvValue("SHARD_COUNT"), totalQueries: String(args.totalQueries), timeoutSeconds: String(args.timeoutSeconds), @@ -1220,7 +1389,9 @@ async function main() { if (benchmarkManifestSnapshot.git_commit_short) { console.log(`Using gitCommit=${benchmarkManifestSnapshot.git_commit_short}`); } - let queries = readQueries(args.queryPath); + let queries = args.suppliedDocBundlePath + ? readSuppliedDocBundleQueries(args.suppliedDocBundlePath) + : readQueries(args.queryPath); if (args.limit > 0) { queries = queries.slice(0, args.limit); } @@ -1235,6 +1406,7 @@ async function main() { model: args.model, queryPath: args.queryPath, qrelsPath: args.qrelsPath, + suppliedDocBundlePath: args.suppliedDocBundlePath, totalQueries: queries.length, timeoutSeconds: args.timeoutSeconds, indexPath: args.indexPath, @@ -1292,7 +1464,7 @@ async function main() { } try { - for (const [index, { queryId, query }] of queries.entries()) { + for (const [index, { queryId, query, suppliedContext, suppliedDocids }] of queries.entries()) { const outputPath = getOutputPath(args.outputDir, queryId); const rawEventsPath = getRawEventsPath(args.outputDir, queryId); const stderrPath = getStderrPath(args.outputDir, queryId); @@ -1346,6 +1518,7 @@ async function main() { args.toolInterface, args.rankedListDepth, args.rankedListCount, + suppliedContext, ), queryId, timeoutSeconds: args.timeoutSeconds, @@ -1377,6 +1550,7 @@ async function main() { phase.exitCode, phase.timedOut, phase.elapsedSeconds, + suppliedDocids ?? [], ); } catch (error) { const message = error instanceof Error ? (error.stack ?? error.message) : String(error); @@ -1406,6 +1580,7 @@ async function main() { error.details.exitCode, error.details.timedOut, error.details.elapsedSeconds, + suppliedDocids ?? [], ); } else { run = finalizeRun( @@ -1427,6 +1602,7 @@ async function main() { null, false, (Date.now() - queryStartedAt) / 1000, + suppliedDocids ?? [], ); } } diff --git a/src/pi-search/agent_prompt.ts b/src/pi-search/agent_prompt.ts index d760559..95f20b4 100644 --- a/src/pi-search/agent_prompt.ts +++ b/src/pi-search/agent_prompt.ts @@ -54,6 +54,13 @@ function rankedListInstructions(depth: number, count?: number): string { export type PiSearchPromptVariant = "plain_minimal"; export type PiSearchOutputMode = "answer" | "ranked_list"; export type PiSearchOutputModes = readonly PiSearchOutputMode[]; +export type PiSearchPromptOptions = { + outputMode?: PiSearchOutputMode; + outputModes?: PiSearchOutputModes; + toolInterface?: PiSearchToolInterface; + rankedListDepth?: number; + rankedListCount?: number; +}; export const DEFAULT_PI_SEARCH_OUTPUT_MODES: PiSearchOutputModes = ["answer"]; @@ -89,13 +96,7 @@ export function hasPiSearchOutputMode( export function formatPiSearchPrompt( query: string, _variant: PiSearchPromptVariant = "plain_minimal", - options?: { - outputMode?: PiSearchOutputMode; - outputModes?: PiSearchOutputModes; - toolInterface?: PiSearchToolInterface; - rankedListDepth?: number; - rankedListCount?: number; - }, + options?: PiSearchPromptOptions, ): string { const modes = options?.outputModes ?? @@ -131,3 +132,63 @@ ${SUBMIT_NOW_REMINDER} Question: ${query}`; } + +export function formatPiSearchPromptWithSuppliedContext( + query: string, + suppliedContext: string, + options?: PiSearchPromptOptions, +): string { + const modes = + options?.outputModes ?? + (options?.outputMode ? [options.outputMode] : DEFAULT_PI_SEARCH_OUTPUT_MODES); + const wantsAnswer = hasPiSearchOutputMode(modes, "answer"); + const wantsRankedList = hasPiSearchOutputMode(modes, "ranked_list"); + const outputInstructions = [ + wantsAnswer ? ANSWER_INSTRUCTIONS : undefined, + wantsRankedList + ? rankedListInstructions(options?.rankedListDepth ?? 1000, options?.rankedListCount) + : undefined, + ].filter((instruction): instruction is string => Boolean(instruction)); + const responseSections = [ + wantsAnswer ? ANSWER_RESPONSE_FORMAT : undefined, + wantsRankedList ? RANKED_LIST_RESPONSE_FORMAT : undefined, + ].filter((section): section is string => Boolean(section)); + const toolInterface = options?.toolInterface ?? DEFAULT_PI_SEARCH_TOOL_INTERFACE; + const toolNames = + toolInterface === "pyserini-rest-2tool" + ? "search and read_document" + : "search, read_search_results, and read_document"; + const browseConstraint = + toolInterface === "pyserini-rest-2tool" + ? "" + : " Do not use read_search_results unless the single search result page is unusable without it."; + + return `You are a deep research agent answering a question using supplied documents and optional search tools. + +Workflow: +1. First use the supplied documents below. They are already supplied documents, grouped by document group, and include docids, cited snippets, and full text. Do not search just to rediscover or re-open these supplied docs. +2. Do not assume the supplied documents are sufficient or perfectly complete. +3. Most questions should be answerable directly from the supplied documents. If they establish the requested output, use them even if one original clue is not independently proven. +4. Search budget: at most one search call total, and at most one follow-up read_document call if that search returns a specific document you need.${browseConstraint} +5. Use that limited search budget only if the supplied documents do not establish the requested output, or if one narrowly targeted lookup is likely to resolve a concrete missing link. +6. Do not perform broad searches just to verify every clue after the supplied documents already establish the result. +7. Every call to ${toolNames} must include reason as the first argument. Keep it specific, under 100 words, and focused on the clue, gap, candidate, or ranking issue. +8. Satisfy every requested output below; use the same research pass for all outputs. +9. As soon as you have enough evidence and candidates, stop using tools and answer in plain assistant text. +10. Do not mention or rely on any gold answer field; answer only from supplied or retrieved evidence. + +${outputInstructions.join("\n\n")} + +Your final response must contain exactly these sections in this order: +${responseSections.join("\n\n")} + +Do not include any text before, after, or between those sections beyond the requested fields and ranked document lines. + +${SUBMIT_NOW_REMINDER} + +Question: ${query} + +Supplied context: +${suppliedContext} +`; +} diff --git a/tests/benchmark_query_set_launch.test.ts b/tests/benchmark_query_set_launch.test.ts index 42a9e2d..be4f8b0 100644 --- a/tests/benchmark_query_set_launch.test.ts +++ b/tests/benchmark_query_set_launch.test.ts @@ -3,6 +3,7 @@ import test from "node:test"; import { buildBenchmarkQuerySetLaunchEnv, + buildRunPiBenchmarkCommand, resolveBenchmarkQuerySetLaunchPlan, } from "../src/orchestration/benchmark_query_set_launch"; @@ -113,3 +114,16 @@ void test("benchmark launch composes answer and ranked-list outputs", () => { assert.equal(env.OUTPUT_MODE, "answer+ranked_list"); assert.equal(env.RANKED_LIST_DEPTH, "25"); }); + +void test("benchmark query-set launch plan propagates supplied document bundle", () => { + const plan = resolveBenchmarkQuerySetLaunchPlan({ + benchmarkId: "benchmark-template", + querySetId: "test", + suppliedDocBundlePath: "data/supplied-docs/test.jsonl", + }); + const env = buildBenchmarkQuerySetLaunchEnv(plan, {}); + const command = buildRunPiBenchmarkCommand(plan); + + assert.equal(env.SUPPLIED_DOC_BUNDLE, "data/supplied-docs/test.jsonl"); + assert.deepEqual(command.slice(-2), ["--supplied-doc-bundle", "data/supplied-docs/test.jsonl"]); +}); diff --git a/tests/installed_benchmark_manifest.test.ts b/tests/installed_benchmark_manifest.test.ts new file mode 100644 index 0000000..0e3f05b --- /dev/null +++ b/tests/installed_benchmark_manifest.test.ts @@ -0,0 +1,93 @@ +import assert from "node:assert/strict"; +import { mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; + +import { installBenchmarkManifest, loadInstalledBenchmarks } from "../src/benchmarks/installed"; + +function writeManifest(path: string, id = "custom-example"): void { + writeFileSync( + path, + JSON.stringify({ + id, + aliases: ["example"], + displayName: "Custom Example", + datasetId: "custom-example", + piSearchPromptVariant: "plain_minimal", + defaultQuerySetId: "test", + defaultQueryPath: "data/custom/queries.tsv", + querySets: { test: "data/custom/queries.tsv" }, + defaultQrelsPath: "data/custom/qrels.txt", + defaultGroundTruthPath: "data/custom/ground-truth.jsonl", + defaultIndexPath: "example-index", + managedPresets: {}, + setup: { steps: {} }, + retrievalEvaluation: { runFileBackend: "internal", runDirBackend: "internal" }, + judgeEvaluation: { + supportedModes: ["gold-answer", "reference-free"], + defaultMode: "gold-answer", + }, + }), + "utf8", + ); +} + +void test("generic manifest installer validates and installs custom benchmarks", () => { + const root = mkdtempSync(join(tmpdir(), "piika-install-manifest-")); + const sourcePath = join(root, "source.json"); + const installRoot = join(root, "installed"); + writeManifest(sourcePath); + + const result = installBenchmarkManifest({ sourcePath, root: installRoot }); + assert.equal(result.benchmark.id, "custom-example"); + assert.equal(result.written, true); + assert.equal(result.targetPath, join(installRoot, "custom-example", "benchmark.json")); + assert.equal(JSON.parse(readFileSync(result.targetPath, "utf8")).id, "custom-example"); + assert.equal(loadInstalledBenchmarks(installRoot)[0].defaultIndexPath, "example-index"); +}); + +void test("generic manifest installer dry-run validates without writing", () => { + const root = mkdtempSync(join(tmpdir(), "piika-install-manifest-dry-")); + const sourcePath = join(root, "source.json"); + writeManifest(sourcePath); + + const result = installBenchmarkManifest({ + sourcePath, + root: join(root, "installed"), + dryRun: true, + }); + assert.equal(result.written, false); + assert.equal(loadInstalledBenchmarks(join(root, "installed")).length, 0); +}); + +void test("generic manifest installer is idempotent and protects differing manifests", () => { + const root = mkdtempSync(join(tmpdir(), "piika-install-manifest-existing-")); + const sourcePath = join(root, "source.json"); + const installRoot = join(root, "installed"); + writeManifest(sourcePath); + installBenchmarkManifest({ sourcePath, root: installRoot }); + assert.equal(installBenchmarkManifest({ sourcePath, root: installRoot }).written, false); + + const changed = JSON.parse(readFileSync(sourcePath, "utf8")) as { displayName: string }; + changed.displayName = "Changed Example"; + writeFileSync(sourcePath, JSON.stringify(changed), "utf8"); + assert.throws( + () => installBenchmarkManifest({ sourcePath, root: installRoot }), + /already exists with different contents/, + ); + assert.equal( + installBenchmarkManifest({ sourcePath, root: installRoot, force: true }).written, + true, + ); +}); + +void test("installed benchmark ids cannot escape the manifest root", () => { + const root = mkdtempSync(join(tmpdir(), "piika-install-manifest-unsafe-")); + const sourcePath = join(root, "source.json"); + writeManifest(sourcePath, "../outside"); + assert.throws( + () => installBenchmarkManifest({ sourcePath, root: join(root, "installed") }), + /missing required fields/, + ); +}); diff --git a/tests/judge_prompt.test.ts b/tests/judge_prompt.test.ts new file mode 100644 index 0000000..57b6eff --- /dev/null +++ b/tests/judge_prompt.test.ts @@ -0,0 +1,28 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { createJudgePrompt } from "../src/evaluation/judge_prompt"; + +void test("gold-answer judge accepts any alternative in a JSON answer array", () => { + const prompt = createJudgePrompt({ + mode: "gold-answer", + question: "What is the answer?", + response: "Exact Answer: Alpha", + correctAnswer: '["Alpha","A"]', + }); + + assert.match(prompt, /JSON array of acceptable alternatives/); + assert.match(prompt, /matching any one alternative is correct/); + assert.match(prompt, /Correct answer: \["Alpha","A"\]/); +}); + +void test("reference-free judge does not mention acceptable gold alternatives", () => { + const prompt = createJudgePrompt({ + mode: "reference-free", + question: "What is the answer?", + response: "Exact Answer: Alpha", + }); + + assert.doesNotMatch(prompt, /acceptable alternatives/); + assert.match(prompt, /reference-free mode/); +}); diff --git a/tests/launcher_wrappers.test.ts b/tests/launcher_wrappers.test.ts index e8b057c..4e29ac3 100644 --- a/tests/launcher_wrappers.test.ts +++ b/tests/launcher_wrappers.test.ts @@ -763,6 +763,32 @@ void test("node benchmark query-set entrypoint resolves manifest-aligned default ]); }); +void test("node benchmark query-set entrypoint forwards supplied document bundle", () => { + const output = execFileSync( + "npx", + [ + "tsx", + "src/orchestration/query_set.ts", + "--dry-run", + "--benchmark", + "benchmark-template", + "--supplied-doc-bundle", + "data/supplied-docs/dev.jsonl", + ], + { + cwd: process.cwd(), + env: process.env, + encoding: "utf8", + }, + ); + + assert.match(output, /SUPPLIED_DOC_BUNDLE=data\/supplied-docs\/dev\.jsonl/); + assert.deepEqual(parseCommandJson(output).slice(-2), [ + "--supplied-doc-bundle", + "data/supplied-docs/dev.jsonl", + ]); +}); + void test("node low-level shared benchmark entrypoint resolves shared defaults", () => { const output = execFileSync( "npx", diff --git a/tests/prepare_multi_answer_ground_truth.test.ts b/tests/prepare_multi_answer_ground_truth.test.ts new file mode 100644 index 0000000..6f67051 --- /dev/null +++ b/tests/prepare_multi_answer_ground_truth.test.ts @@ -0,0 +1,68 @@ +import assert from "node:assert/strict"; +import { mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; + +import { prepareMultiAnswerGroundTruth } from "../src/adapters/prepare_multi_answer_ground_truth"; + +void test("multi-answer adapter joins queries and configurable answer fields", () => { + const root = mkdtempSync(join(tmpdir(), "piika-multi-answer-")); + const queriesPath = join(root, "queries.tsv"); + const answersPath = join(root, "answers.jsonl"); + const outputPath = join(root, "ground-truth", "answers.jsonl"); + writeFileSync(queriesPath, "q1\tFirst question?\nq2\tSecond\tquestion?\n", "utf8"); + writeFileSync( + answersPath, + [ + JSON.stringify({ query_id: "q1", acceptable: ["Alpha", " Alpha ", "", 42] }), + JSON.stringify({ query_id: "q2", acceptable: ["Beta"] }), + ].join("\n"), + "utf8", + ); + + const result = prepareMultiAnswerGroundTruth({ + queriesPath, + answersPath, + outputPath, + idField: "query_id", + answersField: "acceptable", + }); + + assert.equal(result.rowCount, 2); + assert.deepEqual( + readFileSync(outputPath, "utf8") + .trim() + .split("\n") + .map((line) => JSON.parse(line)), + [ + { query_id: "q1", query: "First question?", answer: '["Alpha"]' }, + { query_id: "q2", query: "Second\tquestion?", answer: '["Beta"]' }, + ], + ); +}); + +void test("multi-answer adapter rejects missing and duplicate answer rows", () => { + const root = mkdtempSync(join(tmpdir(), "piika-multi-answer-invalid-")); + const queriesPath = join(root, "queries.tsv"); + const answersPath = join(root, "answers.jsonl"); + writeFileSync(queriesPath, "q1\tQuestion?\n", "utf8"); + writeFileSync( + answersPath, + `${JSON.stringify({ qid: "q1", answer: ["Alpha"] })}\n${JSON.stringify({ + qid: "q1", + answer: ["Beta"], + })}\n`, + "utf8", + ); + + assert.throws( + () => + prepareMultiAnswerGroundTruth({ + queriesPath, + answersPath, + outputPath: join(root, "out.jsonl"), + }), + /Duplicate answer row for query q1/, + ); +}); diff --git a/tests/run_pi_benchmark_pi_search_contract.test.ts b/tests/run_pi_benchmark_pi_search_contract.test.ts index 2b022ac..0aaf394 100644 --- a/tests/run_pi_benchmark_pi_search_contract.test.ts +++ b/tests/run_pi_benchmark_pi_search_contract.test.ts @@ -349,3 +349,117 @@ void test("run_pi_benchmark records recoverable search-tool backend failures as ), ); }); + +void test("run_pi_benchmark accepts a generic supplied document bundle and records supplied docids", () => { + const root = mkdtempSync(join(tmpdir(), "run-pi-benchmark-supplied-docs-")); + const queryPath = join(root, "queries.tsv"); + const qrelsPath = join(root, "qrels.txt"); + const suppliedDocBundlePath = join(root, "supplied-docs.jsonl"); + const outputDir = join(root, "run"); + const fakePiPath = join(root, "fake-pi.sh"); + const promptCapturePath = join(root, "prompt.txt"); + + writeFileSync(queryPath, "1\talpha query\n", "utf8"); + writeFileSync(qrelsPath, "1 0 supplied-1 1\n", "utf8"); + writeFileSync( + suppliedDocBundlePath, + `${JSON.stringify({ + qid: "1", + question: "alpha query", + groups: [ + { + docs: [ + { + doc_id: "supplied-1", + cited_snippet: "snippet text", + full_text: "full supplied document text", + }, + ], + }, + ], + })}\n`, + "utf8", + ); + writeFileSync( + fakePiPath, + [ + "#!/bin/sh", + "prompt=''", + 'for arg do prompt="$arg"; done', + `printf '%s' "$prompt" > ${JSON.stringify(promptCapturePath)}`, + "printf '%s\\n' '{\"type\":\"session\"}'", + 'printf \'%s\\n\' \'{"type":"message_end","message":{"role":"assistant","content":[{"type":"text","text":"Explanation: used the supplied document. Exact Answer: alpha. Confidence: 90%"}]}}\'', + "printf '%s\\n' '{\"type\":\"agent_end\"}'", + ].join("\n"), + "utf8", + ); + chmodSync(fakePiPath, 0o755); + + const output = execFileSync( + "npx", + [ + "tsx", + "src/orchestration/run_pi_benchmark.ts", + "--benchmark", + "benchmark-template", + "--querySet", + "dev", + "--query", + queryPath, + "--qrels", + qrelsPath, + "--outputDir", + outputDir, + "--model", + "openai-codex/gpt-5.4-mini", + "--thinking", + "medium", + "--extension", + "src/extensions/pi_search.ts", + "--pi", + fakePiPath, + "--timeoutSeconds", + "5", + "--limit", + "1", + "--promptVariant", + "plain_minimal", + "--supplied-doc-bundle", + suppliedDocBundlePath, + ], + { + cwd: process.cwd(), + env: { + ...process.env, + PI_BM25_RPC_HOST: "127.0.0.1", + PI_BM25_RPC_PORT: "65535", + }, + encoding: "utf8", + }, + ); + + assert.match(output, /Finished 1\/1 queries/); + + const prompt = readFileSync(promptCapturePath, "utf8"); + assert.match(prompt, /using supplied documents and optional search tools/); + assert.match(prompt, /Provided documents:/); + assert.match(prompt, /grouped by document group/); + assert.match(prompt, /Document group 1:/); + assert.match(prompt, /docid=supplied-1/); + + const setup = JSON.parse(readFileSync(join(outputDir, "run_setup.json"), "utf8")) as { + suppliedDocBundle?: string; + }; + assert.equal(setup.suppliedDocBundle, suppliedDocBundlePath); + + const run = JSON.parse(readFileSync(join(outputDir, "1.json"), "utf8")) as { + metadata: { supplied_docids?: string[] }; + surfaced_docids: string[]; + previewed_docids: string[]; + agent_docids: string[]; + }; + assert.deepEqual(run.metadata.supplied_docids, ["supplied-1"]); + assert.deepEqual(run.surfaced_docids, ["supplied-1"]); + assert.deepEqual(run.previewed_docids, ["supplied-1"]); + assert.deepEqual(run.agent_docids, ["supplied-1"]); +});