Support generic supplied document injection

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## [Unreleased]
 
+### Added
+
+- Added generic supplied document bundles for benchmark runs via `--supplied-doc-bundle` or `SUPPLIED_DOC_BUNDLE`. Bundles inject per-query grouped evidence into the agent prompt, persist `metadata.supplied_docids`, and count supplied documents in surfaced, previewed, and agent-visible retrieval views.
+
 ### Changed
 
 - Updated the repository branding from `pi-serini` to `piika`, including the new logo and successor note while preserving historical release notes under their original name.

README.md

@@ -137,6 +137,17 @@ MODEL=openai-codex/gpt-5.4-mini \
 npm run run:benchmark:query-set
 ```
 
+To inject supplied evidence directly into each query prompt, pass a JSONL bundle with one row per query:
+
+```bash
+BENCHMARK=benchmark-template \
+QUERY_SET=test \
+MODEL=openai-codex/gpt-5.4-mini \
+npm run run:benchmark:query-set -- --supplied-doc-bundle data/my-run/supplied-docs.jsonl
+```
+
+Rows use `{ "qid", "question", "groups": [{ "docs": [{ "doc_id", "cited_snippet", "full_text" }] }] }`. Supplied document ids are recorded in run metadata and counted in surfaced, previewed, and agent-visible retrieval views. See [docs/running-benchmarks.md](docs/running-benchmarks.md#supplied-document-bundles) for details.
+
 ### BM25 tuning during benchmark runs
 
 Benchmark runs accept BM25 tuning through environment variables:

docs/running-benchmarks.md

@@ -141,6 +141,60 @@ MODEL=openai-codex/gpt-5.4-mini \
 npm run run:benchmark:query-set
 ```
 
+### Supplied document bundles
+
+Use a supplied document bundle when a run should inject known evidence directly into the agent prompt before it uses search tools. The runner still loads the normal benchmark query and qrels paths for identity and retrieval evaluation, but the question text and supplied documents come from the bundle rows.
+
+Pass the bundle either as a flag:
+
+```bash
+BENCHMARK=benchmark-template \
+QUERY_SET=test \
+MODEL=openai-codex/gpt-5.4-mini \
+npm run run:benchmark:query-set -- --supplied-doc-bundle data/my-run/supplied-docs.jsonl
+```
+
+Or as an environment variable:
+
+```bash
+SUPPLIED_DOC_BUNDLE=data/my-run/supplied-docs.jsonl \
+BENCHMARK=benchmark-template \
+QUERY_SET=test \
+MODEL=openai-codex/gpt-5.4-mini \
+npm run run:benchmark:query-set
+```
+
+The bundle is JSONL, one query per line. The example below is formatted for readability; in the file, write each row as one JSON object on a single line:
+
+```json
+{
+  "qid": "1",
+  "question": "Which city hosted the event?",
+  "groups": [
+    {
+      "docs": [
+        {
+          "doc_id": "doc-1",
+          "cited_snippet": "The event took place in Lisbon.",
+          "full_text": "Full document text..."
+        }
+      ]
+    }
+  ]
+}
+```
+
+Each row requires:
+
+- `qid`: query id used for the per-query run artifact
+- `question`: question sent to the agent
+- `groups`: ordered document groups, each with a `docs` array
+- `doc_id`: document id recorded in run metadata and retrieval views
+- `cited_snippet`: short evidence snippet shown before the full text
+- `full_text`: supplied document text shown in the prompt
+
+When a bundle is present, the prompt tells the agent to answer from supplied documents first and use a small optional search budget only when the supplied evidence leaves a concrete gap. Supplied `doc_id` values are recorded in `metadata.supplied_docids` and counted in `surfaced_docids`, `previewed_docids`, and `agent_docids` so retrieval summaries treat injected evidence as visible evidence, not hidden oracle state.
+
 ### Shared BM25 daemon
 
 Use this when multiple benchmark workers should reuse one BM25 server:

src/orchestration/benchmark_query_set_launch.ts

@@ -19,6 +19,7 @@ export type BenchmarkQuerySetLaunchArgs = {
   queryPath?: string;
   qrelsPath?: string;
   indexPath?: string;
+  suppliedDocBundlePath?: string;
 };
 
 export type BenchmarkQuerySetLaunchPlan = {
@@ -34,6 +35,7 @@ export type BenchmarkQuerySetLaunchPlan = {
   queryPath: string;
   qrelsPath: string;
   indexPath: string;
+  suppliedDocBundlePath?: string;
 };
 
 export function readEnv(name: string): string | undefined {
@@ -136,6 +138,7 @@ export function resolveBenchmarkQuerySetLaunchPlan(
     queryPath: config.queryPath,
     qrelsPath: config.qrelsPath,
     indexPath: config.indexPath,
+    suppliedDocBundlePath: args.suppliedDocBundlePath ?? readEnv("SUPPLIED_DOC_BUNDLE"),
   };
 }
 
@@ -159,11 +162,12 @@ export function buildBenchmarkQuerySetLaunchEnv(
     EXTENSION: plan.extensionPath,
     PI_BM25_INDEX_PATH: plan.indexPath,
     PROMPT_VARIANT: plan.piSearchPromptVariant,
+    ...(plan.suppliedDocBundlePath ? { SUPPLIED_DOC_BUNDLE: plan.suppliedDocBundlePath } : {}),
   };
 }
 
 export function buildRunPiBenchmarkCommand(plan: BenchmarkQuerySetLaunchPlan): string[] {
-  return buildTsxCommand("src/orchestration/run_pi_benchmark.ts", [
+  const command = buildTsxCommand("src/orchestration/run_pi_benchmark.ts", [
     "--benchmark",
     plan.benchmarkId,
     "--querySet",
@@ -187,6 +191,10 @@ export function buildRunPiBenchmarkCommand(plan: BenchmarkQuerySetLaunchPlan): s
     "--promptVariant",
     plan.piSearchPromptVariant,
   ]);
+  if (plan.suppliedDocBundlePath) {
+    command.push("--supplied-doc-bundle", plan.suppliedDocBundlePath);
+  }
+  return command;
 }
 
 export function printBenchmarkQuerySetLaunchPlan(plan: BenchmarkQuerySetLaunchPlan): void {
@@ -199,4 +207,7 @@ export function printBenchmarkQuerySetLaunchPlan(plan: BenchmarkQuerySetLaunchPl
   console.log(`OUTPUT_DIR=${plan.outputDir}`);
   console.log(`TIMEOUT_SECONDS=${plan.timeoutSeconds}`);
   console.log(`INDEX_PATH=${plan.indexPath}`);
+  if (plan.suppliedDocBundlePath) {
+    console.log(`SUPPLIED_DOC_BUNDLE=${plan.suppliedDocBundlePath}`);
+  }
 }

src/orchestration/query_set.ts

@@ -97,6 +97,11 @@ function parseArgs(argv: string[]): Args {
         args.indexPath = next;
         index += 1;
         break;
+      case "--supplied-doc-bundle":
+        if (!next) throw new Error(`${arg} requires a value`);
+        args.suppliedDocBundlePath = next;
+        index += 1;
+        break;
       case "--dryRun":
       case "--dry-run":
         args.dryRun = true;
@@ -138,6 +143,7 @@ Options:
   --query-file <path>            Explicit override; wins over benchmark defaults
   --qrels <path>                 Explicit override; wins over benchmark defaults
   --index-path <path>            Explicit override; wins over benchmark defaults
+  --supplied-doc-bundle <path>   JSONL bundle of supplied documents grouped per query
   --dry-run
 
 Benchmarks: