castorini · lilyjge · Jul 16, 2026 · Jul 18, 2026 · Jul 19, 2026
diff --git a/README.md b/README.md
@@ -137,6 +137,79 @@ MODEL=openai-codex/gpt-5.4-mini \
 npm run run:benchmark:query-set
 ```
 
+Benchmark launches default to the direct `pyserini-rest-2tool` interface: `search` returns
+ranked hits directly and `read_document` opens a selected document. This interface is also used
+when the configured backend is local Anserini BM25; the name describes the two-tool contract, not
+a requirement that the backend be remote. To reproduce the cached-search pagination condition,
+opt in explicitly with `PI_SEARCH_TOOL_INTERFACE=pi-serini-3tool` or
+`--tool-interface pi-serini-3tool`.
+
+### Ranked-list output mode
+
+For traditional retrieval benchmarks, set `OUTPUT_MODE=ranked_list` to ask the agent to return a single ranked docid list instead of a final answer. Each per-query JSON records `ranked_docids`, and the run directory receives `ranked_list.trec` for standard qrels-based evaluation.
+
+```bash
+OUTPUT_MODE=ranked_list \
+RANKED_LIST_DEPTH=1000 \
+BENCHMARK=msmarco-v1-passage \
+QUERY_SET=dl20 \
+MODEL=openai-codex/gpt-5.4-mini \
+npm run run:benchmark:query-set:shared-bm25
+```
+
+To request an exact-size ranking, set `RANKED_LIST_COUNT` in addition to the maximum
+depth. For example, this asks the agent for exactly 30 unique docids:
+
+```bash
+OUTPUT_MODE=ranked_list \
+RANKED_LIST_DEPTH=30 \
+RANKED_LIST_COUNT=30 \
+BENCHMARK=msmarco-v1-passage \
+QUERY_SET=dl20 \
+npm run run:benchmark:query-set:shared-bm25
+```
+
+The equivalent CLI option is `--ranked-list-count 30`. Outputs longer than the requested
+count are truncated. Shorter outputs are preserved and marked with
+`ranked_list_count_error` in the per-query JSON; Piika does not silently pad the ranking
+with documents the agent did not select.
+
+Omitting `RANKED_LIST_COUNT` is intentional and remains the default. In that condition,
+`RANKED_LIST_DEPTH` is only a maximum and the agent chooses the final list length. Exact count is
+not merely output formatting: a DL19 ablation found that requiring exactly 10 documents reduced
+search effort and lowered nDCG@10 relative to an otherwise matched optional-count run. Treat an
+exact count as an experimental retrieval constraint, not as a harmless serialization option. See
+the [recorded ablation](docs/msmarco-v1-passage-ranked-list-results.md#exact-count-ablation-on-dl19).
+
+Evaluate the generated run file with the existing trec_eval wrapper:
+
+```bash
+npx tsx src/evaluation/eval_retrieval_trec_eval.ts \
+  --benchmark msmarco-v1-passage \
+  --query-set dl20 \
+  --run-file runs/<run>/ranked_list.trec
+```
+
+The reproducible MS MARCO comparison protocol and recorded GPT-5.5 results are tracked in
+[`docs/msmarco-v1-passage-ranked-list-results.md`](docs/msmarco-v1-passage-ranked-list-results.md).
+
+### Composing answer and ranked-list outputs
+
+Output modes are atomic and composable. To have one agent research a query once and return both
+an answer and a document ranking, join the modes with `+`:
+
+```bash
+OUTPUT_MODE=answer+ranked_list \
+RANKED_LIST_DEPTH=30 \
+npm run run:benchmark:query-set -- --benchmark benchmark-template
+```
+
+The response contains an `Answer` section followed by a `Ranked List` section. Per-query JSON keeps
+the assistant response for answer evaluation and records `ranked_docids`; the run also produces
+`ranked_list.trec`. Internally this is the composition of the `answer` and `ranked_list` capabilities,
+not a separate third mode. A comma (`answer,ranked_list`) is also accepted and normalized to the
+canonical `answer+ranked_list` form.
+
 ### BM25 tuning during benchmark runs
 
 Benchmark runs accept BM25 tuning through environment variables:

diff --git a/docs/msmarco-v1-passage-ranked-list-results.md b/docs/msmarco-v1-passage-ranked-list-results.md
@@ -0,0 +1,202 @@
+# MS MARCO v1 Passage ranked-list results
+
+This note tracks agent-generated ranked-list runs against the three query sets in the
+[Pyserini MS MARCO v1 Passage reproduction matrix](https://castorini.github.io/pyserini/2cr/msmarco-v1-passage.html).
+
+## Comparison protocol
+
+The reproduction matrix reports these official metrics:
+
+| Query set            |           Topics | Metrics                             |
+| -------------------- | ---------------: | ----------------------------------- |
+| TREC DL 2019         | 43 judged topics | AP (`-l 2`), nDCG@10, R@1K (`-l 2`) |
+| TREC DL 2020         | 54 judged topics | AP (`-l 2`), nDCG@10, R@1K (`-l 2`) |
+| MS MARCO passage dev |     6,980 topics | RR@10, R@1K                         |
+
+Agent runs use `openai-codex/gpt-5.5`, thinking level `medium`, the Pyserini REST
+`msmarco-v1-passage` index, and the `pyserini-rest-2tool` interface. The two exposed
+tools are `search` and `read_document`. Search returns ranked hits directly; the model
+chooses a hit count from 1 to 100 for each call.
+
+Pyserini reference runs retrieve 1,000 documents per query. The current agent-only
+condition ranks only unique docids explicitly selected in the model's final response.
+Therefore AP, nDCG@10, and RR@10 are useful ranking comparisons, while R@1K is reported
+but is not depth-matched and must not be presented as a fair recall comparison. A future
+depth-1,000 condition should be labeled separately, for example by reranking a fixed
+1,000-document candidate set or by using a documented deterministic tail policy.
+
+For TREC DL, judged-only topic files are used to avoid model calls for unjudged topics.
+This is score-equivalent under the official qrels because `trec_eval` averages over the
+judged qids. The full topic files remain the benchmark defaults.
+
+## GPT-5.5 results
+
+| Query set | Status                |     AP | nDCG@10 |  RR@10 |   R@1K | Final depth                           |
+| --------- | --------------------- | -----: | ------: | -----: | -----: | ------------------------------------- |
+| DL20      | 54/54 completed       | 0.4492 |  0.7412 |    n/a | 0.5376 | min 3, median 20, mean 23.35, max 70  |
+| DL19      | 43/43 completed       | 0.3780 |  0.7381 |    n/a | 0.4811 | min 3, median 25, mean 27.47, max 60  |
+| dev       | 6,980/6,980 completed |    n/a |     n/a | 0.3300 | 0.6725 | min 0, median 24, mean 26.34, max 149 |
+
+DL20 run details:
+
+- Run: `runs/msmarco-v1-passage-dl20-judged-ranked-gpt55-pyserini-rest-2tool`
+- Merged run file: `merged/ranked_list.trec`
+- 316 search calls requested 6,535 hits total (mean 20.68 per call; range 10-50).
+- The merged run contains 1,261 unique ranked lines across 54 queries.
+
+DL19 run details:
+
+- Run: `runs/msmarco-v1-passage-dl19-judged-ranked-gpt55-pyserini-rest-2tool`
+- Merged run file: `merged/ranked_list.trec`
+- The merged run contains 1,181 unique ranked lines across 43 queries.
+
+Dev run details:
+
+- Run: `runs/msmarco-v1-passage-dev-ranked-gpt55-pyserini-rest-2tool-sharded8`
+- Eight shards processed all 6,980 topics with the same model, thinking level, REST index,
+  tool interface, and prompt as DL19/DL20.
+- The merged run contains 183,847 ranked lines across 6,947 nonempty rankings; 33
+  queries produced no parseable ranked list and score as misses.
+- A 16-shard launch was rejected after two shards failed initial credential resolution;
+  eight shards passed a 100-query stability checkpoint with no failures.
+
+## Reference points
+
+The Pyserini matrix reports the following representative depth-1,000 results:
+
+| System                  | DL19 AP | DL19 nDCG@10 | DL19 R@1K | DL20 AP | DL20 nDCG@10 | DL20 R@1K | dev RR@10 | dev R@1K |
+| ----------------------- | ------: | -----------: | --------: | ------: | -----------: | --------: | --------: | -------: |
+| BM25, k1=0.9, b=0.4     |  0.3013 |       0.5058 |    0.7501 |  0.2856 |       0.4796 |    0.7863 |    0.1840 |   0.8526 |
+| SPLADE++ EnsembleDistil |  0.5050 |       0.7308 |    0.8728 |  0.4999 |       0.7197 |    0.8998 |    0.3828 |   0.9831 |
+
+Reference values are snapshots from the linked Pyserini reproduction page and should be
+rechecked when updating this note.
+
+The locally generated depth-1,000 BM25 runs reproduce all eight BM25 values in the table
+exactly. Their summaries are under `evals/retrieval/msmarco-v1-passage/data/`.
+
+## Exact-count ablation on DL19
+
+### Motivation
+
+`RANKED_LIST_DEPTH` and `RANKED_LIST_COUNT` have deliberately different semantics:
+
+- `RANKED_LIST_DEPTH` is a maximum output depth. When count is omitted, the agent decides how
+  many unique documents it has enough evidence to rank.
+- `RANKED_LIST_COUNT` is an exact requirement. It changes the prompt, validates the returned
+  length, truncates overlong output, and records an error for short output.
+
+The exact-count option was introduced while answer and ranked-list outputs were being made
+atomic and composable. DL19 was used to test whether requiring exactly 10 documents was only a
+serialization constraint or whether it also changed the agent's retrieval policy. It changed the
+retrieval policy substantially.
+
+### Controlled conditions
+
+All corrected-prompt ablations used the same conditions except for output mode and exact count:
+
+- 43 judged TREC DL 2019 passage topics;
+- `openai-codex/gpt-5.5`, thinking level `medium`;
+- prompt variant `plain_minimal`;
+- four shards and a 900-second per-query timeout;
+- Pyserini REST index `msmarco-v1-passage`;
+- the direct `pyserini-rest-2tool` interface (`search` and `read_document`);
+- BM25 defaults k1=0.9 and b=0.4;
+- `trec_eval -c -l 2 -m map`, `trec_eval -c -m ndcg_cut.10`, and
+  `trec_eval -c -l 2 -m recall.1000`.
+
+The optional-count run set `RANKED_LIST_DEPTH=1000` and omitted `RANKED_LIST_COUNT`. The
+exact-10 runs set both depth and count to 10. Because the agent produces only the documents it
+selects, `recall.1000` means recall at the available run depth; it is not depth-matched across
+these conditions.
+
+### Results
+
+| Prompt / output condition                     | Exact count |         AP |    nDCG@10 |       R@1K | Returned depth                                        |
+| --------------------------------------------- | ----------: | ---------: | ---------: | ---------: | ----------------------------------------------------- |
+| Historical ranked-list-only                   |     omitted |     0.3780 |     0.7381 |     0.4811 | min 3, median 25, mean 27.47, max 60; 1,181 total     |
+| Pre-fix ranked-list-only                      |          10 |     0.2280 |     0.7148 |     0.2768 | exactly 10; 430 total                                 |
+| Pre-fix answer + ranked list                  |          10 |     0.2289 |     0.6910 |     0.2636 | exactly 10; 430 total                                 |
+| Corrected ranked-list-only                    |          10 |     0.2211 |     0.6946 |     0.2608 | exactly 10; 430 total                                 |
+| Corrected answer + ranked list                |          10 |     0.2474 |     0.7044 |     0.2765 | exactly 10; 430 total                                 |
+| **Corrected ranked-list-only, count omitted** | **omitted** | **0.3538** | **0.7343** | **0.4339** | **min 3, median 18, mean 23.86, max 73; 1,026 total** |
+
+The key controlled comparison is corrected ranked-list-only with count omitted versus corrected
+ranked-list-only at exactly 10. Omitting count improved aggregate nDCG@10 by **0.0397**, from
+0.6946 to 0.7343. It also nearly reproduced the historical optional-count result of 0.7381; the
+aggregate difference was only -0.0038.
+
+The optional-count ranking lengths were:
+
+```text
+3, 3, 3, 3, 5, 6, 10, 10, 10, 10, 12, 12, 12, 13, 14, 15, 15, 16,
+17, 17, 18, 18, 20, 20, 20, 20, 22, 24, 25, 27, 34, 37, 38, 39, 40,
+40, 46, 49, 49, 49, 51, 61, 73
+```
+
+All 43 optional-count lists contained unique document IDs. There were no parse, count, timeout,
+or completion errors.
+
+### Paired topic-level analysis
+
+Paired comparisons used per-topic nDCG@10, a 100,000-sample topic bootstrap for the mean
+difference confidence interval, and a 200,000-sample paired randomization test with random sign
+flips. Positive deltas favor the first condition named.
+
+| Comparison                                      | Mean delta | Wins / losses / ties |  Bootstrap 95% CI | Two-sided p |
+| ----------------------------------------------- | ---------: | -------------------: | ----------------: | ----------: |
+| Corrected optional count vs corrected exact-10  |    +0.0397 |          24 / 15 / 4 | [-0.0172, 0.1058] |       0.234 |
+| Corrected composed exact-10 vs ranked exact-10  |    +0.0098 |          21 / 20 / 2 | [-0.0507, 0.0782] |       0.790 |
+| Corrected exact-10 vs pre-fix exact-10          |    -0.0202 |          18 / 22 / 3 | [-0.0729, 0.0296] |       0.464 |
+| Corrected optional count vs historical optional |    -0.0038 |          19 / 18 / 6 | [-0.0589, 0.0534] |       0.897 |
+
+None of these single-run paired differences is statistically significant on 43 topics. The
+evidence should therefore be read as an ablation pattern, not as proof of a 0.0397 population
+effect. The important observations are that both optional-count runs independently reached about
+0.74 nDCG@10, both corrected exact-10 runs remained near 0.70, and composing answer with ranked
+list did not cause a consistent loss under the matched corrected prompt.
+
+### Behavioral evidence and interpretation
+
+The corrected optional-count agent did more retrieval work than corrected exact-10 ranked-list
+only:
+
+| Condition               | Tool calls | Search calls | Document reads | Sum elapsed seconds |
+| ----------------------- | ---------: | -----------: | -------------: | ------------------: |
+| Corrected exact-10      |        501 |          180 |            321 |             1,434.3 |
+| Corrected count omitted |        628 |          202 |            426 |             1,697.5 |
+
+The optional-count run used 25% more tool calls, 12% more searches, and 33% more document reads.
+Its system-surfaced macro recall was 0.5283, compared with the shallower exact-10 behavior. This
+supports the following mechanism: when asked for exactly 10 documents, the model can satisfy the
+task as soon as it has ten plausible candidates; when allowed to choose the output length, it
+continues gathering alternatives and then constructs a better-considered top ten. The effect is
+therefore upstream of TREC serialization. Truncating a deep, already-produced ranking to 10 would
+leave nDCG@10 unchanged, whereas prompting the agent to produce exactly 10 changes how it searches.
+
+Other explanations remain possible. These are stochastic agent runs, the prompt wording differs
+between exact and optional conditions, and the optional condition permits more output tokens. A
+larger repeated-run study would be required to separate stopping behavior, candidate-set breadth,
+and token-budget effects.
+
+### Recommendation
+
+- Keep `RANKED_LIST_COUNT` optional in APIs, environment variables, and CLI launchers.
+- Use omitted count as the default agent-retrieval condition; `RANKED_LIST_DEPTH` remains a safety
+  maximum rather than a target.
+- Treat exact count as a substantive retrieval ablation and label it in run names and metadata.
+- If a downstream consumer needs exactly 10 documents without changing agent behavior, first run
+  the optional-count condition and deterministically truncate the produced TREC ranking to 10.
+- Compare answer + ranked list against ranked-list-only under the same count policy. The matched
+  corrected exact-10 comparison provides no evidence that modular output composition itself harms
+  nDCG@10.
+
+### Run artifacts
+
+- Corrected exact-10 ranked-list-only:
+  `runs/msmarco-v1-passage-dl19-judged-ranked-top10-corrected-gpt55-pyserini-rest-2tool/merged`
+- Corrected exact-10 answer + ranked list:
+  `runs/msmarco-v1-passage-dl19-judged-answer-ranked-top10-corrected-gpt55-pyserini-rest-2tool/merged`
+- Corrected optional-count ranked-list-only:
+  `runs/msmarco-v1-passage-dl19-judged-ranked-optional-corrected-gpt55-pyserini-rest-2tool/merged`
+- Evaluation summaries mirror those paths under `evals/retrieval/msmarco-v1-passage/`.
diff --git a/scripts/benchmarks/msmarco_v1_passage/generate_query_slices.sh b/scripts/benchmarks/msmarco_v1_passage/generate_query_slices.sh
@@ -6,8 +6,14 @@ DATASET_ROOT="$ROOT/data/msmarco-v1-passage"
 QUERY_DIR="$DATASET_ROOT/queries"
 SOURCE_DL19="$DATASET_ROOT/source/topics.dl19-passage.tsv"
 SOURCE_DL20="$DATASET_ROOT/source/topics.dl20.tsv"
+SOURCE_DEV="$DATASET_ROOT/source/topics.dev-passage.tsv"
+QRELS_DL19="$DATASET_ROOT/qrels/qrels.dl19-passage.txt"
+QRELS_DL20="$DATASET_ROOT/qrels/qrels.dl20-passage.txt"
 DL19_QUERIES="$QUERY_DIR/dl19.tsv"
 DL20_QUERIES="$QUERY_DIR/dl20.tsv"
+DL19_JUDGED_QUERIES="$QUERY_DIR/dl19-judged.tsv"
+DL20_JUDGED_QUERIES="$QUERY_DIR/dl20-judged.tsv"
+DEV_QUERIES="$QUERY_DIR/dev.tsv"
 
 log() {
   printf '[setup:msmarco-v1-passage:query-slices] %s\n' "$*"
@@ -25,10 +31,22 @@ copy_query_set() {
   log "Wrote $label query set to $target_path"
 }
 
+write_judged_query_set() {
+  local source_path="$1"
+  local qrels_path="$2"
+  local target_path="$3"
+  local label="$4"
+  awk 'NR == FNR { judged[$1] = 1; next } $1 in judged' "$qrels_path" "$source_path" > "$target_path"
+  log "Wrote $label judged query set to $target_path"
+}
+
 main() {
   mkdir -p "$QUERY_DIR"
   copy_query_set "$SOURCE_DL19" "$DL19_QUERIES" 'dl19'
   copy_query_set "$SOURCE_DL20" "$DL20_QUERIES" 'dl20'
+  copy_query_set "$SOURCE_DEV" "$DEV_QUERIES" 'dev'
+  write_judged_query_set "$SOURCE_DL19" "$QRELS_DL19" "$DL19_JUDGED_QUERIES" 'dl19'
+  write_judged_query_set "$SOURCE_DL20" "$QRELS_DL20" "$DL20_JUDGED_QUERIES" 'dl20'
 }
 
 main "$@"
diff --git a/scripts/benchmarks/msmarco_v1_passage/setup.sh b/scripts/benchmarks/msmarco_v1_passage/setup.sh
@@ -19,12 +19,17 @@ DL19_TOPICS_URL="${MSMARCO_V1_PASSAGE_DL19_TOPICS_URL:-https://raw.githubusercon
 DL19_QRELS_URL="${MSMARCO_V1_PASSAGE_DL19_QRELS_URL:-https://raw.githubusercontent.com/castorini/anserini-tools/303096fd01ab1ee5048adc6b4a25d55761e6c860/topics-and-qrels/qrels.dl19-passage.txt}"
 DL20_TOPICS_URL="${MSMARCO_V1_PASSAGE_DL20_TOPICS_URL:-https://raw.githubusercontent.com/castorini/anserini-tools/303096fd01ab1ee5048adc6b4a25d55761e6c860/topics-and-qrels/topics.dl20.txt}"
 DL20_QRELS_URL="${MSMARCO_V1_PASSAGE_DL20_QRELS_URL:-https://raw.githubusercontent.com/castorini/anserini-tools/303096fd01ab1ee5048adc6b4a25d55761e6c860/topics-and-qrels/qrels.dl20-passage.txt}"
+DEV_TOPICS_URL="${MSMARCO_V1_PASSAGE_DEV_TOPICS_URL:-https://raw.githubusercontent.com/castorini/anserini-tools/12982126736f2ed7dc45bf30acb2af9fed13c0ef/topics-and-qrels/topics.msmarco-passage.dev-subset.txt}"
+DEV_QRELS_URL="${MSMARCO_V1_PASSAGE_DEV_QRELS_URL:-https://raw.githubusercontent.com/castorini/anserini-tools/12982126736f2ed7dc45bf30acb2af9fed13c0ef/topics-and-qrels/qrels.msmarco-passage.dev-subset.txt}"
 DL19_SOURCE_QUERIES="$SOURCE_DIR/topics.dl19-passage.tsv"
 DL20_SOURCE_QUERIES="$SOURCE_DIR/topics.dl20.tsv"
+DEV_SOURCE_QUERIES="$SOURCE_DIR/topics.dev-passage.tsv"
 DL19_QRELS_FILE="$QRELS_DIR/qrels.dl19-passage.txt"
 DL20_QRELS_FILE="$QRELS_DIR/qrels.dl20-passage.txt"
+DEV_QRELS_FILE="$QRELS_DIR/qrels.dev-passage.txt"
 DL19_BASELINE_RUN="$SOURCE_DIR/bm25_pure.dl19.trec"
 DL20_BASELINE_RUN="$SOURCE_DIR/bm25_pure.dl20.trec"
+DEV_BASELINE_RUN="$SOURCE_DIR/bm25_pure.dev.trec"
 
 log() {
   printf '[setup:msmarco-v1-passage] %s\n' "$*"
@@ -132,6 +137,11 @@ main() {
   log "Downloading MSMARCO dl20 qrels from $DL20_QRELS_URL"
   fetch_file "$DL20_QRELS_URL" "$DL20_QRELS_FILE"
 
+  log "Downloading MSMARCO dev topics from $DEV_TOPICS_URL"
+  fetch_file "$DEV_TOPICS_URL" "$DEV_SOURCE_QUERIES"
+  log "Downloading MSMARCO dev qrels from $DEV_QRELS_URL"
+  fetch_file "$DEV_QRELS_URL" "$DEV_QRELS_FILE"
+
   log 'Materializing benchmark query sets'
   bash scripts/benchmarks/msmarco_v1_passage/generate_query_slices.sh
 
@@ -148,17 +158,24 @@ main() {
 
   write_baseline_run "$DL19_SOURCE_QUERIES" "$DL19_BASELINE_RUN" 'dl19'
   write_baseline_run "$DL20_SOURCE_QUERIES" "$DL20_BASELINE_RUN" 'dl20'
+  write_baseline_run "$DEV_SOURCE_QUERIES" "$DEV_BASELINE_RUN" 'dev'
 
   log 'Setup complete.'
   log 'Prepared local outputs:'
   log "- $DL19_SOURCE_QUERIES"
   log "- $DL20_SOURCE_QUERIES"
+  log "- $DEV_SOURCE_QUERIES"
   log "- $DL19_QRELS_FILE"
   log "- $DL20_QRELS_FILE"
+  log "- $DEV_QRELS_FILE"
   log "- $DATASET_ROOT/queries/dl19.tsv"
   log "- $DATASET_ROOT/queries/dl20.tsv"
+  log "- $DATASET_ROOT/queries/dl19-judged.tsv"
+  log "- $DATASET_ROOT/queries/dl20-judged.tsv"
+  log "- $DATASET_ROOT/queries/dev.tsv"
   log "- $DL19_BASELINE_RUN"
   log "- $DL20_BASELINE_RUN"
+  log "- $DEV_BASELINE_RUN"
   log "- $INDEX_DIR"
   log "- $INDEX_ARCHIVE"
   log "- $ANSERINI_JAR"