diff --git a/apps/dev-playground/server/agents/query/evals/dataset.eval.ts b/apps/dev-playground/server/agents/query/evals/dataset.eval.ts
index 98cdd9d3e..25184283d 100644
--- a/apps/dev-playground/server/agents/query/evals/dataset.eval.ts
+++ b/apps/dev-playground/server/agents/query/evals/dataset.eval.ts
@@ -1,4 +1,8 @@
-import { defineEval, isJudgeConfigured } from "@databricks/appkit/beta";
+import {
+  defineEval,
+  isJudgeConfigured,
+  userTurns,
+} from "@databricks/appkit/beta";
 
 /**
  * Dataset-driven eval: runs once per row of a Databricks managed evaluation
@@ -13,17 +17,12 @@ import { defineEval, isJudgeConfigured } from "@databricks/appkit/beta";
  * Row shape produced by the MLflow managed-dataset UI:
  *   inputs        {"messages":[{"role":"user","content":"..."}]}
  *   expectations  {"guidelines":{"value":["...","..."]}}   (optional)
+ *
+ * A row's `messages` can be a full multi-turn conversation. We replay each USER
+ * turn in order against one shared thread (below); interleaved assistant turns
+ * in the row are ignored — the agent generates its own responses.
  */
 
-/** Pull the last user message out of an MLflow `{messages:[...]}` input. */
-function userMessage(input: Record<string, unknown>): string {
-  const messages = Array.isArray(input.messages)
-    ? (input.messages as Array<{ role?: string; content?: string }>)
-    : [];
-  const last = [...messages].reverse().find((m) => m.role === "user");
-  return last?.content ?? "";
-}
-
 /** Read `expectations.guidelines` — the UI wraps the array as `{value: [...]}`. */
 function guidelines(expected: Record<string, unknown> | undefined): string[] {
   const g = (expected?.guidelines as { value?: unknown } | undefined)?.value;
@@ -35,9 +34,12 @@ export default defineEval({
   // Point at your own managed evaluation dataset (catalog.schema.table).
   dataset: { table: "main.mario.appkit_eval_dataset" },
   async test(t) {
-    // One turn per row. For a multi-turn conversation, call `t.send` again
-    // (same thread); to start an independent turn in the same test, `t.reset()`.
-    await t.send(userMessage(t.input));
+    // Replay every user turn in the row against one thread, so the agent sees
+    // the accumulating conversation. A single-user-turn row sends once. The
+    // runner gives each row a fresh driver, so rows don't bleed into each other.
+    for (const turn of userTurns(t.input)) {
+      await t.send(turn);
+    }
     t.succeeded();
 
     // Each guideline is judged against the reply — gate by default, so a miss
diff --git a/packages/appkit/src/evals/dataset.ts b/packages/appkit/src/evals/dataset.ts
index 44d656c59..3f8c81cc6 100644
--- a/packages/appkit/src/evals/dataset.ts
+++ b/packages/appkit/src/evals/dataset.ts
@@ -21,6 +21,25 @@ export interface ReadEvalDatasetOptions {
   limit?: number;
 }
 
+/**
+ * Extract every user-message content, in order, from an MLflow
+ * `{"messages":[{"role":"user","content":"..."}]}` input. A dataset row can
+ * carry a full multi-turn conversation; replaying these against one thread (one
+ * `t.send` per returned string) lets the agent see the accumulating history.
+ *
+ * Only `role === "user"` turns are returned — any interleaved `assistant`/
+ * `system` messages in the row are ignored, since the agent generates its own
+ * responses; you never inject the dataset's assistant turns. A single-user-turn
+ * row yields a one-element array (backward compatible); a row with no `messages`
+ * yields `[]`.
+ */
+export function userTurns(input: Record<string, unknown>): string[] {
+  const messages = Array.isArray(input.messages)
+    ? (input.messages as Array<{ role?: string; content?: string }>)
+    : [];
+  return messages.filter((m) => m.role === "user").map((m) => m.content ?? "");
+}
+
 /** A managed eval dataset is a UC table; only 3-level names are valid. */
 const UC_TABLE = /^[A-Za-z0-9_]+\.[A-Za-z0-9_]+\.[A-Za-z0-9_]+$/;
 
diff --git a/packages/appkit/src/evals/define-eval.ts b/packages/appkit/src/evals/define-eval.ts
index 3e31cb191..815e08050 100644
--- a/packages/appkit/src/evals/define-eval.ts
+++ b/packages/appkit/src/evals/define-eval.ts
@@ -1,4 +1,4 @@
-import type { EvalDefinition } from "./types";
+import type { EvalConfig, EvalDefinition } from "./types";
 
 /**
  * Define an agent eval. Default-export the result from a
@@ -25,3 +25,8 @@ export function defineEval(def: EvalDefinition): EvalDefinition {
   }
   return def;
 }
+
+/** Define per-directory eval config. Default-export from `evals.config.ts`. */
+export function defineEvalConfig(config: EvalConfig): EvalConfig {
+  return config;
+}
diff --git a/packages/appkit/src/evals/discover.ts b/packages/appkit/src/evals/discover.ts
index 47029c3ad..63dfa0fc6 100644
--- a/packages/appkit/src/evals/discover.ts
+++ b/packages/appkit/src/evals/discover.ts
@@ -1,4 +1,4 @@
-import { type Dirent, readdirSync } from "node:fs";
+import { type Dirent, existsSync, readdirSync } from "node:fs";
 import path from "node:path";
 
 import { agentDirNames } from "../core/agent/agent-dirs";
@@ -14,6 +14,14 @@ export interface DiscoveredEval {
   agent: string;
 }
 
+/** A per-agent `evals.config.ts` found under `server/agents/<agent>/evals/`. */
+export interface DiscoveredEvalConfig {
+  /** Absolute path to the `evals.config.ts` file. */
+  file: string;
+  /** The agent id whose evals this config applies to. */
+  agent: string;
+}
+
 /** Recursively collect `*.eval.ts` files under `dir`. Empty when `dir` is absent. */
 function evalFilesIn(dir: string): string[] {
   try {
@@ -58,3 +66,28 @@ export function discoverEvalFiles(rootDir: string): DiscoveredEval[] {
     (a, b) => a.agent.localeCompare(b.agent) || a.id.localeCompare(b.id),
   );
 }
+
+/**
+ * Discover the per-agent `evals.config.ts` (from {@link defineEvalConfig}) at
+ * `<rootDir>/server/agents/<agent>/evals/evals.config.ts`. Config is per-agent:
+ * each agent's config applies only to that agent's evals. Agents without a
+ * config file are omitted. Returns a stable, sorted list.
+ */
+export function discoverEvalConfigs(rootDir: string): DiscoveredEvalConfig[] {
+  const agentsDir = path.join(rootDir, CODE_AGENTS_SOURCE_DIR);
+  const out: DiscoveredEvalConfig[] = [];
+
+  let entries: Dirent[];
+  try {
+    entries = readdirSync(agentsDir, { withFileTypes: true });
+  } catch {
+    return out;
+  }
+
+  for (const agent of agentDirNames(entries)) {
+    const file = path.join(agentsDir, agent, "evals", "evals.config.ts");
+    if (existsSync(file)) out.push({ file, agent });
+  }
+
+  return out.sort((a, b) => a.agent.localeCompare(b.agent));
+}
diff --git a/packages/appkit/src/evals/http-driver.ts b/packages/appkit/src/evals/http-driver.ts
index 4b113c359..1664ca25c 100644
--- a/packages/appkit/src/evals/http-driver.ts
+++ b/packages/appkit/src/evals/http-driver.ts
@@ -26,22 +26,34 @@ export interface HttpDriverOptions {
 /** Mutable running totals accumulated while draining one turn's SSE stream. */
 interface DriveState {
   reply: string;
-  toolCalls: string[];
-  seen: Set<string>;
+  /** Captured tool calls, keyed by `call_id ?? name` (dedupe). */
+  toolCalls: Map<string, ToolCall>;
   ok: boolean;
   traceId?: string;
 }
 
-/** Record a `function_call` output item once per call id (deduped). */
+/**
+ * Record a `function_call` output item, keyed by `call_id ?? name`, capturing
+ * its parsed arguments. A later `done` event's fuller args win over the initial
+ * `added` event's (often empty) args.
+ */
 function recordToolCall(
-  item: { type?: string; name?: string; call_id?: string } | undefined,
+  item:
+    | { type?: string; name?: string; call_id?: string; arguments?: string }
+    | undefined,
   state: DriveState,
 ): void {
   if (item?.type !== "function_call" || !item.name) return;
   const key = item.call_id ?? item.name;
-  if (state.seen.has(key)) return;
-  state.seen.add(key);
-  state.toolCalls.push(item.name);
+  const args = parseArgs(item.arguments);
+  const existing = state.toolCalls.get(key);
+  if (!existing) {
+    state.toolCalls.set(key, { name: item.name, args });
+  } else if (Object.keys(args).length > 0) {
+    // The initial `added` event may carry empty args while the later `done`
+    // carries the full arguments — keep the fuller set.
+    existing.args = args;
+  }
 }
 
 /** Apply an `appkit.metadata` event's thread/trace ids. */
@@ -54,6 +66,24 @@ function applyMetadata(
   if (data?.mlflowTraceId) state.traceId = data.mlflowTraceId;
 }
 
+/** A single captured tool call, deduped by `call_id ?? name`. */
+type ToolCall = { name: string; args: Record<string, unknown> };
+
+/** Parse a function-call `arguments` JSON string; `{}` on missing/invalid. */
+function parseArgs(raw: unknown): Record<string, unknown> {
+  if (typeof raw !== "string" || raw.trim() === "") return {};
+  try {
+    const parsed = JSON.parse(raw);
+    return parsed !== null &&
+      typeof parsed === "object" &&
+      !Array.isArray(parsed)
+      ? (parsed as Record<string, unknown>)
+      : {};
+  } catch {
+    return {};
+  }
+}
+
 /** Parse a single Responses-API SSE `data:` payload into the running totals. */
 function applyEvent(
   event: Record<string, unknown>,
@@ -73,6 +103,7 @@ function applyEvent(
       type?: string;
       name?: string;
       call_id?: string;
+      arguments?: string;
       content?: Array<{ text?: string }>;
     };
     recordToolCall(item, state);
@@ -147,13 +178,19 @@ export function createHttpDriver(options: HttpDriverOptions): EvalDriver {
           signal,
         });
       } catch {
-        return { reply: "", toolCalls: [], succeeded: false };
+        return {
+          reply: "",
+          toolCalls: [],
+          toolCallDetails: [],
+          succeeded: false,
+        };
       }
 
       if (!res.ok || !res.body) {
         return {
           reply: "",
           toolCalls: [],
+          toolCallDetails: [],
           succeeded: false,
           sessionId: threadId,
         };
@@ -161,8 +198,7 @@ export function createHttpDriver(options: HttpDriverOptions): EvalDriver {
 
       const state: DriveState = {
         reply: "",
-        toolCalls: [],
-        seen: new Set<string>(),
+        toolCalls: new Map<string, ToolCall>(),
         ok: true,
       };
       const setThread = (id: string) => {
@@ -193,9 +229,11 @@ export function createHttpDriver(options: HttpDriverOptions): EvalDriver {
       // throwing, so mark an aborted turn failed explicitly.
       if (signal.aborted) state.ok = false;
 
+      const toolCallDetails = [...state.toolCalls.values()];
       return {
         reply: state.reply,
-        toolCalls: state.toolCalls,
+        toolCalls: toolCallDetails.map((c) => c.name),
+        toolCallDetails,
         succeeded: state.ok,
         sessionId: threadId,
         traceId: state.traceId,
diff --git a/packages/appkit/src/evals/index.ts b/packages/appkit/src/evals/index.ts
index 3714c5378..afed3046a 100644
--- a/packages/appkit/src/evals/index.ts
+++ b/packages/appkit/src/evals/index.ts
@@ -11,9 +11,15 @@ export {
   type DatasetRow,
   type ReadEvalDatasetOptions,
   readEvalDataset,
+  userTurns,
 } from "./dataset";
-export { defineEval } from "./define-eval";
-export { type DiscoveredEval, discoverEvalFiles } from "./discover";
+export { defineEval, defineEvalConfig } from "./define-eval";
+export {
+  type DiscoveredEval,
+  type DiscoveredEvalConfig,
+  discoverEvalConfigs,
+  discoverEvalFiles,
+} from "./discover";
 export { createHttpDriver, type HttpDriverOptions } from "./http-driver";
 export {
   configureJudge,
@@ -34,6 +40,8 @@ export {
   formatEvalDetail,
   formatEvalHeadline,
   formatEvalResults,
+  formatResultsJson,
+  formatResultsJUnit,
   formatSummaryLine,
   summarize,
 } from "./report";
@@ -43,6 +51,7 @@ export {
   type EvalRunSummary,
   type RunEvalsOptions,
   runEvalsInDir,
+  runWithRetries,
 } from "./run-evals";
 export type {
   AssertionHandle,
diff --git a/packages/appkit/src/evals/report.ts b/packages/appkit/src/evals/report.ts
index 2e42c5b1c..8b903f0d9 100644
--- a/packages/appkit/src/evals/report.ts
+++ b/packages/appkit/src/evals/report.ts
@@ -7,6 +7,8 @@ export interface EvalSummary {
   skipped: number;
   /** True when no eval failed (skips don't count as failures). */
   allPassed: boolean;
+  /** Fraction of scored (non-skipped) evals that passed, 0..1 (1 when none scored). */
+  passRate: number;
 }
 
 export function summarize(results: EvalResult[]): EvalSummary {
@@ -18,12 +20,14 @@ export function summarize(results: EvalResult[]): EvalSummary {
     else if (r.passed) passed++;
     else failed++;
   }
+  const scored = passed + failed;
   return {
     total: results.length,
     passed,
     failed,
     skipped,
     allPassed: failed === 0,
+    passRate: scored === 0 ? 1 : passed / scored,
   };
 }
 
@@ -74,3 +78,67 @@ export function formatEvalResults(results: EvalResult[]): string {
   lines.push(formatSummaryLine(results));
   return lines.join("\n");
 }
+
+/**
+ * Render results as a machine-readable JSON report (2-space indented):
+ * `{ summary: EvalSummary, results: EvalResult[] }`. Faithful to the types —
+ * every field present on a result round-trips.
+ */
+export function formatResultsJson(results: EvalResult[]): string {
+  return JSON.stringify({ summary: summarize(results), results }, null, 2);
+}
+
+/** Escape a value for use in XML text/attribute content. */
+function escapeXml(value: string): string {
+  return value
+    .replace(/&/g, "&amp;")
+    .replace(/</g, "&lt;")
+    .replace(/>/g, "&gt;")
+    .replace(/"/g, "&quot;")
+    .replace(/'/g, "&apos;");
+}
+
+/** One-line reason a result failed: its error, else its failing gate labels. */
+function failureMessage(result: EvalResult): string {
+  if (result.error) return result.error;
+  const gates = result.assertions
+    .filter((a) => !a.pass && a.severity === "gate")
+    .map((a) => (a.detail ? `${a.label} — ${a.detail}` : a.label));
+  return gates.length ? gates.join("; ") : "eval failed";
+}
+
+/**
+ * Render results as JUnit XML for standard CI test reporters: a single
+ * `<testsuite name="appkit-agent-evals">` with one `<testcase>` per result.
+ * Failures carry a `<failure>` (error or failing-gate summary); skips a
+ * `<skipped>`. All attribute/text values are XML-escaped.
+ */
+export function formatResultsJUnit(results: EvalResult[]): string {
+  const s = summarize(results);
+  const lines: string[] = [];
+  lines.push('<?xml version="1.0" encoding="UTF-8"?>');
+  lines.push(
+    `<testsuite name="appkit-agent-evals" tests="${s.total}" failures="${s.failed}" skipped="${s.skipped}">`,
+  );
+  for (const r of results) {
+    const open = `  <testcase name="${escapeXml(r.id)}" classname="agent-eval"`;
+    if (r.skipped) {
+      lines.push(`${open}>`);
+      lines.push(
+        r.skipped.reason
+          ? `    <skipped message="${escapeXml(r.skipped.reason)}"/>`
+          : "    <skipped/>",
+      );
+      lines.push("  </testcase>");
+    } else if (!r.passed) {
+      const message = failureMessage(r);
+      lines.push(`${open}>`);
+      lines.push(`    <failure message="${escapeXml(message)}"/>`);
+      lines.push("  </testcase>");
+    } else {
+      lines.push(`${open}/>`);
+    }
+  }
+  lines.push("</testsuite>");
+  return lines.join("\n");
+}
diff --git a/packages/appkit/src/evals/run-eval.ts b/packages/appkit/src/evals/run-eval.ts
index c0ec2a57f..7129c9e40 100644
--- a/packages/appkit/src/evals/run-eval.ts
+++ b/packages/appkit/src/evals/run-eval.ts
@@ -3,6 +3,7 @@ import { judgeClosedQA, judgeCustom, judgeFactuality } from "./judge";
 import type {
   AssertionHandle,
   AssertionResult,
+  DriveResult,
   EvalDefinition,
   EvalDriver,
   EvalResult,
@@ -21,6 +22,32 @@ class SkipSignal extends Error {
   }
 }
 
+/** Rejects the test race when a per-eval timeout elapses. */
+class TimeoutSignal extends Error {
+  constructor(ms: number) {
+    super(`eval timed out after ${ms}ms`);
+    this.name = "TimeoutSignal";
+  }
+}
+
+/**
+ * Deep partial match: every key in `expected` is present in `actual` and equal,
+ * recursing into nested plain objects so extra actual keys are ignored.
+ */
+function deepContains(actual: unknown, expected: unknown): boolean {
+  if (isPlainObject(expected)) {
+    if (!isPlainObject(actual)) return false;
+    return Object.keys(expected).every((key) =>
+      deepContains(actual[key], expected[key]),
+    );
+  }
+  return actual === expected;
+}
+
+function isPlainObject(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
 export interface RunEvalOptions {
   /** Stable id for the eval (e.g. its file path relative to the evals dir). */
   id: string;
@@ -30,6 +57,11 @@ export interface RunEvalOptions {
   strict?: boolean;
   /** Dataset row bound to `t.input`/`t.expected` for dataset-driven evals. */
   row?: DatasetRow;
+  /**
+   * Runner-level default per-eval timeout (ms). `def.timeoutMs` wins over this;
+   * when both are unset the eval runs unbounded (current behavior).
+   */
+  timeoutMs?: number;
 }
 
 /**
@@ -45,6 +77,7 @@ export async function runEval(
   let reply = "";
   let lastInput = "";
   let toolCalls: string[] = [];
+  let toolCallDetails: DriveResult["toolCallDetails"] = [];
   let sessionId: string | undefined;
   let lastTraceId: string | undefined;
   let lastSucceeded = false;
@@ -98,6 +131,7 @@ export async function runEval(
       const r = await options.driver.send(message);
       reply = r.reply;
       toolCalls = r.toolCalls;
+      toolCallDetails = r.toolCallDetails;
       sessionId = r.sessionId;
       lastSucceeded = r.succeeded;
       if (r.traceId) lastTraceId = r.traceId;
@@ -138,6 +172,21 @@ export async function runEval(
         })`,
       );
     },
+    calledToolWith(name, expected) {
+      const matching = toolCallDetails.filter((c) => c.name === name);
+      const pass = matching.some((c) => deepContains(c.args, expected));
+      const seen = matching.length
+        ? matching.map((c) => JSON.stringify(c.args)).join(", ")
+        : "not called";
+      return record(
+        `calledToolWith(${name})`,
+        pass,
+        undefined,
+        `expected tool "${name}" to be called with ${JSON.stringify(
+          expected,
+        )} (args seen: ${seen})`,
+      );
+    },
     check(value: string, matcher: Matcher) {
       const m = matcher(value);
       return record("check", m.pass, m.score, m.detail);
@@ -172,8 +221,26 @@ export async function runEval(
     },
   };
 
+  // `def.timeoutMs` (per-eval) wins over the runner default; when both are
+  // unset the eval runs unbounded (undefined = no timeout).
+  const timeoutMs = def.timeoutMs ?? options.timeoutMs;
+  let timer: ReturnType<typeof setTimeout> | undefined;
+
   try {
-    await def.test(t);
+    if (timeoutMs === undefined) {
+      await def.test(t);
+    } else {
+      // Race the test against a timeout; on elapse the sentinel rejects and we
+      // convert it to a non-passing result. The timer is cleared in `finally`
+      // so it can't keep the process alive after the test settles.
+      const timeout = new Promise<never>((_, reject) => {
+        timer = setTimeout(
+          () => reject(new TimeoutSignal(timeoutMs)),
+          timeoutMs,
+        );
+      });
+      await Promise.race([Promise.resolve(def.test(t)), timeout]);
+    }
   } catch (err) {
     if (err instanceof SkipSignal) {
       return {
@@ -193,6 +260,8 @@ export async function runEval(
       error: err instanceof Error ? err.message : String(err),
       traceId: lastTraceId,
     };
+  } finally {
+    if (timer) clearTimeout(timer);
   }
 
   const passed = assertions.every(
diff --git a/packages/appkit/src/evals/run-evals.ts b/packages/appkit/src/evals/run-evals.ts
index c5c33203a..1b1d0cb85 100644
--- a/packages/appkit/src/evals/run-evals.ts
+++ b/packages/appkit/src/evals/run-evals.ts
@@ -3,14 +3,18 @@ import { pathToFileURL } from "node:url";
 import { MlflowClient } from "../connectors/mlflow";
 import type { WorkspaceClient } from "../workspace-client";
 import { type DatasetRow, readEvalDataset } from "./dataset";
-import { type DiscoveredEval, discoverEvalFiles } from "./discover";
+import {
+  type DiscoveredEval,
+  discoverEvalConfigs,
+  discoverEvalFiles,
+} from "./discover";
 import { createHttpDriver } from "./http-driver";
 import { configureJudge, teardownJudge } from "./judge";
 import { type ReportOutcome, reportToMlflow } from "./mlflow-report";
 import { createEvalRun, type FinishOutcome, finishEvalRun } from "./mlflow-run";
 import { mapPool } from "./pool";
 import { runEval } from "./run-eval";
-import type { EvalDefinition, EvalResult } from "./types";
+import type { EvalConfig, EvalDefinition, EvalResult } from "./types";
 
 export interface RunEvalsOptions {
   /** Project root containing `server/agents/`. Defaults to `process.cwd()`. */
@@ -19,12 +23,15 @@ export interface RunEvalsOptions {
   baseUrl: string;
   /** Substring filter on `<agent>/<id>` (or an exact agent id). */
   filter?: string;
+  /**
+   * Only run evals whose `tags` intersect this list. Empty/undefined runs all.
+   * Tags live on the eval def, so filtering happens after each file is loaded.
+   */
+  tags?: string[];
   /** Soft assertion failures also fail the eval. */
   strict?: boolean;
   /** Extra request headers for the driver (e.g. auth for a deployed app). */
   headers?: Record<string, string>;
-  /** Per-turn wall-clock timeout (ms) before a turn is failed. Defaults to 120s. */
-  timeoutMs?: number;
   /**
    * Max evals to drive concurrently. Each eval opens one stream to the app as
    * the same user, so keep this at or below the app's
@@ -58,6 +65,19 @@ export interface RunEvalsOptions {
   warehouseId?: string;
   /** Wall-clock timestamp (ms) for run create/finish — pass `Date.now()`. */
   now?: number;
+  /**
+   * Default per-eval timeout (ms): `runEval` races the whole test against it and
+   * it also caps each driver turn. A per-eval `def.timeoutMs` overrides it, and
+   * it wins over an agent's `evals.config.ts` `timeoutMs`. Unbounded when unset.
+   */
+  timeoutMs?: number;
+  /**
+   * Re-run an eval up to this many extra times when it fails on an
+   * infrastructure error (a thrown error or timeout — `result.error` set), to
+   * absorb transient turn/stream flakiness. Assertion failures are NEVER
+   * retried (a wrong reply is real signal, not flake). Defaults to `0`.
+   */
+  retries?: number;
   /** Progress callback, invoked as evals are discovered, started, and finished. */
   onEvent?: (event: EvalProgress) => void;
 }
@@ -75,12 +95,11 @@ export interface EvalRunSummary {
 }
 
 /**
- * Load a `*.eval.ts` file and return its default-exported {@link EvalDefinition}.
- * Uses tsx's programmatic loader so TypeScript eval files run without a build
- * step. The specifier is indirected so the type checker doesn't try to resolve
- * tsx's internal entry.
+ * Import a TypeScript file with tsx's programmatic loader so eval files run
+ * without a build step. The specifier is indirected so the type checker doesn't
+ * try to resolve tsx's internal entry.
  */
-async function loadEval(file: string): Promise<EvalDefinition> {
+async function tsImportFile(file: string): Promise<unknown> {
   const tsxApi = "tsx/esm/api";
   let tsImport: (specifier: string, parentURL: string) => Promise<unknown>;
   try {
@@ -92,8 +111,14 @@ async function loadEval(file: string): Promise<EvalDefinition> {
       "Running .eval.ts files requires `tsx`. Install it as a dev dependency (`pnpm add -D tsx`).",
     );
   }
+  return tsImport(pathToFileURL(file).href, import.meta.url);
+}
 
-  const mod = await tsImport(pathToFileURL(file).href, import.meta.url);
+/**
+ * Load a `*.eval.ts` file and return its default-exported {@link EvalDefinition}.
+ */
+async function loadEval(file: string): Promise<EvalDefinition> {
+  const mod = await tsImportFile(file);
   const def = resolveEvalDefault(mod);
   if (!def) {
     throw new Error(`${file}: must default-export defineEval({ test })`);
@@ -101,6 +126,37 @@ async function loadEval(file: string): Promise<EvalDefinition> {
   return def;
 }
 
+/**
+ * Load an `evals.config.ts` file and return its default-exported
+ * {@link EvalConfig}. A malformed/missing default surfaces as `undefined` so a
+ * bad config never aborts a whole run.
+ */
+async function loadEvalConfig(file: string): Promise<EvalConfig | undefined> {
+  const mod = await tsImportFile(file);
+  return resolveConfigDefault(mod);
+}
+
+/**
+ * Unwrap the config default export across module-interop shapes (see
+ * {@link resolveEvalDefault}). A config has no `.test`, so the first plain
+ * object reached through the `default` chain is taken as the config.
+ */
+export function resolveConfigDefault(mod: unknown): EvalConfig | undefined {
+  const seen = new Set<unknown>();
+  let candidate: unknown = mod;
+  for (let i = 0; i < 4 && candidate && !seen.has(candidate); i++) {
+    const next = (candidate as { default?: unknown }).default;
+    if (next === undefined) {
+      return typeof candidate === "object"
+        ? (candidate as EvalConfig)
+        : undefined;
+    }
+    seen.add(candidate);
+    candidate = next;
+  }
+  return undefined;
+}
+
 /**
  * Unwrap the eval default export across module-interop shapes. Depending on
  * whether the eval file is treated as ESM or CJS, the value lands at
@@ -133,17 +189,25 @@ async function runOne(
   options: RunEvalsOptions,
 ): Promise<EvalResult> {
   try {
-    // A fresh driver per row: each row is an independent conversation whose
-    // thread must not carry over the previous row's history. (Multiple
-    // `t.send`s within one row still share the thread — the driver's behavior.)
-    const driver = createHttpDriver({
-      baseUrl: options.baseUrl,
-      agent: def.agent ?? d.agent,
-      headers: options.headers,
-      mlflowRunId: runId,
-      timeoutMs: options.timeoutMs,
-    });
-    return await runEval(def, { id, driver, strict: options.strict, row });
+    // Retry only on an infrastructure error (`result.error` — a thrown error or
+    // timeout), to absorb transient turn/stream flakiness; assertion failures
+    // are real signal and returned on the first try. Each attempt gets a fresh
+    // driver, so its thread never carries over the failed attempt's history.
+    return await runWithRetries(options.retries ?? 0, () =>
+      runEval(def, {
+        id,
+        driver: createHttpDriver({
+          baseUrl: options.baseUrl,
+          agent: def.agent ?? d.agent,
+          headers: options.headers,
+          mlflowRunId: runId,
+          timeoutMs: options.timeoutMs,
+        }),
+        strict: options.strict,
+        row,
+        timeoutMs: options.timeoutMs,
+      }),
+    );
   } catch (err) {
     return {
       id,
@@ -237,6 +301,38 @@ async function runDiscovered(
   }
 }
 
+/**
+ * Run `attempt` up to `1 + retries` times, stopping as soon as it returns a
+ * result without an `error` (infra failures — thrown errors or timeouts — set
+ * `error`; assertion failures do not, so a failed-but-completed eval is returned
+ * on the first try and never retried). Returns the last result when every
+ * attempt errored. `retries` below 0 is treated as 0.
+ */
+export async function runWithRetries(
+  retries: number,
+  attempt: (attemptNumber: number) => Promise<EvalResult>,
+): Promise<EvalResult> {
+  const maxAttempts = 1 + Math.max(0, retries);
+  let result: EvalResult;
+  for (let n = 1; ; n++) {
+    result = await attempt(n);
+    if (!result.error || n >= maxAttempts) return result;
+  }
+}
+
+/**
+ * Whether an eval's `tags` satisfy a `--tag` filter: `true` when the filter is
+ * empty/undefined (no filtering), otherwise only when the eval shares at least
+ * one tag with it. An eval with no tags never matches a non-empty filter.
+ */
+export function matchesTags(
+  defTags: string[] | undefined,
+  filterTags: string[] | undefined,
+): boolean {
+  if (!filterTags || filterTags.length === 0) return true;
+  return defTags?.some((t) => filterTags.includes(t)) ?? false;
+}
+
 /** Configure the LLM judge when judge creds were supplied; otherwise a no-op. */
 async function maybeConfigureJudge(options: RunEvalsOptions): Promise<void> {
   if (!options.judge) return;
@@ -308,7 +404,61 @@ export async function runEvalsInDir(
   }
 
   const emit = options.onEvent ?? (() => {});
-  const total = discovered.length;
+
+  // Load each agent's `evals.config.ts` (best-effort, per-agent): its settings
+  // apply only to that agent's evals. A malformed/missing config never aborts
+  // the run — the agent just falls back to CLI options and built-in defaults.
+  const configs = new Map<string, EvalConfig>();
+  for (const c of discoverEvalConfigs(root)) {
+    try {
+      const cfg = await loadEvalConfig(c.file);
+      if (cfg) configs.set(c.agent, cfg);
+    } catch {
+      // Ignore: fall back to CLI options / defaults for this agent.
+    }
+  }
+
+  // Load each eval def and apply the `--tag` filter up front. Tags live on the
+  // def, so a tag miss removes the eval entirely (like the substring filter
+  // excludes files) rather than surfacing as a result. Load failures are kept
+  // so a broken file still reports as a non-passing result.
+  const loaded: Array<{
+    d: DiscoveredEval;
+    def: EvalDefinition;
+    loadError?: string;
+  }> = [];
+  for (const d of discovered) {
+    let def: EvalDefinition;
+    try {
+      def = await loadEval(d.file);
+    } catch (err) {
+      loaded.push({
+        d,
+        // No def loaded; placeholder def is never run (error short-circuits).
+        def: { test: () => {} },
+        loadError: err instanceof Error ? err.message : String(err),
+      });
+      continue;
+    }
+    if (!matchesTags(def.tags, options.tags)) continue;
+    loaded.push({ d, def });
+  }
+
+  // `evals.config.ts` `maxConcurrency` governs the single shared work pool, so
+  // it can't be applied per-agent without splitting the pool. The `--concurrency`
+  // flag wins; else the highest value any agent's config requests (the pool
+  // ceiling); else the built-in default.
+  const configMaxConcurrency = [...configs.values()]
+    .map((c) => c.maxConcurrency)
+    .filter((n): n is number => typeof n === "number")
+    .reduce<number | undefined>(
+      (max, n) => (max === undefined ? n : Math.max(max, n)),
+      undefined,
+    );
+  const concurrency =
+    options.concurrency ?? configMaxConcurrency ?? DEFAULT_CONCURRENCY;
+
+  const total = loaded.length;
   emit({ type: "discovered", total });
 
   // The judge sets OPENAI_* env vars globally (autoevals reads them per call),
@@ -334,21 +484,31 @@ export async function runEvalsInDir(
       emit({ type: "run-created", runId });
     }
 
-    // Run each eval through the bounded pool — one in-flight stream per eval, so
-    // the pool respects the server's per-user stream cap (see mapPool/concurrency).
-    // A dataset eval expands into per-row runs that execute serially within its
-    // slot; results preserve discovery order (mapPool writes by index) and row
-    // order within each file. `total` counts eval files, not dataset rows — per-row
-    // detail is carried in the result id (`[row i/n]`).
-    const perFile = await mapPool(
-      discovered,
-      options.concurrency ?? DEFAULT_CONCURRENCY,
-      async (d, index) => {
-        const fileResults: EvalResult[] = [];
-        await runDiscovered(d, index, total, runId, options, emit, fileResults);
-        return fileResults;
-      },
-    );
+    // Run each loaded (tag-filtered) eval through the bounded pool — one in-flight
+    // stream per eval, so the pool respects the server's per-user stream cap (see
+    // mapPool/concurrency). A dataset eval expands into per-row runs that execute
+    // serially within its slot; results preserve discovery order (mapPool writes
+    // by index) and row order within each file. Per-agent timeout is folded into
+    // the file's options (CLI wins over `evals.config.ts`; `def.timeoutMs` still
+    // overrides, applied inside runEval). `total` counts eval files, not dataset
+    // rows — per-row detail is carried in the result id (`[row i/n]`).
+    const perFile = await mapPool(loaded, concurrency, async ({ d }, index) => {
+      const fileResults: EvalResult[] = [];
+      const fileOptions: RunEvalsOptions = {
+        ...options,
+        timeoutMs: options.timeoutMs ?? configs.get(d.agent)?.timeoutMs,
+      };
+      await runDiscovered(
+        d,
+        index,
+        total,
+        runId,
+        fileOptions,
+        emit,
+        fileResults,
+      );
+      return fileResults;
+    });
     const results = perFile.flat();
 
     const summary: EvalRunSummary = { results };
diff --git a/packages/appkit/src/evals/tests/dataset.test.ts b/packages/appkit/src/evals/tests/dataset.test.ts
index b6787cd89..3c912cd6e 100644
--- a/packages/appkit/src/evals/tests/dataset.test.ts
+++ b/packages/appkit/src/evals/tests/dataset.test.ts
@@ -8,7 +8,7 @@ vi.mock("../../connectors", () => ({
   },
 }));
 
-import { readEvalDataset } from "../dataset";
+import { readEvalDataset, userTurns } from "../dataset";
 
 const client = {} as never;
 
@@ -90,3 +90,45 @@ describe("readEvalDataset", () => {
     expect(executeStatement).not.toHaveBeenCalled();
   });
 });
+
+describe("userTurns", () => {
+  test("returns all user contents in order", () => {
+    expect(
+      userTurns({
+        messages: [
+          { role: "user", content: "first" },
+          { role: "user", content: "second" },
+        ],
+      }),
+    ).toEqual(["first", "second"]);
+  });
+
+  test("ignores assistant/system turns, keeps user order", () => {
+    expect(
+      userTurns({
+        messages: [
+          { role: "system", content: "be helpful" },
+          { role: "user", content: "hi" },
+          { role: "assistant", content: "hello" },
+          { role: "user", content: "follow up" },
+        ],
+      }),
+    ).toEqual(["hi", "follow up"]);
+  });
+
+  test("a single user message yields one turn", () => {
+    expect(
+      userTurns({ messages: [{ role: "user", content: "only" }] }),
+    ).toEqual(["only"]);
+  });
+
+  test("missing content becomes an empty string", () => {
+    expect(userTurns({ messages: [{ role: "user" }] })).toEqual([""]);
+  });
+
+  test("missing or non-array messages yields []", () => {
+    expect(userTurns({})).toEqual([]);
+    expect(userTurns({ messages: "nope" })).toEqual([]);
+    expect(userTurns({ messages: [] })).toEqual([]);
+  });
+});
diff --git a/packages/appkit/src/evals/tests/discover.test.ts b/packages/appkit/src/evals/tests/discover.test.ts
index e93124d70..cd2d949af 100644
--- a/packages/appkit/src/evals/tests/discover.test.ts
+++ b/packages/appkit/src/evals/tests/discover.test.ts
@@ -4,7 +4,7 @@ import path from "node:path";
 
 import { afterEach, beforeEach, describe, expect, test } from "vitest";
 
-import { discoverEvalFiles } from "../discover";
+import { discoverEvalConfigs, discoverEvalFiles } from "../discover";
 
 let root: string;
 
@@ -42,3 +42,22 @@ describe("discoverEvalFiles", () => {
     expect(discoverEvalFiles(root)).toEqual([]);
   });
 });
+
+describe("discoverEvalConfigs", () => {
+  test("finds each agent's evals.config.ts, omits agents without one", () => {
+    write("server/agents/support/evals/basic.eval.ts");
+    write("server/agents/support/evals/evals.config.ts");
+    write("server/agents/analyst/evals/sql.eval.ts");
+
+    const found = discoverEvalConfigs(root);
+
+    expect(found.map((c) => c.agent)).toEqual(["support"]);
+    expect(found[0].file).toBe(
+      path.join(root, "server/agents/support/evals/evals.config.ts"),
+    );
+  });
+
+  test("returns empty when there is no server/agents dir", () => {
+    expect(discoverEvalConfigs(root)).toEqual([]);
+  });
+});
diff --git a/packages/appkit/src/evals/tests/http-driver.test.ts b/packages/appkit/src/evals/tests/http-driver.test.ts
index 99e9a1eed..5c57e7ae3 100644
--- a/packages/appkit/src/evals/tests/http-driver.test.ts
+++ b/packages/appkit/src/evals/tests/http-driver.test.ts
@@ -1,7 +1,15 @@
 import { createServer, type Server } from "node:http";
 import type { AddressInfo } from "node:net";
 
-import { afterAll, beforeAll, describe, expect, test } from "vitest";
+import {
+  afterAll,
+  afterEach,
+  beforeAll,
+  describe,
+  expect,
+  test,
+  vi,
+} from "vitest";
 
 import { createHttpDriver } from "../http-driver";
 
@@ -103,6 +111,22 @@ afterAll(async () => {
   await new Promise<void>((resolve) => server.close(() => resolve()));
 });
 
+afterEach(() => {
+  vi.restoreAllMocks();
+});
+
+/** Build a mock SSE `Response` from a list of Responses-API events. */
+function sseResponse(events: Array<Record<string, unknown>>): Response {
+  const body = events.map((e) => `data: ${JSON.stringify(e)}\n`).join("\n");
+  const stream = new ReadableStream<Uint8Array>({
+    start(controller) {
+      controller.enqueue(new TextEncoder().encode(body));
+      controller.close();
+    },
+  });
+  return new Response(stream, { status: 200 });
+}
+
 describe("createHttpDriver", () => {
   test("captures the reply and succeeds on a normal stream", async () => {
     const driver = createHttpDriver({ baseUrl, path: "/ok" });
@@ -142,4 +166,62 @@ describe("createHttpDriver", () => {
     expect(result.succeeded).toBe(false);
     expect(Date.now() - started).toBeLessThan(2000);
   });
+
+  test("captures tool-call names and parses their arguments", async () => {
+    // `added` carries empty args; `done` carries the full JSON string.
+    vi.spyOn(globalThis, "fetch").mockResolvedValue(
+      sseResponse([
+        {
+          type: "response.output_item.added",
+          item: {
+            type: "function_call",
+            name: "get_weather",
+            call_id: "c1",
+            arguments: "",
+          },
+        },
+        {
+          type: "response.output_item.done",
+          item: {
+            type: "function_call",
+            name: "get_weather",
+            call_id: "c1",
+            arguments: '{"city":"Paris","units":"metric"}',
+          },
+        },
+        { type: "response.output_text.delta", delta: "Sunny" },
+      ]),
+    );
+
+    const driver = createHttpDriver({ baseUrl: "http://localhost:3000" });
+    const result = await driver.send("weather in Paris?");
+
+    expect(result.reply).toBe("Sunny");
+    expect(result.toolCalls).toEqual(["get_weather"]);
+    expect(result.toolCallDetails).toEqual([
+      { name: "get_weather", args: { city: "Paris", units: "metric" } },
+    ]);
+    expect(result.succeeded).toBe(true);
+  });
+
+  test("defaults args to {} when the arguments JSON is malformed", async () => {
+    vi.spyOn(globalThis, "fetch").mockResolvedValue(
+      sseResponse([
+        {
+          type: "response.output_item.done",
+          item: {
+            type: "function_call",
+            name: "broken",
+            call_id: "c1",
+            arguments: "{not json",
+          },
+        },
+      ]),
+    );
+
+    const driver = createHttpDriver({ baseUrl: "http://localhost:3000" });
+    const result = await driver.send("go");
+
+    expect(result.toolCallDetails).toEqual([{ name: "broken", args: {} }]);
+  });
 });
diff --git a/packages/appkit/src/evals/tests/report.test.ts b/packages/appkit/src/evals/tests/report.test.ts
index 49b80fe89..c752e9d2f 100644
--- a/packages/appkit/src/evals/tests/report.test.ts
+++ b/packages/appkit/src/evals/tests/report.test.ts
@@ -1,6 +1,11 @@
 import { describe, expect, test } from "vitest";
 
-import { formatEvalResults, summarize } from "../report";
+import {
+  formatEvalResults,
+  formatResultsJson,
+  formatResultsJUnit,
+  summarize,
+} from "../report";
 import type { EvalResult } from "../types";
 
 const results: EvalResult[] = [
@@ -30,18 +35,25 @@ const results: EvalResult[] = [
 ];
 
 describe("eval reporting", () => {
-  test("summarize counts pass/fail/skip and allPassed", () => {
+  test("summarize counts pass/fail/skip, allPassed, and passRate", () => {
     expect(summarize(results)).toEqual({
       total: 3,
       passed: 1,
       failed: 1,
       skipped: 1,
       allPassed: false,
+      passRate: 0.5, // 1 passed of 2 scored; the skip is excluded
     });
   });
 
-  test("summarize allPassed is true when nothing failed", () => {
-    expect(summarize([results[0], results[2]]).allPassed).toBe(true);
+  test("summarize allPassed is true and passRate is 1 when nothing failed", () => {
+    const s = summarize([results[0], results[2]]);
+    expect(s.allPassed).toBe(true);
+    expect(s.passRate).toBe(1); // 1 passed of 1 scored (skip excluded)
+  });
+
+  test("passRate is 1 when every eval was skipped (nothing scored)", () => {
+    expect(summarize([results[2]]).passRate).toBe(1);
   });
 
   test("formatEvalResults shows status, failing assertions, and a summary line", () => {
@@ -52,4 +64,73 @@ describe("eval reporting", () => {
     expect(out).toContain("a/skip (skipped: no data)");
     expect(out).toContain("FAIL — 1 passed, 1 failed, 1 skipped (3 total)");
   });
+
+  test("formatResultsJson round-trips summary and result fields", () => {
+    const parsed = JSON.parse(formatResultsJson(results));
+    expect(parsed.summary).toEqual({
+      total: 3,
+      passed: 1,
+      failed: 1,
+      skipped: 1,
+      allPassed: false,
+      passRate: 0.5,
+    });
+    expect(parsed.results).toHaveLength(3);
+    const fail = parsed.results.find((r: EvalResult) => r.id === "a/fail");
+    expect(fail.passed).toBe(false);
+    expect(fail.assertions).toEqual([
+      {
+        label: "calledTool(x)",
+        severity: "gate",
+        pass: false,
+        detail: "not called",
+      },
+    ]);
+    const skip = parsed.results.find((r: EvalResult) => r.id === "a/skip");
+    expect(skip.skipped).toEqual({ reason: "no data" });
+  });
+
+  test("formatResultsJson round-trips a result's error field", () => {
+    const errored: EvalResult[] = [
+      {
+        id: "a/threw",
+        assertions: [],
+        passed: false,
+        error: "boom: turn failed",
+      },
+    ];
+    const parsed = JSON.parse(formatResultsJson(errored));
+    expect(parsed.results[0].error).toBe("boom: turn failed");
+    expect(parsed.summary.failed).toBe(1);
+  });
+
+  test("formatResultsJUnit emits suite counts, failure, skipped, and escapes special chars", () => {
+    const withSpecial: EvalResult[] = [
+      ...results,
+      {
+        id: 'a/b<x> & "q"',
+        assertions: [
+          {
+            label: "check",
+            severity: "gate",
+            pass: false,
+            detail: 'reply had <tag> & "quote"',
+          },
+        ],
+        passed: false,
+      },
+    ];
+    const xml = formatResultsJUnit(withSpecial);
+    expect(xml).toContain(
+      '<testsuite name="appkit-agent-evals" tests="4" failures="2" skipped="1">',
+    );
+    expect(xml).toContain('<testcase name="a/pass" classname="agent-eval"/>');
+    expect(xml).toContain("<failure");
+    expect(xml).toContain("<skipped");
+    // Special chars are escaped in both the id attribute and the failure message.
+    expect(xml).toContain("a/b&lt;x&gt; &amp; &quot;q&quot;");
+    expect(xml).toContain("reply had &lt;tag&gt; &amp; &quot;quote&quot;");
+    // Raw unescaped special chars from the id/detail must not leak through.
+    expect(xml).not.toContain("a/b<x>");
+  });
 });
diff --git a/packages/appkit/src/evals/tests/resolve-default.test.ts b/packages/appkit/src/evals/tests/resolve-default.test.ts
index 2dcc8a449..0c3927343 100644
--- a/packages/appkit/src/evals/tests/resolve-default.test.ts
+++ b/packages/appkit/src/evals/tests/resolve-default.test.ts
@@ -1,6 +1,10 @@
 import { describe, expect, test } from "vitest";
 
-import { resolveEvalDefault } from "../run-evals";
+import {
+  matchesTags,
+  resolveConfigDefault,
+  resolveEvalDefault,
+} from "../run-evals";
 
 const def = { description: "x", test: async () => {} };
 
@@ -24,3 +28,40 @@ describe("resolveEvalDefault (module interop)", () => {
     expect(resolveEvalDefault(null)).toBeUndefined();
   });
 });
+
+const config = { maxConcurrency: 4, timeoutMs: 1000 };
+
+describe("resolveConfigDefault (module interop)", () => {
+  test("pure ESM: mod.default", () => {
+    expect(resolveConfigDefault({ default: config })).toBe(config);
+  });
+
+  test("CJS __esModule double-wrap: mod.default.default", () => {
+    expect(
+      resolveConfigDefault({ default: { __esModule: true, default: config } }),
+    ).toBe(config);
+  });
+
+  test("no default export → undefined", () => {
+    expect(resolveConfigDefault(null)).toBeUndefined();
+    expect(resolveConfigDefault(undefined)).toBeUndefined();
+  });
+});
+
+describe("matchesTags", () => {
+  test("no filter runs everything", () => {
+    expect(matchesTags(["a"], undefined)).toBe(true);
+    expect(matchesTags(undefined, [])).toBe(true);
+    expect(matchesTags(undefined, undefined)).toBe(true);
+  });
+
+  test("matches when tags intersect the filter", () => {
+    expect(matchesTags(["smoke", "slow"], ["smoke"])).toBe(true);
+  });
+
+  test("excludes when tags don't intersect or the def has none", () => {
+    expect(matchesTags(["slow"], ["smoke"])).toBe(false);
+    expect(matchesTags(undefined, ["smoke"])).toBe(false);
+    expect(matchesTags([], ["smoke"])).toBe(false);
+  });
+});
diff --git a/packages/appkit/src/evals/tests/run-eval.test.ts b/packages/appkit/src/evals/tests/run-eval.test.ts
index 57b4aea7b..fdcff6269 100644
--- a/packages/appkit/src/evals/tests/run-eval.test.ts
+++ b/packages/appkit/src/evals/tests/run-eval.test.ts
@@ -11,6 +11,7 @@ function fakeDriver(result: Partial<DriveResult>): EvalDriver {
     send: async () => ({
       reply: "",
       toolCalls: [],
+      toolCallDetails: [],
       succeeded: true,
       ...result,
     }),
@@ -55,6 +56,82 @@ describe("runEval", () => {
     expect(result.assertions[0].pass).toBe(false);
   });
 
+  test("calledToolWith passes when a call's args deep-contain the expected", async () => {
+    const def = defineEval({
+      async test(t) {
+        await t.send("weather in Paris?");
+        t.calledToolWith("get_weather", { city: "Paris" });
+      },
+    });
+    const result = await runEval(def, {
+      id: "args-match",
+      driver: fakeDriver({
+        toolCalls: ["get_weather"],
+        toolCallDetails: [
+          { name: "get_weather", args: { city: "Paris", units: "metric" } },
+        ],
+      }),
+    });
+    expect(result.passed).toBe(true);
+    expect(result.assertions[0].pass).toBe(true);
+  });
+
+  test("calledToolWith fails when the tool was called with different args", async () => {
+    const def = defineEval({
+      async test(t) {
+        await t.send("weather in Paris?");
+        t.calledToolWith("get_weather", { city: "Paris" });
+      },
+    });
+    const result = await runEval(def, {
+      id: "args-mismatch",
+      driver: fakeDriver({
+        toolCalls: ["get_weather"],
+        toolCallDetails: [{ name: "get_weather", args: { city: "London" } }],
+      }),
+    });
+    expect(result.passed).toBe(false);
+    expect(result.assertions[0].pass).toBe(false);
+  });
+
+  test("calledToolWith fails when the tool was not called", async () => {
+    const def = defineEval({
+      async test(t) {
+        await t.send("hi");
+        t.calledToolWith("get_weather", { city: "Paris" });
+      },
+    });
+    const result = await runEval(def, {
+      id: "args-not-called",
+      driver: fakeDriver({ toolCalls: [], toolCallDetails: [] }),
+    });
+    expect(result.passed).toBe(false);
+    expect(result.assertions[0].pass).toBe(false);
+    expect(result.assertions[0].detail).toContain("not called");
+  });
+
+  test("calledToolWith matches nested args and ignores extra keys", async () => {
+    const def = defineEval({
+      async test(t) {
+        await t.send("book it");
+        t.calledToolWith("book", { where: { city: "Paris" } });
+      },
+    });
+    const result = await runEval(def, {
+      id: "args-nested",
+      driver: fakeDriver({
+        toolCalls: ["book"],
+        toolCallDetails: [
+          {
+            name: "book",
+            args: { where: { city: "Paris", zip: "75001" }, when: "today" },
+          },
+        ],
+      }),
+    });
+    expect(result.passed).toBe(true);
+  });
+
   test("soft failures don't fail the eval unless strict", async () => {
     const def = defineEval({
       async test(t) {
@@ -159,7 +236,12 @@ describe("runEval", () => {
   test("t.reset() forwards to the driver to start a fresh conversation", async () => {
     const reset = vi.fn();
     const driver: EvalDriver = {
-      send: async () => ({ reply: "", toolCalls: [], succeeded: true }),
+      send: async () => ({
+        reply: "",
+        toolCalls: [],
+        toolCallDetails: [],
+        succeeded: true,
+      }),
       reset,
     };
     const def = defineEval({
@@ -187,4 +269,63 @@ describe("runEval", () => {
     });
     expect(result.passed).toBe(true);
   });
+
+  test("def.timeoutMs turns a hanging test into a non-passing timeout result", async () => {
+    const def = defineEval({
+      timeoutMs: 20,
+      async test() {
+        // Never resolves; only the timeout can settle the eval.
+        await new Promise<void>(() => {});
+      },
+    });
+    const result = await runEval(def, { id: "hang", driver: fakeDriver({}) });
+    expect(result.passed).toBe(false);
+    expect(result.error).toBe("eval timed out after 20ms");
+  });
+
+  test("a fast eval passes well under the same timeout", async () => {
+    const def = defineEval({
+      timeoutMs: 20,
+      async test(t) {
+        await t.send("hi");
+        t.succeeded();
+      },
+    });
+    const result = await runEval(def, {
+      id: "fast",
+      driver: fakeDriver({ succeeded: true }),
+    });
+    expect(result.passed).toBe(true);
+    expect(result.error).toBeUndefined();
+  });
+
+  test("RunEvalOptions.timeoutMs applies when the def has none", async () => {
+    const def = defineEval({
+      async test() {
+        await new Promise<void>(() => {});
+      },
+    });
+    const result = await runEval(def, {
+      id: "runner-timeout",
+      driver: fakeDriver({}),
+      timeoutMs: 20,
+    });
+    expect(result.passed).toBe(false);
+    expect(result.error).toBe("eval timed out after 20ms");
+  });
+
+  test("def.timeoutMs overrides the runner-level default", async () => {
+    const def = defineEval({
+      timeoutMs: 15,
+      async test() {
+        await new Promise<void>(() => {});
+      },
+    });
+    const result = await runEval(def, {
+      id: "per-eval-wins",
+      driver: fakeDriver({}),
+      timeoutMs: 5000,
+    });
+    expect(result.error).toBe("eval timed out after 15ms");
+  });
 });
diff --git a/packages/appkit/src/evals/tests/run-with-retries.test.ts b/packages/appkit/src/evals/tests/run-with-retries.test.ts
new file mode 100644
index 000000000..f4bbd1614
--- /dev/null
+++ b/packages/appkit/src/evals/tests/run-with-retries.test.ts
@@ -0,0 +1,62 @@
+import { describe, expect, test } from "vitest";
+
+import { runWithRetries } from "../run-evals";
+import type { EvalResult } from "../types";
+
+describe("runWithRetries", () => {
+  const errored = (n: number): EvalResult => ({
+    id: `try-${n}`,
+    assertions: [],
+    passed: false,
+    error: "turn failed",
+  });
+  const ok = (n: number): EvalResult => ({
+    id: `try-${n}`,
+    assertions: [],
+    passed: true,
+  });
+  const assertionFail = (n: number): EvalResult => ({
+    id: `try-${n}`,
+    assertions: [{ label: "check", severity: "gate", pass: false }],
+    passed: false,
+  });
+
+  test("retries an infra error up to `retries` extra times, then returns the last", async () => {
+    let calls = 0;
+    const result = await runWithRetries(2, async (n) => {
+      calls = n;
+      return errored(n);
+    });
+    expect(calls).toBe(3); // 1 initial + 2 retries
+    expect(result.error).toBe("turn failed");
+  });
+
+  test("stops as soon as an attempt succeeds", async () => {
+    let calls = 0;
+    const result = await runWithRetries(5, async (n) => {
+      calls = n;
+      return n < 2 ? errored(n) : ok(n);
+    });
+    expect(calls).toBe(2); // errored once, then ok
+    expect(result.passed).toBe(true);
+  });
+
+  test("never retries an assertion failure (no error set)", async () => {
+    let calls = 0;
+    const result = await runWithRetries(3, async (n) => {
+      calls = n;
+      return assertionFail(n);
+    });
+    expect(calls).toBe(1);
+    expect(result.passed).toBe(false);
+  });
+
+  test("retries=0 runs exactly once", async () => {
+    let calls = 0;
+    await runWithRetries(0, async (n) => {
+      calls = n;
+      return errored(n);
+    });
+    expect(calls).toBe(1);
+  });
+});
diff --git a/packages/appkit/src/evals/types.ts b/packages/appkit/src/evals/types.ts
index fae6f4048..463c92533 100644
--- a/packages/appkit/src/evals/types.ts
+++ b/packages/appkit/src/evals/types.ts
@@ -56,6 +56,8 @@ export interface DriveResult {
   reply: string;
   /** Names of tools the agent called during the turn. */
   toolCalls: string[];
+  /** Tool calls with their parsed arguments, in call order. */
+  toolCallDetails: Array<{ name: string; args: Record<string, unknown> }>;
   /** Whether the turn completed without an agent/stream error. */
   succeeded: boolean;
   /** Thread/session id, when the driver exposes one. */
@@ -107,6 +109,15 @@ export interface TestContext {
   succeeded(): AssertionHandle;
   /** Assert a tool was called during the run (gate by default). */
   calledTool(name: string): AssertionHandle;
+  /**
+   * Assert a tool was called with arguments that deep-contain `expected`: every
+   * key in `expected` must equal the actual argument (recursively for nested
+   * objects), so extra arguments are ignored. Gate by default.
+   */
+  calledToolWith(
+    name: string,
+    expected: Record<string, unknown>,
+  ): AssertionHandle;
   /** Assert a value against a matcher, e.g. `t.check(t.reply, includes("Sunny"))`. */
   check(value: string, matcher: Matcher): AssertionHandle;
   /**
@@ -141,6 +152,13 @@ export interface EvalDefinition {
   description?: string;
   /** Target agent id. Defaults to the eval's parent `server/agents/<id>` dir. */
   agent?: string;
+  /** Free-form tags for filtering (see the runner's `tags` / `--tag` option). */
+  tags?: string[];
+  /**
+   * Per-eval timeout (ms): `runEval` races the test against it and records a
+   * non-passing result instead of hanging. Overrides the runner/CLI default.
+   */
+  timeoutMs?: number;
   /**
    * Run this eval once per row of a Databricks managed evaluation dataset (a
    * Unity Catalog `catalog.schema.table` with `inputs`/`expectations` columns).
@@ -152,6 +170,16 @@ export interface EvalDefinition {
   test(t: TestContext): Promise<void> | void;
 }
 
+/** Per-directory config from `evals.config.ts` (see {@link defineEvalConfig}). */
+export interface EvalConfig {
+  /** LLM judge config. Defaults to the agent's own serving endpoint. */
+  judge?: { model?: string };
+  /** Max evals to run concurrently. */
+  maxConcurrency?: number;
+  /** Default per-eval timeout. */
+  timeoutMs?: number;
+}
+
 /** The outcome of running one eval. */
 export interface EvalResult {
   id: string;
diff --git a/packages/shared/src/cli/commands/agent/eval.ts b/packages/shared/src/cli/commands/agent/eval.ts
index 5e0fefd3d..a638769ad 100644
--- a/packages/shared/src/cli/commands/agent/eval.ts
+++ b/packages/shared/src/cli/commands/agent/eval.ts
@@ -1,4 +1,6 @@
-import { Command } from "commander";
+import fs from "node:fs";
+
+import { Command, Option } from "commander";
 
 interface EvalRunSummary {
   results: unknown[];
@@ -25,6 +27,7 @@ interface EvalRunner {
     rootDir?: string;
     baseUrl: string;
     filter?: string;
+    tags?: string[];
     strict?: boolean;
     headers?: Record<string, string>;
     concurrency?: number;
@@ -37,6 +40,8 @@ interface EvalRunner {
     judge?: { host: string; token: string; model: string };
     workspaceClient?: unknown;
     warehouseId?: string;
+    timeoutMs?: number;
+    retries?: number;
     onEvent?: (event: EvalProgress) => void;
   }): Promise<EvalRunSummary>;
   resolveDatabricksAuth(opts: {
@@ -53,7 +58,9 @@ interface EvalRunner {
   evalGlyph(result: unknown): string;
   formatEvalDetail(result: unknown): string[];
   formatSummaryLine(results: unknown[]): string;
-  summarize(results: unknown[]): { allPassed: boolean };
+  formatResultsJson(results: unknown[]): string;
+  formatResultsJUnit(results: unknown[]): string;
+  summarize(results: unknown[]): { allPassed: boolean; passRate: number };
 }
 
 /**
@@ -89,6 +96,7 @@ interface EvalOptions {
   strict?: boolean;
   root?: string;
   header?: string[];
+  tag?: string[];
   profile?: string;
   databricksHost?: string;
   databricksToken?: string;
@@ -96,6 +104,11 @@ interface EvalOptions {
   judgeModel?: string;
   concurrency?: number;
   warehouseId?: string;
+  timeout?: string;
+  retries?: string;
+  minPassRate?: string;
+  reporter?: "text" | "json" | "junit";
+  output?: string;
 }
 
 /** Resolved Databricks host + bearer (either field may be absent). */
@@ -130,22 +143,29 @@ function resolveJudge(opts: EvalOptions, auth: Auth) {
     : undefined;
 }
 
-/** Progress reporter: stream each eval as it runs instead of going silent. */
+/**
+ * Progress reporter: stream each eval as it runs instead of going silent. In a
+ * machine reporter (json/junit) the live per-eval streaming is suppressed and
+ * banners go to stderr (via `info`), keeping stdout clean for the report.
+ */
 function makeProgressReporter(
   runner: EvalRunner,
   url: string,
+  machine: boolean,
+  info: (msg: string) => void,
 ): (event: EvalProgress) => void {
   return (event) => {
     switch (event.type) {
       case "discovered":
-        console.log(
+        info(
           `Running ${event.total} eval${event.total === 1 ? "" : "s"} against ${url}\n`,
         );
         break;
       case "run-created":
-        console.log(`MLflow evaluation run: ${event.runId}\n`);
+        info(`MLflow evaluation run: ${event.runId}\n`);
         break;
       case "result": {
+        if (machine) break;
         // One full line per completion — evals run concurrently, so a split
         // "start … glyph" prefix would interleave into garbage.
         console.log(
@@ -168,12 +188,17 @@ function formatFailureLine(f: {
   return `  ✗ trace ${f.traceId}: ${f.status ?? ""} ${f.error ?? ""}`.trim();
 }
 
-/** Print the MLflow assessment/finish outcome after a run that created one. */
+/**
+ * Print the MLflow assessment/finish outcome after a run that created one. The
+ * summary line goes through `info` (stderr under a machine reporter); per-trace
+ * failures and finish errors always go to stderr.
+ */
 function printMlflowOutcome(
   mlflow: NonNullable<EvalRunSummary["mlflow"]>,
+  info: (msg: string) => void,
 ): void {
   const { report, finish } = mlflow;
-  console.log(
+  info(
     `MLflow: ${report.written} assessment(s) written` +
       (report.skipped ? `, ${report.skipped} skipped` : "") +
       (report.failures.length ? `, ${report.failures.length} failed` : ""),
@@ -197,24 +222,49 @@ async function runAgentEval(
 ): Promise<void> {
   const runner = await loadRunner();
 
+  // Databricks credentials shared by auth resolution and the workspace client:
+  // an explicit flag/DATABRICKS_* env wins, else the SDK resolves from the CLI
+  // profile.
+  const credentials = {
+    profile: opts.profile ?? process.env.DATABRICKS_CONFIG_PROFILE,
+    host: opts.databricksHost ?? process.env.DATABRICKS_HOST,
+    token: opts.databricksToken ?? process.env.DATABRICKS_TOKEN,
+  };
+
   // Resolve Databricks host + bearer the AppKit-native way: an explicit
-  // host/token (or DATABRICKS_* env) wins; otherwise the SDK mints an OAuth
-  // token from the CLI profile — so no hand-set PAT is required.
-  const auth: Auth =
-    (await runner.resolveDatabricksAuth({
-      profile: opts.profile ?? process.env.DATABRICKS_CONFIG_PROFILE,
-      host: opts.databricksHost ?? process.env.DATABRICKS_HOST,
-      token: opts.databricksToken ?? process.env.DATABRICKS_TOKEN,
-    })) ?? {};
+  // host/token wins; otherwise the SDK mints an OAuth token from the CLI
+  // profile — so no hand-set PAT is required.
+  const auth: Auth = (await runner.resolveDatabricksAuth(credentials)) ?? {};
 
   // Managed-dataset reads: a workspace client (same profile/host/token) + a SQL
   // warehouse. Only needed by evals that declare `dataset`.
   const warehouseId = opts.warehouseId ?? process.env.DATABRICKS_WAREHOUSE_ID;
-  const workspaceClient = runner.resolveWorkspaceClient({
-    profile: opts.profile ?? process.env.DATABRICKS_CONFIG_PROFILE,
-    host: opts.databricksHost ?? process.env.DATABRICKS_HOST,
-    token: opts.databricksToken ?? process.env.DATABRICKS_TOKEN,
-  });
+  const workspaceClient = runner.resolveWorkspaceClient(credentials);
+
+  // Runner-level default per-eval timeout (ms). A per-eval `timeoutMs` wins.
+  const parsedTimeout = opts.timeout
+    ? Number.parseInt(opts.timeout, 10)
+    : undefined;
+  const timeoutMs =
+    parsedTimeout && parsedTimeout > 0 ? parsedTimeout : undefined;
+
+  // Extra attempts for evals that fail on an infra error (turn/timeout). Junk
+  // or negative input falls back to no retries.
+  const parsedRetries = opts.retries
+    ? Number.parseInt(opts.retries, 10)
+    : undefined;
+  const retries =
+    parsedRetries && parsedRetries > 0 ? parsedRetries : undefined;
+
+  // In a machine reporter (json/junit), stdout is reserved for the report (it
+  // may be piped), so human-facing lines go to stderr and the per-eval live
+  // streaming is suppressed. Text mode keeps its current stdout behavior.
+  const reporter = opts.reporter ?? "text";
+  const machine = reporter !== "text";
+  const info = (msg: string): void => {
+    if (machine) console.error(msg);
+    else console.log(msg);
+  };
 
   let summary: EvalRunSummary;
   try {
@@ -222,6 +272,7 @@ async function runAgentEval(
       rootDir: opts.root,
       baseUrl: opts.url,
       filter,
+      tags: opts.tag,
       strict: opts.strict,
       headers: opts.header ? parseHeaders(opts.header) : undefined,
       concurrency: opts.concurrency,
@@ -229,7 +280,9 @@ async function runAgentEval(
       judge: resolveJudge(opts, auth),
       workspaceClient,
       warehouseId,
-      onEvent: makeProgressReporter(runner, opts.url),
+      timeoutMs,
+      retries,
+      onEvent: makeProgressReporter(runner, opts.url, machine, info),
     });
   } catch (err) {
     // Setup failures (e.g. a bad --experiment for the MLflow run) reject before
@@ -241,18 +294,50 @@ async function runAgentEval(
     process.exitCode = 1;
     return;
   }
-  console.log(`\n${runner.formatSummaryLine(summary.results)}`);
+
+  // The final human summary always shows (stderr for machine reporters so it
+  // never pollutes the report on stdout/file).
+  info(`\n${runner.formatSummaryLine(summary.results)}`);
 
   if (summary.mlflow) {
-    printMlflowOutcome(summary.mlflow);
+    printMlflowOutcome(summary.mlflow, info);
   } else {
-    console.log(
+    info(
       "\nMLflow evaluation run skipped — pass --experiment (or set" +
         " MLFLOW_EXPERIMENT_ID) plus --profile/--databricks-host to create one.",
     );
   }
 
-  if (!runner.summarize(summary.results).allPassed) {
+  // Machine-readable report: build the string with a pure formatter, then emit
+  // it to --output <file> or stdout (kept clean of the human noise above).
+  if (machine) {
+    const report =
+      reporter === "json"
+        ? runner.formatResultsJson(summary.results)
+        : runner.formatResultsJUnit(summary.results);
+    if (opts.output) {
+      fs.writeFileSync(opts.output, `${report}\n`);
+      info(`Wrote ${reporter} report to ${opts.output}`);
+    } else {
+      process.stdout.write(`${report}\n`);
+    }
+  }
+
+  const stats = runner.summarize(summary.results);
+  const minPassRate = opts.minPassRate
+    ? Number.parseFloat(opts.minPassRate)
+    : undefined;
+  if (minPassRate !== undefined && !Number.isNaN(minPassRate)) {
+    // Threshold mode: gate on the aggregate pass rate rather than requiring
+    // every eval to pass.
+    const ok = stats.passRate >= minPassRate;
+    info(
+      `Pass rate ${(stats.passRate * 100).toFixed(0)}% (threshold ${(
+        minPassRate * 100
+      ).toFixed(0)}%) — ${ok ? "OK" : "below threshold"}`,
+    );
+    if (!ok) process.exitCode = 1;
+  } else if (!stats.allPassed) {
     process.exitCode = 1;
   }
 }
@@ -280,6 +365,10 @@ export const agentEvalCommand = new Command("eval")
     "--header <header...>",
     "Extra request header as 'Key: value' (repeatable)",
   )
+  .option(
+    "--tag <tag...>",
+    "Only run evals tagged with one of these tags (repeatable)",
+  )
   .option(
     "--profile <name>",
     "Databricks CLI profile to authenticate with via OAuth (default: DATABRICKS_CONFIG_PROFILE)",
@@ -304,4 +393,28 @@ export const agentEvalCommand = new Command("eval")
     "--judge-model <endpoint>",
     "Databricks serving endpoint to use as the LLM judge for t.judge.* (default: APPKIT_JUDGE_MODEL)",
   )
+  .option(
+    "--timeout <ms>",
+    "Default per-eval timeout in ms (a per-eval timeoutMs overrides it)",
+  )
+  .option(
+    "--retries <n>",
+    "Re-run an eval up to N times when it fails on an infra error (turn/timeout); assertion failures are not retried",
+  )
+  .option(
+    "--min-pass-rate <rate>",
+    "Gate on aggregate pass rate (0..1) instead of requiring every eval to pass; exit 1 when below",
+  )
+  .addOption(
+    new Option(
+      "--reporter <format>",
+      "Report format: text (live console), json (dashboards), or junit (CI test reporters)",
+    )
+      .choices(["text", "json", "junit"])
+      .default("text"),
+  )
+  .option(
+    "--output <file>",
+    "Write the json/junit report to this file instead of stdout (ignored for text)",
+  )
   .action(runAgentEval);