feat(appkit): eval suite features — concurrency, retries, reporters (stack 4/5)

apps/dev-playground/server/agents/query/evals/dataset.eval.ts

@@ -1,4 +1,8 @@
-import { defineEval, isJudgeConfigured } from "@databricks/appkit/beta";
+import {
+  defineEval,
+  isJudgeConfigured,
+  userTurns,
+} from "@databricks/appkit/beta";
 
 /**
  * Dataset-driven eval: runs once per row of a Databricks managed evaluation
@@ -13,17 +17,12 @@ import { defineEval, isJudgeConfigured } from "@databricks/appkit/beta";
  * Row shape produced by the MLflow managed-dataset UI:
  *   inputs        {"messages":[{"role":"user","content":"..."}]}
  *   expectations  {"guidelines":{"value":["...","..."]}}   (optional)
+ *
+ * A row's `messages` can be a full multi-turn conversation. We replay each USER
+ * turn in order against one shared thread (below); interleaved assistant turns
+ * in the row are ignored — the agent generates its own responses.
  */
 
-/** Pull the last user message out of an MLflow `{messages:[...]}` input. */
-function userMessage(input: Record<string, unknown>): string {
-  const messages = Array.isArray(input.messages)
-    ? (input.messages as Array<{ role?: string; content?: string }>)
-    : [];
-  const last = [...messages].reverse().find((m) => m.role === "user");
-  return last?.content ?? "";
-}
-
 /** Read `expectations.guidelines` — the UI wraps the array as `{value: [...]}`. */
 function guidelines(expected: Record<string, unknown> | undefined): string[] {
   const g = (expected?.guidelines as { value?: unknown } | undefined)?.value;
@@ -35,9 +34,12 @@ export default defineEval({
   // Point at your own managed evaluation dataset (catalog.schema.table).
   dataset: { table: "main.mario.appkit_eval_dataset" },
   async test(t) {
-    // One turn per row. For a multi-turn conversation, call `t.send` again
-    // (same thread); to start an independent turn in the same test, `t.reset()`.
-    await t.send(userMessage(t.input));
+    // Replay every user turn in the row against one thread, so the agent sees
+    // the accumulating conversation. A single-user-turn row sends once. The
+    // runner gives each row a fresh driver, so rows don't bleed into each other.
+    for (const turn of userTurns(t.input)) {
+      await t.send(turn);
+    }
     t.succeeded();
 
     // Each guideline is judged against the reply — gate by default, so a miss

packages/appkit/src/evals/dataset.ts

@@ -21,6 +21,25 @@ export interface ReadEvalDatasetOptions {
   limit?: number;
 }
 
+/**
+ * Extract every user-message content, in order, from an MLflow
+ * `{"messages":[{"role":"user","content":"..."}]}` input. A dataset row can
+ * carry a full multi-turn conversation; replaying these against one thread (one
+ * `t.send` per returned string) lets the agent see the accumulating history.
+ *
+ * Only `role === "user"` turns are returned — any interleaved `assistant`/
+ * `system` messages in the row are ignored, since the agent generates its own
+ * responses; you never inject the dataset's assistant turns. A single-user-turn
+ * row yields a one-element array (backward compatible); a row with no `messages`
+ * yields `[]`.
+ */
+export function userTurns(input: Record<string, unknown>): string[] {
+  const messages = Array.isArray(input.messages)
+    ? (input.messages as Array<{ role?: string; content?: string }>)
+    : [];
+  return messages.filter((m) => m.role === "user").map((m) => m.content ?? "");
+}
+
 /** A managed eval dataset is a UC table; only 3-level names are valid. */
 const UC_TABLE = /^[A-Za-z0-9_]+\.[A-Za-z0-9_]+\.[A-Za-z0-9_]+$/;
 

packages/appkit/src/evals/define-eval.ts

@@ -1,4 +1,4 @@
-import type { EvalDefinition } from "./types";
+import type { EvalConfig, EvalDefinition } from "./types";
 
 /**
  * Define an agent eval. Default-export the result from a
@@ -25,3 +25,8 @@ export function defineEval(def: EvalDefinition): EvalDefinition {
   }
   return def;
 }
+
+/** Define per-directory eval config. Default-export from `evals.config.ts`. */
+export function defineEvalConfig(config: EvalConfig): EvalConfig {
+  return config;
+}

packages/appkit/src/evals/discover.ts

@@ -1,4 +1,4 @@
-import { type Dirent, readdirSync } from "node:fs";
+import { type Dirent, existsSync, readdirSync } from "node:fs";
 import path from "node:path";
 
 import { agentDirNames } from "../core/agent/agent-dirs";
@@ -14,6 +14,14 @@ export interface DiscoveredEval {
   agent: string;
 }
 
+/** A per-agent `evals.config.ts` found under `server/agents/<agent>/evals/`. */
+export interface DiscoveredEvalConfig {
+  /** Absolute path to the `evals.config.ts` file. */
+  file: string;
+  /** The agent id whose evals this config applies to. */
+  agent: string;
+}
+
 /** Recursively collect `*.eval.ts` files under `dir`. Empty when `dir` is absent. */
 function evalFilesIn(dir: string): string[] {
   try {
@@ -58,3 +66,28 @@ export function discoverEvalFiles(rootDir: string): DiscoveredEval[] {
     (a, b) => a.agent.localeCompare(b.agent) || a.id.localeCompare(b.id),
   );
 }
+
+/**
+ * Discover the per-agent `evals.config.ts` (from {@link defineEvalConfig}) at
+ * `<rootDir>/server/agents/<agent>/evals/evals.config.ts`. Config is per-agent:
+ * each agent's config applies only to that agent's evals. Agents without a
+ * config file are omitted. Returns a stable, sorted list.
+ */
+export function discoverEvalConfigs(rootDir: string): DiscoveredEvalConfig[] {
+  const agentsDir = path.join(rootDir, CODE_AGENTS_SOURCE_DIR);
+  const out: DiscoveredEvalConfig[] = [];
+
+  let entries: Dirent[];
+  try {
+    entries = readdirSync(agentsDir, { withFileTypes: true });
+  } catch {
+    return out;
+  }
+
+  for (const agent of agentDirNames(entries)) {
+    const file = path.join(agentsDir, agent, "evals", "evals.config.ts");
+    if (existsSync(file)) out.push({ file, agent });
+  }
+
+  return out.sort((a, b) => a.agent.localeCompare(b.agent));
+}

packages/appkit/src/evals/http-driver.ts

@@ -26,22 +26,34 @@ export interface HttpDriverOptions {
 /** Mutable running totals accumulated while draining one turn's SSE stream. */
 interface DriveState {
   reply: string;
-  toolCalls: string[];
-  seen: Set<string>;
+  /** Captured tool calls, keyed by `call_id ?? name` (dedupe). */
+  toolCalls: Map<string, ToolCall>;
   ok: boolean;
   traceId?: string;
 }
 
-/** Record a `function_call` output item once per call id (deduped). */
+/**
+ * Record a `function_call` output item, keyed by `call_id ?? name`, capturing
+ * its parsed arguments. A later `done` event's fuller args win over the initial
+ * `added` event's (often empty) args.
+ */
 function recordToolCall(
-  item: { type?: string; name?: string; call_id?: string } | undefined,
+  item:
+    | { type?: string; name?: string; call_id?: string; arguments?: string }
+    | undefined,
   state: DriveState,
 ): void {
   if (item?.type !== "function_call" || !item.name) return;
   const key = item.call_id ?? item.name;
-  if (state.seen.has(key)) return;
-  state.seen.add(key);
-  state.toolCalls.push(item.name);
+  const args = parseArgs(item.arguments);
+  const existing = state.toolCalls.get(key);
+  if (!existing) {
+    state.toolCalls.set(key, { name: item.name, args });
+  } else if (Object.keys(args).length > 0) {
+    // The initial `added` event may carry empty args while the later `done`
+    // carries the full arguments — keep the fuller set.
+    existing.args = args;
+  }
 }
 
 /** Apply an `appkit.metadata` event's thread/trace ids. */
@@ -54,6 +66,24 @@ function applyMetadata(
   if (data?.mlflowTraceId) state.traceId = data.mlflowTraceId;
 }
 
+/** A single captured tool call, deduped by `call_id ?? name`. */
+type ToolCall = { name: string; args: Record<string, unknown> };
+
+/** Parse a function-call `arguments` JSON string; `{}` on missing/invalid. */
+function parseArgs(raw: unknown): Record<string, unknown> {
+  if (typeof raw !== "string" || raw.trim() === "") return {};
+  try {
+    const parsed = JSON.parse(raw);
+    return parsed !== null &&
+      typeof parsed === "object" &&
+      !Array.isArray(parsed)
+      ? (parsed as Record<string, unknown>)
+      : {};
+  } catch {
+    return {};
+  }
+}
+
 /** Parse a single Responses-API SSE `data:` payload into the running totals. */
 function applyEvent(
   event: Record<string, unknown>,
@@ -73,6 +103,7 @@ function applyEvent(
       type?: string;
       name?: string;
       call_id?: string;
+      arguments?: string;
       content?: Array<{ text?: string }>;
     };
     recordToolCall(item, state);
@@ -147,22 +178,27 @@ export function createHttpDriver(options: HttpDriverOptions): EvalDriver {
           signal,
         });
       } catch {
-        return { reply: "", toolCalls: [], succeeded: false };
+        return {
+          reply: "",
+          toolCalls: [],
+          toolCallDetails: [],
+          succeeded: false,
+        };
       }
 
       if (!res.ok || !res.body) {
         return {
           reply: "",
           toolCalls: [],
+          toolCallDetails: [],
           succeeded: false,
           sessionId: threadId,
         };
       }
 
       const state: DriveState = {
         reply: "",
-        toolCalls: [],
-        seen: new Set<string>(),
+        toolCalls: new Map<string, ToolCall>(),
         ok: true,
       };
       const setThread = (id: string) => {
@@ -193,9 +229,11 @@ export function createHttpDriver(options: HttpDriverOptions): EvalDriver {
       // throwing, so mark an aborted turn failed explicitly.
       if (signal.aborted) state.ok = false;
 
+      const toolCallDetails = [...state.toolCalls.values()];
       return {
         reply: state.reply,
-        toolCalls: state.toolCalls,
+        toolCalls: toolCallDetails.map((c) => c.name),
+        toolCallDetails,
         succeeded: state.ok,
         sessionId: threadId,
         traceId: state.traceId,

packages/appkit/src/evals/index.ts

@@ -11,9 +11,15 @@ export {
   type DatasetRow,
   type ReadEvalDatasetOptions,
   readEvalDataset,
+  userTurns,
 } from "./dataset";
-export { defineEval } from "./define-eval";
-export { type DiscoveredEval, discoverEvalFiles } from "./discover";
+export { defineEval, defineEvalConfig } from "./define-eval";
+export {
+  type DiscoveredEval,
+  type DiscoveredEvalConfig,
+  discoverEvalConfigs,
+  discoverEvalFiles,
+} from "./discover";
 export { createHttpDriver, type HttpDriverOptions } from "./http-driver";
 export {
   configureJudge,
@@ -34,6 +40,8 @@ export {
   formatEvalDetail,
   formatEvalHeadline,
   formatEvalResults,
+  formatResultsJson,
+  formatResultsJUnit,
   formatSummaryLine,
   summarize,
 } from "./report";
@@ -43,6 +51,7 @@ export {
   type EvalRunSummary,
   type RunEvalsOptions,
   runEvalsInDir,
+  runWithRetries,
 } from "./run-evals";
 export type {
   AssertionHandle,

packages/appkit/src/evals/report.ts

@@ -7,6 +7,8 @@ export interface EvalSummary {
   skipped: number;
   /** True when no eval failed (skips don't count as failures). */
   allPassed: boolean;
+  /** Fraction of scored (non-skipped) evals that passed, 0..1 (1 when none scored). */
+  passRate: number;
 }
 
 export function summarize(results: EvalResult[]): EvalSummary {
@@ -18,12 +20,14 @@ export function summarize(results: EvalResult[]): EvalSummary {
     else if (r.passed) passed++;
     else failed++;
   }
+  const scored = passed + failed;
   return {
     total: results.length,
     passed,
     failed,
     skipped,
     allPassed: failed === 0,
+    passRate: scored === 0 ? 1 : passed / scored,
   };
 }
 
@@ -74,3 +78,67 @@ export function formatEvalResults(results: EvalResult[]): string {
   lines.push(formatSummaryLine(results));
   return lines.join("\n");
 }
+
+/**
+ * Render results as a machine-readable JSON report (2-space indented):
+ * `{ summary: EvalSummary, results: EvalResult[] }`. Faithful to the types —
+ * every field present on a result round-trips.
+ */
+export function formatResultsJson(results: EvalResult[]): string {
+  return JSON.stringify({ summary: summarize(results), results }, null, 2);
+}
+
+/** Escape a value for use in XML text/attribute content. */
+function escapeXml(value: string): string {
+  return value
+    .replace(/&/g, "&")
+    .replace(/</g, "&lt;")
+    .replace(/>/g, "&gt;")
+    .replace(/"/g, "&quot;")
+    .replace(/'/g, "&apos;");
+}
+
+/** One-line reason a result failed: its error, else its failing gate labels. */
+function failureMessage(result: EvalResult): string {
+  if (result.error) return result.error;
+  const gates = result.assertions
+    .filter((a) => !a.pass && a.severity === "gate")
+    .map((a) => (a.detail ? `${a.label} — ${a.detail}` : a.label));
+  return gates.length ? gates.join("; ") : "eval failed";
+}
+
+/**
+ * Render results as JUnit XML for standard CI test reporters: a single
+ * `<testsuite name="appkit-agent-evals">` with one `<testcase>` per result.
+ * Failures carry a `<failure>` (error or failing-gate summary); skips a
+ * `<skipped>`. All attribute/text values are XML-escaped.
+ */
+export function formatResultsJUnit(results: EvalResult[]): string {
+  const s = summarize(results);
+  const lines: string[] = [];
+  lines.push('<?xml version="1.0" encoding="UTF-8"?>');
+  lines.push(
+    `<testsuite name="appkit-agent-evals" tests="${s.total}" failures="${s.failed}" skipped="${s.skipped}">`,
+  );
+  for (const r of results) {
+    const open = `  <testcase name="${escapeXml(r.id)}" classname="agent-eval"`;
+    if (r.skipped) {
+      lines.push(`${open}>`);
+      lines.push(
+        r.skipped.reason
+          ? `    <skipped message="${escapeXml(r.skipped.reason)}"/>`
+          : "    <skipped/>",
+      );
+      lines.push("  </testcase>");
+    } else if (!r.passed) {
+      const message = failureMessage(r);
+      lines.push(`${open}>`);
+      lines.push(`    <failure message="${escapeXml(message)}"/>`);
+      lines.push("  </testcase>");
+    } else {
+      lines.push(`${open}/>`);
+    }
+  }
+  lines.push("</testsuite>");
+  return lines.join("\n");
+}