From a75229123573c4f901c06af89573378c3a541291 Mon Sep 17 00:00:00 2001 From: Jim Bennett Date: Wed, 24 Jun 2026 13:21:27 -0700 Subject: [PATCH 01/11] chore: upgrade mastra-arize-ax-tracing cookbook to latest stack The cookbook was pinned to a Mastra 0.15-era stack and `npm i` failed with an ERESOLVE peer conflict. The bigger issue: @arizeai/openinference-mastra is deprecated and relied on Mastra's legacy telemetry system, removed 2025-11-04. Upgrade everything to the current stack and migrate the code accordingly: Dependencies: - @mastra/core 0.15 -> 1.46, libsql/loggers/memory/mastra CLI to latest 1.x - @ai-sdk/openai 1 -> 3, zod 3 -> 4, typescript 5 -> 6, @types/node 24 -> 26 - replace deprecated @arizeai/openinference-mastra with native @mastra/arize - add @mastra/observability and ai (now required directly) Code (Mastra 0.x -> 1.x API changes): - tracing: legacy telemetry.export.custom -> new AI Tracing via new Observability({ configs: { arize: { exporters: [new ArizeExporter(...)] }}}) - tool execute: input is now the first arg directly ({ context } -> { ...input }) - add required `id` to Agent and every LibSQLStore - main storage :memory: -> file:../mastra.db so Mastra's internal scheduler tables (mastra_workflow_snapshot) persist and are shared across connections (":memory:" gives each libsql connection its own DB -> SQLITE_ERROR) - gitignore *.db* artifacts; refresh README tracing references Verified: npm i clean, tsc --noEmit clean, `mastra dev` boots with no errors. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../mastra-arize-ax-tracing/.gitignore | 3 ++ .../mastra-arize-ax-tracing/README.md | 4 +- .../mastra-arize-ax-tracing/package.json | 24 ++++++----- .../agents/weather-orchestrator-agent.ts | 2 + .../src/mastra/index.ts | 43 ++++++++++--------- .../mastra/tools/activity-planning-tool.ts | 4 +- .../src/mastra/tools/weather-analysis-tool.ts | 4 +- .../src/mastra/tools/weather-tool.ts | 4 +- 8 files changed, 46 insertions(+), 42 deletions(-) diff --git a/typescript/cookbooks/mastra-arize-ax-tracing/.gitignore b/typescript/cookbooks/mastra-arize-ax-tracing/.gitignore index 03bb9ce..3192fdb 100644 --- a/typescript/cookbooks/mastra-arize-ax-tracing/.gitignore +++ b/typescript/cookbooks/mastra-arize-ax-tracing/.gitignore @@ -1,3 +1,6 @@ node_modules .mastra/ package-lock.json +*.db +*.db-shm +*.db-wal diff --git a/typescript/cookbooks/mastra-arize-ax-tracing/README.md b/typescript/cookbooks/mastra-arize-ax-tracing/README.md index 56a2abe..1e14bdd 100644 --- a/typescript/cookbooks/mastra-arize-ax-tracing/README.md +++ b/typescript/cookbooks/mastra-arize-ax-tracing/README.md @@ -48,7 +48,7 @@ activityPlanningTool → recommends activities Final response ``` -Tracing is configured in [src/mastra/index.ts](src/mastra/index.ts) using `@arizeai/openinference-mastra`. Every agent invocation and tool call is exported as an OpenInference span to `https://otlp.arize.com/v1/traces`. +Tracing is configured in [src/mastra/index.ts](src/mastra/index.ts) using `@mastra/arize`, wired into Mastra's AI Tracing (`observability`) system. Every agent invocation and tool call is exported as an OpenInference span to `https://otlp.arize.com/v1/traces`. ## Project structure @@ -66,5 +66,5 @@ src/mastra/ ## Resources - [Arize AX Documentation](https://arize.com/docs/ax/) -- [OpenInference for Mastra](https://github.com/Arize-ai/openinference/tree/main/js/packages/openinference-mastra) +- [Mastra Arize exporter (`@mastra/arize`)](https://mastra.ai/en/docs/observability/ai-tracing/exporters/arize) - [Mastra Documentation](https://mastra.ai/docs) diff --git a/typescript/cookbooks/mastra-arize-ax-tracing/package.json b/typescript/cookbooks/mastra-arize-ax-tracing/package.json index ef9f1c5..2e5230d 100644 --- a/typescript/cookbooks/mastra-arize-ax-tracing/package.json +++ b/typescript/cookbooks/mastra-arize-ax-tracing/package.json @@ -17,18 +17,20 @@ "node": ">=20.9.0" }, "dependencies": { - "@ai-sdk/openai": "^1.3.24", - "@arizeai/openinference-mastra": "^2.2.0", - "@mastra/core": "^0.15.3", - "@mastra/libsql": "^0.13.7", - "@mastra/loggers": "^0.10.9", - "@mastra/memory": "^0.14.2", - "zod": "^3.25.76" + "@ai-sdk/openai": "^3.0.74", + "@mastra/arize": "^1.3.1", + "@mastra/core": "^1.46.0", + "@mastra/libsql": "^1.14.1", + "@mastra/loggers": "^1.2.0", + "@mastra/memory": "^1.21.1", + "@mastra/observability": "^1.15.1", + "ai": "^6.0.209", + "zod": "^4.4.3" }, "devDependencies": { - "@types/node": "^24.3.0", - "mastra": "^0.11.3", - "tsx": "^4.0.0", - "typescript": "^5.9.2" + "@types/node": "^26.0.0", + "mastra": "^1.15.1", + "tsx": "^4.22.4", + "typescript": "^6.0.3" } } diff --git a/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/agents/weather-orchestrator-agent.ts b/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/agents/weather-orchestrator-agent.ts index 475e2f7..cde6f2e 100644 --- a/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/agents/weather-orchestrator-agent.ts +++ b/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/agents/weather-orchestrator-agent.ts @@ -7,6 +7,7 @@ import { weatherAnalysisTool } from '../tools/weather-analysis-tool'; import { activityPlanningTool } from '../tools/activity-planning-tool'; export const weatherOrchestratorAgent = new Agent({ + id: 'weatherOrchestratorAgent', name: 'WeatherOrchestratorAgent', instructions: ` You are the Weather Orchestrator Agent. You MUST use the available tools to provide weather information and recommendations. @@ -46,6 +47,7 @@ export const weatherOrchestratorAgent = new Agent({ }, memory: new Memory({ storage: new LibSQLStore({ + id: 'agent-memory', url: 'file:../mastra.db', }), }), diff --git a/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/index.ts b/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/index.ts index 5a31dd0..ef0d6cb 100644 --- a/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/index.ts +++ b/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/index.ts @@ -1,14 +1,11 @@ import { Mastra } from '@mastra/core/mastra'; import { PinoLogger } from '@mastra/loggers'; import { LibSQLStore } from '@mastra/libsql'; +import { Observability } from '@mastra/observability'; +import { ArizeExporter } from '@mastra/arize'; // Import orchestrator/worker agents - this is the only workflow pattern now import { weatherOrchestratorAgent } from './agents/weather-orchestrator-agent'; -import { - isOpenInferenceSpan, - OpenInferenceOTLPTraceExporter, -} from "@arizeai/openinference-mastra"; - const ARIZE_SPACE_ID = process.env.ARIZE_SPACE_ID; const ARIZE_API_KEY = process.env.ARIZE_API_KEY; const OPENAI_API_KEY = process.env.OPENAI_API_KEY; @@ -25,26 +22,30 @@ export const mastra = new Mastra({ weatherOrchestratorAgent }, storage: new LibSQLStore({ - // stores telemetry, evals, ... into memory storage, if it needs to persist, change to file:../mastra.db - url: ":memory:", + // File-backed so Mastra's internal workflow/scheduler tables (e.g. + // mastra_workflow_snapshot) persist and are shared across connections. + // ":memory:" gives each libsql connection its own DB, which breaks them. + id: "mastra-storage", + url: "file:../mastra.db", }), logger: new PinoLogger({ name: 'Mastra', level: 'info', }), - telemetry: { - enabled: true, - serviceName: "mastra-orchestrator-workflow", - export: { - type: "custom", - exporter: new OpenInferenceOTLPTraceExporter({ - url: "https://otlp.arize.com/v1/traces", - headers: { - "arize-space-id": ARIZE_SPACE_ID, - "arize-api-key": ARIZE_API_KEY, - }, - spanFilter: isOpenInferenceSpan, - }), + // Mastra AI Tracing: export every agent and tool span to Arize AX via the + // native @mastra/arize exporter (replaces the deprecated legacy telemetry path). + observability: new Observability({ + configs: { + arize: { + serviceName: "mastra-orchestrator-workflow", + exporters: [ + new ArizeExporter({ + spaceId: ARIZE_SPACE_ID, + apiKey: ARIZE_API_KEY, + projectName: "mastra-orchestrator-workflow", + }), + ], + }, }, - }, + }), }); diff --git a/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/tools/activity-planning-tool.ts b/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/tools/activity-planning-tool.ts index 0ff10d6..9a568d7 100644 --- a/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/tools/activity-planning-tool.ts +++ b/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/tools/activity-planning-tool.ts @@ -14,9 +14,7 @@ export const activityPlanningTool = createTool({ outputSchema: z.object({ activityPlan: z.string().describe('Detailed activity recommendations and schedule') }), - execute: async ({ context }) => { - const { weatherData, weatherAnalysis, location } = context; - + execute: async ({ weatherData, weatherAnalysis, location }) => { const result = await generateText({ model: openai('gpt-4o-mini'), prompt: `Based on the weather data${weatherAnalysis ? ' and analysis' : ''} for ${location}, create a detailed activity plan: diff --git a/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/tools/weather-analysis-tool.ts b/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/tools/weather-analysis-tool.ts index 64fdb38..b276b71 100644 --- a/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/tools/weather-analysis-tool.ts +++ b/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/tools/weather-analysis-tool.ts @@ -13,9 +13,7 @@ export const weatherAnalysisTool = createTool({ outputSchema: z.object({ analysis: z.string().describe('Detailed weather analysis and forecast interpretation') }), - execute: async ({ context }) => { - const { weatherData, location } = context; - + execute: async ({ weatherData, location }) => { const result = await generateText({ model: openai('gpt-4o-mini'), prompt: `Analyze the following weather data for ${location} and provide detailed insights: diff --git a/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/tools/weather-tool.ts b/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/tools/weather-tool.ts index b58e256..e40fb02 100644 --- a/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/tools/weather-tool.ts +++ b/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/tools/weather-tool.ts @@ -35,8 +35,8 @@ export const weatherTool = createTool({ conditions: z.string(), location: z.string(), }), - execute: async ({ context }) => { - return await getWeather(context.location); + execute: async ({ location }) => { + return await getWeather(location); }, }); From eaeb6c2f01730bd6ca53d051c0ec2a7218334a0e Mon Sep 17 00:00:00 2001 From: Jim Bennett Date: Wed, 24 Jun 2026 13:28:35 -0700 Subject: [PATCH 02/11] refactor: route tool LLM calls through Mastra agents for tracing The weather-analysis and activity-planning tools called the ai SDK's generateText() directly, so those LLM calls bypassed Mastra and never appeared as spans in Arize AX. Make them first-class Mastra agents instead: - Add weatherAnalysisAgent and activityPlanningAgent (instructions + model), registered on the Mastra instance - Tools now resolve the agent from the tool execution context (mastra.getAgent(...)) and call agent.generate(), so each call is captured as a traced child span of the tool. Using the context avoids importing the Mastra instance into tools (no circular dependency). - Drop the now-unused `ai` dependency and direct @ai-sdk/openai imports in tools Verified: tsc clean, `mastra dev` boots with no errors, all three agents register. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../mastra-arize-ax-tracing/package.json | 1 - .../mastra/agents/activity-planning-agent.ts | 20 +++++++++++++++++++ .../mastra/agents/weather-analysis-agent.ts | 20 +++++++++++++++++++ .../src/mastra/index.ts | 9 +++++++-- .../mastra/tools/activity-planning-tool.ts | 18 ++++++++++------- .../src/mastra/tools/weather-analysis-tool.ts | 18 ++++++++++------- 6 files changed, 69 insertions(+), 17 deletions(-) create mode 100644 typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/agents/activity-planning-agent.ts create mode 100644 typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/agents/weather-analysis-agent.ts diff --git a/typescript/cookbooks/mastra-arize-ax-tracing/package.json b/typescript/cookbooks/mastra-arize-ax-tracing/package.json index 2e5230d..6241ee5 100644 --- a/typescript/cookbooks/mastra-arize-ax-tracing/package.json +++ b/typescript/cookbooks/mastra-arize-ax-tracing/package.json @@ -24,7 +24,6 @@ "@mastra/loggers": "^1.2.0", "@mastra/memory": "^1.21.1", "@mastra/observability": "^1.15.1", - "ai": "^6.0.209", "zod": "^4.4.3" }, "devDependencies": { diff --git a/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/agents/activity-planning-agent.ts b/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/agents/activity-planning-agent.ts new file mode 100644 index 0000000..2b69ef0 --- /dev/null +++ b/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/agents/activity-planning-agent.ts @@ -0,0 +1,20 @@ +import { openai } from '@ai-sdk/openai'; +import { Agent } from '@mastra/core/agent'; + +// LLM-backed planning step. Invoked by activityPlanningTool via the Mastra +// instance so the call is captured as a traced child span in Arize AX. +export const activityPlanningAgent = new Agent({ + id: 'activityPlanningAgent', + name: 'ActivityPlanningAgent', + instructions: ` + You create detailed, location-specific activity plans from weather data + (and an optional weather analysis). Recommend concrete activities with + specific venues, parks, trails, or locations, sensible timing, and the + weather considerations behind each choice. Always include indoor + alternatives and any special considerations (gear, warnings). + + Follow the exact output format requested in the message. Respond with the + plan only. +`, + model: openai('gpt-4o-mini'), +}); diff --git a/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/agents/weather-analysis-agent.ts b/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/agents/weather-analysis-agent.ts new file mode 100644 index 0000000..89c2112 --- /dev/null +++ b/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/agents/weather-analysis-agent.ts @@ -0,0 +1,20 @@ +import { openai } from '@ai-sdk/openai'; +import { Agent } from '@mastra/core/agent'; + +// LLM-backed analysis step. Invoked by weatherAnalysisTool via the Mastra +// instance so the call is captured as a traced child span in Arize AX. +export const weatherAnalysisAgent = new Agent({ + id: 'weatherAnalysisAgent', + name: 'WeatherAnalysisAgent', + instructions: ` + You are a meteorological analyst. Given raw weather data for a location, + provide a concise, specific analysis that covers: + + - Notable current conditions and what they mean in practical terms + - Comfort and safety considerations (heat, cold, wind, precipitation, UV) + - How the conditions are likely to affect outdoor plans for the day + + Respond with the analysis only. Do not ask follow-up questions. +`, + model: openai('gpt-4o-mini'), +}); diff --git a/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/index.ts b/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/index.ts index ef0d6cb..2a9b315 100644 --- a/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/index.ts +++ b/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/index.ts @@ -3,8 +3,10 @@ import { PinoLogger } from '@mastra/loggers'; import { LibSQLStore } from '@mastra/libsql'; import { Observability } from '@mastra/observability'; import { ArizeExporter } from '@mastra/arize'; -// Import orchestrator/worker agents - this is the only workflow pattern now +// Orchestrator agent plus the LLM-backed worker agents its tools delegate to. import { weatherOrchestratorAgent } from './agents/weather-orchestrator-agent'; +import { weatherAnalysisAgent } from './agents/weather-analysis-agent'; +import { activityPlanningAgent } from './agents/activity-planning-agent'; const ARIZE_SPACE_ID = process.env.ARIZE_SPACE_ID; const ARIZE_API_KEY = process.env.ARIZE_API_KEY; @@ -19,7 +21,10 @@ export const mastra = new Mastra({ // No workflows - using pure orchestrator/worker agent pattern agents: { // Orchestrator agent that coordinates the entire workflow - weatherOrchestratorAgent + weatherOrchestratorAgent, + // Worker agents the orchestrator's tools delegate to (traced child spans) + weatherAnalysisAgent, + activityPlanningAgent }, storage: new LibSQLStore({ // File-backed so Mastra's internal workflow/scheduler tables (e.g. diff --git a/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/tools/activity-planning-tool.ts b/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/tools/activity-planning-tool.ts index 9a568d7..5bea653 100644 --- a/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/tools/activity-planning-tool.ts +++ b/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/tools/activity-planning-tool.ts @@ -1,7 +1,5 @@ import { createTool } from '@mastra/core/tools'; import { z } from 'zod'; -import { generateText } from 'ai'; -import { openai } from '@ai-sdk/openai'; export const activityPlanningTool = createTool({ id: 'activity-planning-tool', @@ -14,10 +12,16 @@ export const activityPlanningTool = createTool({ outputSchema: z.object({ activityPlan: z.string().describe('Detailed activity recommendations and schedule') }), - execute: async ({ weatherData, weatherAnalysis, location }) => { - const result = await generateText({ - model: openai('gpt-4o-mini'), - prompt: `Based on the weather data${weatherAnalysis ? ' and analysis' : ''} for ${location}, create a detailed activity plan: + // Delegate to the activityPlanningAgent via the Mastra instance so the LLM + // call is traced as a child span of this tool in Arize AX. + execute: async ({ weatherData, weatherAnalysis, location }, { mastra }) => { + if (!mastra) { + throw new Error('Mastra instance is not available in the tool execution context'); + } + + const agent = mastra.getAgent('activityPlanningAgent'); + const result = await agent.generate( + `Based on the weather data${weatherAnalysis ? ' and analysis' : ''} for ${location}, create a detailed activity plan: WEATHER DATA: ${weatherData} @@ -62,7 +66,7 @@ Guidelines followed: - Consider weather conditions for activity selection - Provide backup indoor options - Include timing recommendations` - }); + ); return { activityPlan: result.text diff --git a/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/tools/weather-analysis-tool.ts b/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/tools/weather-analysis-tool.ts index b276b71..433592c 100644 --- a/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/tools/weather-analysis-tool.ts +++ b/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/tools/weather-analysis-tool.ts @@ -1,7 +1,5 @@ import { createTool } from '@mastra/core/tools'; import { z } from 'zod'; -import { generateText } from 'ai'; -import { openai } from '@ai-sdk/openai'; export const weatherAnalysisTool = createTool({ id: 'weather-analysis-tool', @@ -13,10 +11,16 @@ export const weatherAnalysisTool = createTool({ outputSchema: z.object({ analysis: z.string().describe('Detailed weather analysis and forecast interpretation') }), - execute: async ({ weatherData, location }) => { - const result = await generateText({ - model: openai('gpt-4o-mini'), - prompt: `Analyze the following weather data for ${location} and provide detailed insights: + // Delegate to the weatherAnalysisAgent via the Mastra instance so the LLM + // call is traced as a child span of this tool in Arize AX. + execute: async ({ weatherData, location }, { mastra }) => { + if (!mastra) { + throw new Error('Mastra instance is not available in the tool execution context'); + } + + const agent = mastra.getAgent('weatherAnalysisAgent'); + const result = await agent.generate( + `Analyze the following weather data for ${location} and provide detailed insights: ${weatherData} @@ -26,7 +30,7 @@ Please provide: 3. Optimal timing windows throughout the day 4. Any weather risks or special considerations 5. Overall assessment of weather suitability for outdoor vs indoor activities` - }); + ); return { analysis: result.text From 238b9e2749cc2778ba8b729e975cf5df90dc8739 Mon Sep 17 00:00:00 2001 From: Jim Bennett Date: Wed, 24 Jun 2026 13:32:05 -0700 Subject: [PATCH 03/11] docs: fix README and Node engine for the upgraded cookbook - Point the intro link at the exact guide in its new location (cookbooks/evaluate/align-llm-evals-with-human-judgment) - Bump required Node to >=22.13.0 (engines + README) to match @mastra/core 1.46 - Run instructions: Mastra Studio at http://localhost:4111 (no auto-open), with a concrete example request - "How it works": show the analysis/planning tools delegating to worker agents, and explain the nested tracing - Project structure: add the two new worker agent files Co-Authored-By: Claude Opus 4.8 (1M context) --- .../mastra-arize-ax-tracing/README.md | 21 ++++++++++++------- .../mastra-arize-ax-tracing/package.json | 2 +- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/typescript/cookbooks/mastra-arize-ax-tracing/README.md b/typescript/cookbooks/mastra-arize-ax-tracing/README.md index 1e14bdd..a4f5bee 100644 --- a/typescript/cookbooks/mastra-arize-ax-tracing/README.md +++ b/typescript/cookbooks/mastra-arize-ax-tracing/README.md @@ -1,10 +1,10 @@ # Mastra Agent with Arize AX Tracing -Companion repo for the [Arize AX tutorial](https://arize.com/docs/ax/). A Mastra orchestrator agent instrumented with OpenInference tracing that sends spans to Arize AX. +Companion repo for the Arize AX guide [Align LLM Evals with Human Judgment](https://arize.com/docs/ax/cookbooks/evaluate/align-llm-evals-with-human-judgment). A Mastra orchestrator agent instrumented with OpenInference tracing that sends spans to Arize AX. ## Prerequisites -- [Node.js](https://nodejs.org) v20.9.0+ +- [Node.js](https://nodejs.org) v22.13.0+ - An [Arize AX](https://app.arize.com) account (Space ID and API key) - An [OpenAI](https://platform.openai.com) API key @@ -28,7 +28,7 @@ export ARIZE_SPACE_ID="your-arize-space-id" npm start ``` -This starts the Mastra dev server and opens the agent UI. Send requests to the weather orchestrator agent to generate traces — they will appear in your Arize AX space within a few seconds. +This starts the Mastra dev server. Open [Mastra Studio](https://mastra.ai/docs/studio/overview) at [http://localhost:4111](http://localhost:4111) in your browser (Mastra picks the next free port if 4111 is taken), select the **WeatherOrchestratorAgent**, and send it a request such as "What should I do in London today?" to generate traces — they will appear in your Arize AX space within a few seconds. ## How it works @@ -39,15 +39,20 @@ User request ↓ Weather Orchestrator Agent ↓ -weatherTool → fetches current weather data +weatherTool → fetches current weather data (Open-Meteo) ↓ -weatherAnalysisTool → analyzes weather patterns +weatherAnalysisTool → delegates to Weather Analysis Agent (LLM) ↓ -activityPlanningTool → recommends activities +activityPlanningTool → delegates to Activity Planning Agent (LLM) ↓ Final response ``` +The analysis and planning tools don't call the model directly — they resolve a +dedicated worker agent from the Mastra instance (`mastra.getAgent(...)`) and call +`agent.generate(...)`. Because those calls run inside Mastra, their LLM spans are +captured as nested children of the tool span in the trace. + Tracing is configured in [src/mastra/index.ts](src/mastra/index.ts) using `@mastra/arize`, wired into Mastra's AI Tracing (`observability`) system. Every agent invocation and tool call is exported as an OpenInference span to `https://otlp.arize.com/v1/traces`. ## Project structure @@ -56,7 +61,9 @@ Tracing is configured in [src/mastra/index.ts](src/mastra/index.ts) using `@mast src/mastra/ ├── index.ts # Mastra setup + Arize AX exporter ├── agents/ -│ └── weather-orchestrator-agent.ts +│ ├── weather-orchestrator-agent.ts # coordinates the tools +│ ├── weather-analysis-agent.ts # LLM worker, called by weatherAnalysisTool +│ └── activity-planning-agent.ts # LLM worker, called by activityPlanningTool └── tools/ ├── weather-tool.ts ├── weather-analysis-tool.ts diff --git a/typescript/cookbooks/mastra-arize-ax-tracing/package.json b/typescript/cookbooks/mastra-arize-ax-tracing/package.json index 6241ee5..1441f18 100644 --- a/typescript/cookbooks/mastra-arize-ax-tracing/package.json +++ b/typescript/cookbooks/mastra-arize-ax-tracing/package.json @@ -14,7 +14,7 @@ "description": "Companion repo for Arize AX tutorial: Mastra orchestrator agent with Arize AX tracing", "type": "module", "engines": { - "node": ">=20.9.0" + "node": ">=22.13.0" }, "dependencies": { "@ai-sdk/openai": "^3.0.74", From 3fdf6bd965f45495ec1e83a977098b5ede8db8f8 Mon Sep 17 00:00:00 2001 From: Jim Bennett Date: Wed, 24 Jun 2026 13:34:57 -0700 Subject: [PATCH 04/11] fix: use non-deprecated gpt-4.1-mini model MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit gpt-4o-mini is superseded by gpt-4.1 (per the check-models policy) and gets flagged as an outdated introduced reference. Move all three agents to gpt-4.1-mini — the current non-reasoning mini tier, which (unlike the GPT-5 reasoning minis) still accepts temperature/top_p. Verified with scan-models.mjs: no blocking model findings. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../src/mastra/agents/activity-planning-agent.ts | 2 +- .../src/mastra/agents/weather-analysis-agent.ts | 2 +- .../src/mastra/agents/weather-orchestrator-agent.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/agents/activity-planning-agent.ts b/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/agents/activity-planning-agent.ts index 2b69ef0..4b9c6ea 100644 --- a/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/agents/activity-planning-agent.ts +++ b/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/agents/activity-planning-agent.ts @@ -16,5 +16,5 @@ export const activityPlanningAgent = new Agent({ Follow the exact output format requested in the message. Respond with the plan only. `, - model: openai('gpt-4o-mini'), + model: openai('gpt-4.1-mini'), }); diff --git a/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/agents/weather-analysis-agent.ts b/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/agents/weather-analysis-agent.ts index 89c2112..4cc9fde 100644 --- a/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/agents/weather-analysis-agent.ts +++ b/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/agents/weather-analysis-agent.ts @@ -16,5 +16,5 @@ export const weatherAnalysisAgent = new Agent({ Respond with the analysis only. Do not ask follow-up questions. `, - model: openai('gpt-4o-mini'), + model: openai('gpt-4.1-mini'), }); diff --git a/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/agents/weather-orchestrator-agent.ts b/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/agents/weather-orchestrator-agent.ts index cde6f2e..85baa44 100644 --- a/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/agents/weather-orchestrator-agent.ts +++ b/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/agents/weather-orchestrator-agent.ts @@ -39,7 +39,7 @@ export const weatherOrchestratorAgent = new Agent({ Remember: You are a tool orchestrator, not a direct weather information provider. `, - model: openai('gpt-4o-mini'), + model: openai('gpt-4.1-mini'), tools: { weatherTool, weatherAnalysisTool, From 3a6f8ec7f02d341802aeb2c8a071c3e189a3c152 Mon Sep 17 00:00:00 2001 From: Jim Bennett Date: Wed, 24 Jun 2026 13:37:55 -0700 Subject: [PATCH 05/11] ci(check-models): run on Node 24 Bump the check-models workflow's setup-node from 20 to 24. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/check-models.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check-models.yml b/.github/workflows/check-models.yml index 4bc50f8..e8b1d97 100644 --- a/.github/workflows/check-models.yml +++ b/.github/workflows/check-models.yml @@ -28,7 +28,7 @@ jobs: - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e with: - node-version: 20 + node-version: 24 - name: Scan touched files for outdated models id: scan From 847cca08c5388b569a93ba0f424d7d00a630717b Mon Sep 17 00:00:00 2001 From: Jim Bennett Date: Wed, 24 Jun 2026 13:42:15 -0700 Subject: [PATCH 06/11] fix: use gpt-5.4-mini for the agents Switch all three agents from gpt-4.1-mini to gpt-5.4-mini (the latest mini tier). Verified end-to-end: `mastra dev` boots and a live orchestrator invocation runs the full weather -> analysis -> planning flow and returns a plan with no errors (the reasoning model does not reject the request). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../src/mastra/agents/activity-planning-agent.ts | 2 +- .../src/mastra/agents/weather-analysis-agent.ts | 2 +- .../src/mastra/agents/weather-orchestrator-agent.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/agents/activity-planning-agent.ts b/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/agents/activity-planning-agent.ts index 4b9c6ea..f0b1a38 100644 --- a/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/agents/activity-planning-agent.ts +++ b/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/agents/activity-planning-agent.ts @@ -16,5 +16,5 @@ export const activityPlanningAgent = new Agent({ Follow the exact output format requested in the message. Respond with the plan only. `, - model: openai('gpt-4.1-mini'), + model: openai('gpt-5.4-mini'), }); diff --git a/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/agents/weather-analysis-agent.ts b/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/agents/weather-analysis-agent.ts index 4cc9fde..2864ca2 100644 --- a/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/agents/weather-analysis-agent.ts +++ b/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/agents/weather-analysis-agent.ts @@ -16,5 +16,5 @@ export const weatherAnalysisAgent = new Agent({ Respond with the analysis only. Do not ask follow-up questions. `, - model: openai('gpt-4.1-mini'), + model: openai('gpt-5.4-mini'), }); diff --git a/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/agents/weather-orchestrator-agent.ts b/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/agents/weather-orchestrator-agent.ts index 85baa44..6f2c6b4 100644 --- a/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/agents/weather-orchestrator-agent.ts +++ b/typescript/cookbooks/mastra-arize-ax-tracing/src/mastra/agents/weather-orchestrator-agent.ts @@ -39,7 +39,7 @@ export const weatherOrchestratorAgent = new Agent({ Remember: You are a tool orchestrator, not a direct weather information provider. `, - model: openai('gpt-4.1-mini'), + model: openai('gpt-5.4-mini'), tools: { weatherTool, weatherAnalysisTool, From 3d34d971266396b650296c84c5632bceb4c23914 Mon Sep 17 00:00:00 2001 From: Jim Bennett Date: Wed, 24 Jun 2026 13:42:15 -0700 Subject: [PATCH 07/11] ci(check-models): bump github-script to v8 (Node 24) actions/github-script@v7 declares a Node 20 runtime, which the runner now forces onto Node 24 with a deprecation warning. Bump to v8, which runs on Node 24 natively. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/check-models.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check-models.yml b/.github/workflows/check-models.yml index e8b1d97..47fdd88 100644 --- a/.github/workflows/check-models.yml +++ b/.github/workflows/check-models.yml @@ -40,7 +40,7 @@ jobs: cat model-scan.json - name: Comment and gate - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 (Node 24) with: script: | const fs = require('fs'); From b27740d8a8392d9cfd1c109036c068d06d8ee194 Mon Sep 17 00:00:00 2001 From: Jim Bennett Date: Wed, 24 Jun 2026 13:55:15 -0700 Subject: [PATCH 08/11] feat: add generate-traces script for populating Arize AX Add `npm run generate-traces`, a tsx script that sends a diverse set of prompts to the WeatherOrchestratorAgent and flushes the spans to Arize AX via mastra.shutdown(). Prompts are grouped to exercise each tool path: weather-only, weather+planning, and the full weather+analysis+planning chain, plus some varied/edge phrasings. The script chdirs into src/ so the Mastra config's `file:../mastra.db` lands in the cookbook root (gitignored) rather than the parent directory. Verified: all 12 prompts complete and the observability exporter flushes on shutdown. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../mastra-arize-ax-tracing/README.md | 10 +++ .../mastra-arize-ax-tracing/package.json | 1 + .../src/scripts/generate-traces.ts | 79 +++++++++++++++++++ 3 files changed, 90 insertions(+) create mode 100644 typescript/cookbooks/mastra-arize-ax-tracing/src/scripts/generate-traces.ts diff --git a/typescript/cookbooks/mastra-arize-ax-tracing/README.md b/typescript/cookbooks/mastra-arize-ax-tracing/README.md index a4f5bee..c280dec 100644 --- a/typescript/cookbooks/mastra-arize-ax-tracing/README.md +++ b/typescript/cookbooks/mastra-arize-ax-tracing/README.md @@ -30,6 +30,16 @@ npm start This starts the Mastra dev server. Open [Mastra Studio](https://mastra.ai/docs/studio/overview) at [http://localhost:4111](http://localhost:4111) in your browser (Mastra picks the next free port if 4111 is taken), select the **WeatherOrchestratorAgent**, and send it a request such as "What should I do in London today?" to generate traces — they will appear in your Arize AX space within a few seconds. +### Generate a batch of example traces + +To populate Arize AX with a diverse set of traces at once — covering each tool path (weather lookup, activity planning, and the full analysis-and-planning chain) — run: + +```bash +npm run generate-traces +``` + +This sends a curated set of prompts to the orchestrator agent and flushes the resulting spans to Arize AX. See [src/scripts/generate-traces.ts](src/scripts/generate-traces.ts) to adjust the prompts. + ## How it works A single orchestrator agent coordinates three tools in sequence: diff --git a/typescript/cookbooks/mastra-arize-ax-tracing/package.json b/typescript/cookbooks/mastra-arize-ax-tracing/package.json index 1441f18..1860aa9 100644 --- a/typescript/cookbooks/mastra-arize-ax-tracing/package.json +++ b/typescript/cookbooks/mastra-arize-ax-tracing/package.json @@ -6,6 +6,7 @@ "start": "mastra dev", "dev": "mastra dev", "build": "mastra build", + "generate-traces": "tsx src/scripts/generate-traces.ts", "test": "echo \"Error: no test specified\" && exit 1" }, "keywords": [], diff --git a/typescript/cookbooks/mastra-arize-ax-tracing/src/scripts/generate-traces.ts b/typescript/cookbooks/mastra-arize-ax-tracing/src/scripts/generate-traces.ts new file mode 100644 index 0000000..7474cef --- /dev/null +++ b/typescript/cookbooks/mastra-arize-ax-tracing/src/scripts/generate-traces.ts @@ -0,0 +1,79 @@ +/** + * Sends a diverse set of prompts to the WeatherOrchestratorAgent to generate + * traces in Arize AX. Each prompt is chosen to exercise a different tool path: + * + * - weather-only → weatherTool + * - weather+planning → weatherTool → activityPlanningTool + * - weather+analysis+plan → weatherTool → weatherAnalysisTool → activityPlanningTool + * + * Run with: npm run generate-traces + * Requires the same env vars as the agent: ARIZE_SPACE_ID, ARIZE_API_KEY, OPENAI_API_KEY. + */ +import { fileURLToPath } from 'node:url'; + +// The Mastra config points its storage at `file:../mastra.db`, resolved relative +// to the process working directory. Run from `src/` so that local db lands in +// the cookbook root (where *.db is gitignored) instead of the parent directory. +process.chdir(fileURLToPath(new URL('..', import.meta.url))); + +const { mastra } = await import('../mastra/index'); + +interface Example { + category: string; + prompt: string; +} + +const examples: Example[] = [ + // Simple weather lookups — the orchestrator should call weatherTool and stop. + { category: 'weather-only', prompt: "What's the weather in Tokyo right now?" }, + { category: 'weather-only', prompt: 'Is it raining in Seattle at the moment?' }, + { category: 'weather-only', prompt: 'How hot does it feel in Dubai today?' }, + + // Activity requests — weatherTool then activityPlanningTool. + { category: 'weather+planning', prompt: 'What should I do in Barcelona today?' }, + { category: 'weather+planning', prompt: 'Give me some activity ideas for a day out in Amsterdam.' }, + { category: 'weather+planning', prompt: "I'm spending the day in Cape Town — what's good to do given the weather?" }, + + // Deep dives — the full weatherTool → weatherAnalysisTool → activityPlanningTool chain. + { category: 'weather+analysis+planning', prompt: 'Give me a detailed weather analysis and a full activity plan for London today.' }, + { category: 'weather+analysis+planning', prompt: 'Analyze the conditions in Reykjavik and plan my whole day around them.' }, + { category: 'weather+analysis+planning', prompt: 'I want an in-depth look at the weather in Singapore plus a complete itinerary for today.' }, + + // Varied phrasing / edge cases for span diversity. + { category: 'vague-intent', prompt: 'I might head outside in Chicago later... or maybe not. What do you reckon?' }, + { category: 'multi-location', prompt: 'Is it better weather for a long walk in Oslo or in Lisbon right now, and what should I do there?' }, + { category: 'activity-constrained', prompt: 'Plan an indoor-friendly day in Mumbai in case the weather is rough.' }, +]; + +async function main(): Promise { + const agent = mastra.getAgent('weatherOrchestratorAgent'); + + console.log(`Sending ${examples.length} prompts to WeatherOrchestratorAgent…\n`); + + let ok = 0; + for (const [index, example] of examples.entries()) { + const label = `[${index + 1}/${examples.length}] (${example.category})`; + console.log(`${label} → ${example.prompt}`); + try { + const result = await agent.generate(example.prompt); + const text = (result.text ?? '').replace(/\s+/g, ' ').trim(); + console.log(` ✓ ${text.slice(0, 140)}${text.length > 140 ? '…' : ''}\n`); + ok += 1; + } catch (error) { + console.error(` ✗ failed: ${(error as Error).message}\n`); + } + } + + console.log(`Completed ${ok}/${examples.length} prompts. Flushing traces to Arize AX…`); + // Cleanly shut down so the tracing exporter flushes all buffered spans before exit. + await mastra.shutdown(); + console.log('Done. The traces should appear in your Arize AX space within a few seconds.'); +} + +try { + await main(); + process.exit(0); +} catch (error) { + console.error(error); + process.exit(1); +} From 5cd3f7e485639667b837b95e83b09bc0682e6b2d Mon Sep 17 00:00:00 2001 From: Jim Bennett Date: Wed, 24 Jun 2026 14:02:41 -0700 Subject: [PATCH 09/11] feat: expand example prompts to 50 and extract to JSON Move the trace-generation prompts out of generate-traces.ts into src/scripts/example-prompts.json, grouped by tool path, and expand the set to 50 prompts for broader coverage: - weather-only: 12 - weather+planning: 12 - weather+analysis+planning: 12 - multi-location: 6 - vague-intent: 4 - activity-constrained: 4 The script now loads and flattens the JSON at runtime. Validated: JSON parses to 50 unique prompts, tsc clean. README points at the JSON for editing. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../mastra-arize-ax-tracing/README.md | 2 +- .../src/scripts/example-prompts.json | 64 +++++++++++++++++++ .../src/scripts/generate-traces.ts | 32 ++++------ 3 files changed, 76 insertions(+), 22 deletions(-) create mode 100644 typescript/cookbooks/mastra-arize-ax-tracing/src/scripts/example-prompts.json diff --git a/typescript/cookbooks/mastra-arize-ax-tracing/README.md b/typescript/cookbooks/mastra-arize-ax-tracing/README.md index c280dec..e696baf 100644 --- a/typescript/cookbooks/mastra-arize-ax-tracing/README.md +++ b/typescript/cookbooks/mastra-arize-ax-tracing/README.md @@ -38,7 +38,7 @@ To populate Arize AX with a diverse set of traces at once — covering each tool npm run generate-traces ``` -This sends a curated set of prompts to the orchestrator agent and flushes the resulting spans to Arize AX. See [src/scripts/generate-traces.ts](src/scripts/generate-traces.ts) to adjust the prompts. +This sends a curated set of ~50 prompts to the orchestrator agent and flushes the resulting spans to Arize AX. The prompts live in [src/scripts/example-prompts.json](src/scripts/example-prompts.json), grouped by the tool path each one exercises — edit that file to add or change prompts. ## How it works diff --git a/typescript/cookbooks/mastra-arize-ax-tracing/src/scripts/example-prompts.json b/typescript/cookbooks/mastra-arize-ax-tracing/src/scripts/example-prompts.json new file mode 100644 index 0000000..550b163 --- /dev/null +++ b/typescript/cookbooks/mastra-arize-ax-tracing/src/scripts/example-prompts.json @@ -0,0 +1,64 @@ +{ + "weather-only": [ + "What's the weather in Tokyo right now?", + "Is it raining in Seattle at the moment?", + "How hot does it feel in Dubai today?", + "Give me the current conditions in Reykjavik.", + "What's the temperature in Nairobi right now?", + "Is it windy in Wellington today?", + "How humid is it in Bangkok at the moment?", + "What are the current conditions in Buenos Aires?", + "Is it cold in Helsinki today?", + "Tell me the current weather in Vancouver.", + "How's the weather in Mexico City right now?", + "What's it like outside in Cairo?" + ], + "weather+planning": [ + "What should I do in Barcelona today?", + "Give me some activity ideas for a day out in Amsterdam.", + "I'm spending the day in Cape Town — what's good to do given the weather?", + "Recommend things to do in Lisbon today.", + "What's a good way to spend the afternoon in Berlin?", + "Plan a fun day for me in Sydney.", + "I have a free day in Kyoto — what should I do?", + "Suggest some outdoor activities in Vienna today.", + "What can I do with the kids in Copenhagen today?", + "I'm visiting Edinburgh — recommend a day's itinerary.", + "What's good to do in Marrakech given today's weather?", + "Help me plan a relaxed day in Porto." + ], + "weather+analysis+planning": [ + "Give me a detailed weather analysis and a full activity plan for London today.", + "Analyze the conditions in Reykjavik and plan my whole day around them.", + "I want an in-depth look at the weather in Singapore plus a complete itinerary for today.", + "Break down today's weather in Toronto and build me a full day plan.", + "Give me a thorough weather assessment for Rome and a detailed schedule.", + "Analyze the forecast for Oslo and create a complete day itinerary.", + "I need a deep dive on Chicago's weather and a full plan around it.", + "Assess the conditions in Auckland and map out my entire day.", + "Detailed weather breakdown and full-day plan for Seoul, please.", + "Examine today's weather in Athens and plan everything for me.", + "Give me a complete analysis and itinerary for a day in Prague.", + "Walk me through the weather in Cape Town and a full activity schedule." + ], + "multi-location": [ + "Is it better weather for a long walk in Oslo or in Lisbon right now, and what should I do there?", + "Compare the weather in Madrid and Rome today and tell me which is better for sightseeing.", + "Which has nicer weather right now — San Francisco or Los Angeles — and what should I do?", + "I'm choosing between Berlin and Munich for a day trip; which has better weather and what should I do?", + "Should I go to the beach in Nice or Barcelona today based on the weather?", + "Compare conditions in New York and Boston and suggest activities for the better one." + ], + "vague-intent": [ + "I might head outside in Chicago later... or maybe not. What do you reckon?", + "Not sure what to do today in Dublin, help me out?", + "Thinking about going out in Melbourne, but I'm undecided — thoughts?", + "I was wondering if you could help me figure out my day in Glasgow." + ], + "activity-constrained": [ + "Plan an indoor-friendly day in Mumbai in case the weather is rough.", + "I want only outdoor activities in Lisbon today — is that doable given the weather?", + "Suggest a rainy-day plan for Manchester.", + "Plan a day of mostly walking and parks in Munich if the weather allows." + ] +} diff --git a/typescript/cookbooks/mastra-arize-ax-tracing/src/scripts/generate-traces.ts b/typescript/cookbooks/mastra-arize-ax-tracing/src/scripts/generate-traces.ts index 7474cef..ec0f6a2 100644 --- a/typescript/cookbooks/mastra-arize-ax-tracing/src/scripts/generate-traces.ts +++ b/typescript/cookbooks/mastra-arize-ax-tracing/src/scripts/generate-traces.ts @@ -1,14 +1,17 @@ /** * Sends a diverse set of prompts to the WeatherOrchestratorAgent to generate - * traces in Arize AX. Each prompt is chosen to exercise a different tool path: + * traces in Arize AX. The prompts live in ./example-prompts.json, grouped by + * the tool path each one is meant to exercise: * * - weather-only → weatherTool * - weather+planning → weatherTool → activityPlanningTool * - weather+analysis+plan → weatherTool → weatherAnalysisTool → activityPlanningTool + * - multi-location / vague-intent / activity-constrained → varied phrasing * * Run with: npm run generate-traces * Requires the same env vars as the agent: ARIZE_SPACE_ID, ARIZE_API_KEY, OPENAI_API_KEY. */ +import { readFileSync } from 'node:fs'; import { fileURLToPath } from 'node:url'; // The Mastra config points its storage at `file:../mastra.db`, resolved relative @@ -23,27 +26,14 @@ interface Example { prompt: string; } -const examples: Example[] = [ - // Simple weather lookups — the orchestrator should call weatherTool and stop. - { category: 'weather-only', prompt: "What's the weather in Tokyo right now?" }, - { category: 'weather-only', prompt: 'Is it raining in Seattle at the moment?' }, - { category: 'weather-only', prompt: 'How hot does it feel in Dubai today?' }, +// Prompts grouped by category in the JSON file; flattened to a run list here. +const promptsByCategory = JSON.parse( + readFileSync(new URL('./example-prompts.json', import.meta.url), 'utf8'), +) as Record; - // Activity requests — weatherTool then activityPlanningTool. - { category: 'weather+planning', prompt: 'What should I do in Barcelona today?' }, - { category: 'weather+planning', prompt: 'Give me some activity ideas for a day out in Amsterdam.' }, - { category: 'weather+planning', prompt: "I'm spending the day in Cape Town — what's good to do given the weather?" }, - - // Deep dives — the full weatherTool → weatherAnalysisTool → activityPlanningTool chain. - { category: 'weather+analysis+planning', prompt: 'Give me a detailed weather analysis and a full activity plan for London today.' }, - { category: 'weather+analysis+planning', prompt: 'Analyze the conditions in Reykjavik and plan my whole day around them.' }, - { category: 'weather+analysis+planning', prompt: 'I want an in-depth look at the weather in Singapore plus a complete itinerary for today.' }, - - // Varied phrasing / edge cases for span diversity. - { category: 'vague-intent', prompt: 'I might head outside in Chicago later... or maybe not. What do you reckon?' }, - { category: 'multi-location', prompt: 'Is it better weather for a long walk in Oslo or in Lisbon right now, and what should I do there?' }, - { category: 'activity-constrained', prompt: 'Plan an indoor-friendly day in Mumbai in case the weather is rough.' }, -]; +const examples: Example[] = Object.entries(promptsByCategory).flatMap( + ([category, prompts]) => prompts.map((prompt) => ({ category, prompt })), +); async function main(): Promise { const agent = mastra.getAgent('weatherOrchestratorAgent'); From 444ab1044b55f0f6453a13fb2069081a5f28962b Mon Sep 17 00:00:00 2001 From: Jim Bennett Date: Wed, 24 Jun 2026 15:58:55 -0700 Subject: [PATCH 10/11] docs: add text language hint to diagram code fence Co-Authored-By: Claude Opus 4.8 (1M context) --- typescript/cookbooks/mastra-arize-ax-tracing/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/typescript/cookbooks/mastra-arize-ax-tracing/README.md b/typescript/cookbooks/mastra-arize-ax-tracing/README.md index e696baf..477da0c 100644 --- a/typescript/cookbooks/mastra-arize-ax-tracing/README.md +++ b/typescript/cookbooks/mastra-arize-ax-tracing/README.md @@ -44,7 +44,7 @@ This sends a curated set of ~50 prompts to the orchestrator agent and flushes th A single orchestrator agent coordinates three tools in sequence: -``` +```text User request ↓ Weather Orchestrator Agent From 8666e5cf45d4f9cdd0a7a2affba8c1782aac3eb0 Mon Sep 17 00:00:00 2001 From: Jim Bennett Date: Wed, 24 Jun 2026 18:40:43 -0700 Subject: [PATCH 11/11] chore: remove building-a-custom-evaluator notebook Source for the Arize docs "Creating a Custom LLM Evaluator with a Benchmark Dataset" cookbook, which is being consolidated into "Align LLM Evals with Human Judgment" (Arize-ai/docs#650). No other notebooks or READMEs reference it. Folds in the change from #70. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../building-a-custom-evaluator.ipynb | 602 ------------------ 1 file changed, 602 deletions(-) delete mode 100644 python/llm/evaluation/building-a-custom-evaluator.ipynb diff --git a/python/llm/evaluation/building-a-custom-evaluator.ipynb b/python/llm/evaluation/building-a-custom-evaluator.ipynb deleted file mode 100644 index 4a1aa32..0000000 --- a/python/llm/evaluation/building-a-custom-evaluator.ipynb +++ /dev/null @@ -1,602 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "hdr-logo", - "metadata": {}, - "source": [ - "
\n", - "

\n", - " \"arize\n", - "
\n", - " Docs\n", - " |\n", - " GitHub\n", - " |\n", - " Slack Community\n", - "

\n", - "
" - ] - }, - { - "cell_type": "markdown", - "id": "title", - "metadata": {}, - "source": [ - "# Using a Benchmark Dataset to Build a Custom LLM as a Judge Evaluator" - ] - }, - { - "cell_type": "markdown", - "id": "intro", - "metadata": {}, - "source": [ - "In this tutorial, you'll learn how to build a custom LLM-as-a-Judge Evaluator tailored to your specific use case. While Arize AX provides several [pre-built evaluators](https://arize.com/docs/ax/evaluate/llm-as-a-judge/arize-evaluators-llm-as-a-judge) that have been tested against benchmark datasets, these may not always cover the nuances of your application.\n", - "\n", - "So how can you achieve the same level of rigor when your use case falls outside the scope of standard evaluators?\n", - "\n", - "The discipline is simple: **don't trust a judge until you've measured it against ground truth.** We'll build a small benchmark dataset from a handful of human-annotated examples, then use it to build and refine a custom evaluator — measuring how often the judge agrees with the human labels, and iterating the judge prompt until that agreement is high enough to trust. The use case we'll explore is data extraction from an image of a receipt.\n", - "\n", - "To follow along, you'll need:\n", - "\n", - "* A free [Arize AX](https://app.arize.com/auth/join) account\n", - "* An OpenAI API Key" - ] - }, - { - "cell_type": "markdown", - "id": "setup-hdr", - "metadata": {}, - "source": [ - "## Set up Keys and Dependencies" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "pip", - "metadata": {}, - "outputs": [], - "source": [ - "%pip install -qqqq arize arize-otel openinference-instrumentation-openai openai nest_asyncio pandas" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "keys", - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "import pandas as pd\n", - "import nest_asyncio\n", - "from getpass import getpass\n", - "\n", - "nest_asyncio.apply()\n", - "\n", - "\n", - "if \"SPACE_ID\" not in os.environ:\n", - " os.environ[\"SPACE_ID\"] = getpass(\"🔑 Enter your Arize Space ID: \")\n", - "\n", - "if \"API_KEY\" not in os.environ:\n", - " os.environ[\"API_KEY\"] = getpass(\"🔑 Enter your Arize API Key: \")\n", - "\n", - "if \"OPENAI_API_KEY\" not in os.environ:\n", - " os.environ[\"OPENAI_API_KEY\"] = getpass(\"🔑 Enter your OpenAI API Key: \")" - ] - }, - { - "cell_type": "markdown", - "id": "tracing-hdr", - "metadata": {}, - "source": [ - "# Configure Tracing" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "register", - "metadata": {}, - "outputs": [], - "source": [ - "from arize.otel import register\n", - "from openinference.instrumentation.openai import OpenAIInstrumentor\n", - "\n", - "MODEL = \"gpt-5.4-mini\"\n", - "JUDGE_MODEL = \"gpt-4.1\"\n", - "\n", - "tracer_provider = register(\n", - " space_id=os.environ[\"SPACE_ID\"],\n", - " api_key=os.environ[\"API_KEY\"],\n", - " project_name=\"receipt-classifications\",\n", - ")\n", - "OpenAIInstrumentor().instrument(tracer_provider=tracer_provider)" - ] - }, - { - "cell_type": "markdown", - "id": "traces-hdr", - "metadata": {}, - "source": [ - "# Generate Image Classification Traces" - ] - }, - { - "cell_type": "markdown", - "id": "traces-desc", - "metadata": {}, - "source": [ - "In this tutorial, we'll ask an LLM to generate expense reports from receipt images provided as public URLs. Running the cells below will generate traces, which you can explore directly in Arize AX for annotation. We'll use `gpt-5.4-mini`, which supports image inputs.\n", - "\n", - "\n", - "Dataset Information:\n", - "Jakob (2024). Receipt or Invoice Dataset. Roboflow Universe. CC BY 4.0. Available at: https://universe.roboflow.com/jakob-awn1e/receipt-or-invoice (accessed on 2025-07-29)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "urls", - "metadata": {}, - "outputs": [], - "source": [ - "import uuid\n", - "import pandas as pd\n", - "urls = [\n", - "\"https://source.roboflow.com/Zf1kEIcRTrhHBZ7wgJleS4E92P23/8M5px2yLoNtZ6gOQ2r1D/original.jpg\",\n", - "\"https://source.roboflow.com/Zf1kEIcRTrhHBZ7wgJleS4E92P23/8EVgYMNObyV6kLqBNeFG/original.jpg\",\n", - "\"https://source.roboflow.com/Zf1kEIcRTrhHBZ7wgJleS4E92P23/86aohWmcEfO0XkflO8AB/original.jpg\",\n", - "\"https://source.roboflow.com/HahhKcbQqdf8YAudM4kU3PuVCS72/1eGPBChz7wvovQROk2l8/original.jpg\",\n", - "\"https://source.roboflow.com/HahhKcbQqdf8YAudM4kU3PuVCS72/0WqR2GSfGmxWB7ozo3Pj/original.jpg\",\n", - "\"https://source.roboflow.com/HahhKcbQqdf8YAudM4kU3PuVCS72/FAEJRtviIboCYSKFZcEZ/original.jpg\",\n", - "\"https://source.roboflow.com/HahhKcbQqdf8YAudM4kU3PuVCS72/0AoEaFy8FAw6DVieWCa8/original.jpg\",\n", - "\"https://source.roboflow.com/HahhKcbQqdf8YAudM4kU3PuVCS72/0Q3hAyNwXNpHTeoWU7fz/original.jpg\",\n", - "\"https://source.roboflow.com/HahhKcbQqdf8YAudM4kU3PuVCS72/2r876u4WpaCYFdMPwieK/original.jpg\",\n", - "\"https://source.roboflow.com/HahhKcbQqdf8YAudM4kU3PuVCS72/2ZWeE0yO0oJUDtpgEAPY/original.jpg\",\n", - "\"https://source.roboflow.com/HahhKcbQqdf8YAudM4kU3PuVCS72/37PF6xfHyuqzIBdO7Kgw/original.jpg\",\n", - "\"https://source.roboflow.com/Zf1kEIcRTrhHBZ7wgJleS4E92P23/6mo4M0nJeKZEsdKrRfsR/original.jpg\",\n", - "\"https://source.roboflow.com/Zf1kEIcRTrhHBZ7wgJleS4E92P23/5ezJ8tUBGbNnt0jZi2JU/original.jpg\",\n", - "\"https://source.roboflow.com/Zf1kEIcRTrhHBZ7wgJleS4E92P23/4BCIWGazhCj03oTMWboO/original.jpg\",\n", - "\"https://source.roboflow.com/Zf1kEIcRTrhHBZ7wgJleS4E92P23/4B8vXJNwJ7ZuHEWyjgAv/original.jpg\",\n", - "\"https://source.roboflow.com/Zf1kEIcRTrhHBZ7wgJleS4E92P23/2EpeKbAqsSwciH2IHGyV/original.jpg\",\n", - "\"https://source.roboflow.com/Zf1kEIcRTrhHBZ7wgJleS4E92P23/2LP3g9rKZrYDkNB3I78c/original.jpg\",\n", - "\"https://source.roboflow.com/Zf1kEIcRTrhHBZ7wgJleS4E92P23/1hT6iLEIAFBw8W70u2FY/original.jpg\",\n", - "\"https://source.roboflow.com/Zf1kEIcRTrhHBZ7wgJleS4E92P23/1zaKpaDhRPxkiIDTvMuc/original.jpg\",\n", - "\"https://source.roboflow.com/Zf1kEIcRTrhHBZ7wgJleS4E92P23/1hF1R2Pt41hnlqhlXLDD/original.jpg\"\n", - "]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "extract", - "metadata": {}, - "outputs": [], - "source": [ - "from openai import OpenAI\n", - "client = OpenAI()\n", - "\n", - "def extract_receipt_data(input):\n", - " response = client.chat.completions.create(\n", - " model=MODEL,\n", - " messages=[\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": [\n", - " {\"type\": \"text\", \"text\": \"Analyze this receipt and return a brief summary for an expense report. Only include category of expense, total cost, and summary of items\"},\n", - " {\n", - " \"type\": \"image_url\",\n", - " \"image_url\": {\n", - " \"url\": input,\n", - " },\n", - " },\n", - " ],\n", - " }\n", - " ],\n", - " )\n", - " return response" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "run-extract", - "metadata": {}, - "outputs": [], - "source": [ - "import time\n", - "\n", - "for url in urls:\n", - " extract_receipt_data(url)\n", - "\n", - "# Flush the traces so they're queryable when you annotate / export them.\n", - "tracer_provider.force_flush()\n", - "time.sleep(5)" - ] - }, - { - "cell_type": "markdown", - "id": "dataset-hdr", - "metadata": {}, - "source": [ - "# Create Benchmarked Dataset" - ] - }, - { - "cell_type": "markdown", - "id": "annotate-desc", - "metadata": {}, - "source": [ - "After generating traces, open Arize AX to begin annotating your dataset. In this example, we'll annotate based on \"accuracy\", but you can choose any evaluation criterion that fits your use case. Just be sure to update the query below to match the annotation key you're using—this ensures the annotated examples are included in your benchmark dataset.\n", - "\n", - "Run the cell below to see annotations in action:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "video", - "metadata": {}, - "outputs": [], - "source": [ - "from IPython.display import HTML\n", - "\n", - "video_url = \"https://storage.googleapis.com/arize-phoenix-assets/assets/videos/arize-annotation.mp4\"\n", - "\n", - "HTML(f\"\"\"\n", - "\n", - "\"\"\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "export", - "metadata": {}, - "outputs": [], - "source": [ - "from datetime import datetime, timedelta, timezone\n", - "\n", - "from arize.client import ArizeClient\n", - "\n", - "ax_client = ArizeClient(api_key=os.environ[\"API_KEY\"])\n", - "\n", - "print(\"#### Exporting your traces into a dataframe.\")\n", - "\n", - "primary_df = ax_client.spans.export_to_df(\n", - " space_id=os.environ[\"SPACE_ID\"],\n", - " project_name=\"receipt-classifications\",\n", - " start_time=datetime.now(timezone.utc) - timedelta(days=50),\n", - " end_time=datetime.now(timezone.utc),\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "filter", - "metadata": {}, - "outputs": [], - "source": [ - "filtered_df = primary_df[\n", - " (primary_df[\"annotation.accuracy.label\"].notna())\n", - "][[\n", - " \"attributes.input.value\",\n", - " \"attributes.output.value\",\n", - " \"annotation.accuracy.label\",\n", - "]].rename(columns={\n", - " \"attributes.input.value\": \"image\",\n", - " \"attributes.output.value\": \"response\",\n", - " \"annotation.accuracy.label\": \"accuracy\"\n", - "})\n", - "\n", - "filtered_df" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "flatten", - "metadata": {}, - "outputs": [], - "source": [ - "import json\n", - "\n", - "def extract_url(input_value):\n", - " data = json.loads(input_value)\n", - " return data[\"messages\"][0][\"content\"][1][\"image_url\"][\"url\"]\n", - "\n", - "def extract_content(input_value):\n", - " data = json.loads(input_value)\n", - " return data[\"choices\"][0][\"message\"][\"content\"]\n", - "\n", - "filtered_df[\"image\"] = filtered_df[\"image\"].apply(extract_url)\n", - "filtered_df[\"response\"] = filtered_df[\"response\"].apply(extract_content)\n", - "\n", - "\n", - "filtered_df" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "create-dataset", - "metadata": {}, - "outputs": [], - "source": [ - "from datetime import datetime, timezone\n", - "\n", - "DATASET_NAME = f\"annotated-receipts-{datetime.now(timezone.utc):%Y%m%d-%H%M%S}\"\n", - "dataset = ax_client.datasets.create(\n", - " space=os.environ[\"SPACE_ID\"],\n", - " name=DATASET_NAME,\n", - " examples=filtered_df,\n", - ")\n", - "DATASET_NAME" - ] - }, - { - "cell_type": "markdown", - "id": "dataset-img", - "metadata": {}, - "source": [ - "![Dataset](https://storage.googleapis.com/arize-phoenix-assets/assets/images/eval-tutorial-annotated-dataset.png)" - ] - }, - { - "cell_type": "markdown", - "id": "template-hdr", - "metadata": {}, - "source": [ - "# Create the custom judge" - ] - }, - { - "cell_type": "markdown", - "id": "template-desc", - "metadata": {}, - "source": [ - "Next we'll build a baseline judge and benchmark it against the annotations. The judge is an LLM-as-a-Judge that *reads the receipt image* and the model's expense report, and classifies the report as `accurate`, `almost accurate`, or `inaccurate` — the same labels a human annotator used. `make_judge(prompt)` binds one judge prompt into a task we can run as an experiment; the experiment's evaluator then checks whether the judge's label matches the human annotation." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "judge-v1", - "metadata": {}, - "outputs": [], - "source": [ - "RAILS = [\"accurate\", \"almost accurate\", \"inaccurate\"]\n", - "\n", - "JUDGE_PROMPT_V1 = \"\"\"You are an evaluator tasked with assessing the quality of a model-generated expense report based on a receipt.\n", - "\n", - "MODEL OUTPUT (Expense Report):\n", - "{output}\n", - "\n", - "The input receipt image is attached. Evaluate the report and assign exactly one label:\n", - "- \"accurate\" - Fully correct\n", - "- \"almost accurate\" - Mostly correct\n", - "- \"inaccurate\" - Substantially wrong\n", - "\n", - "Respond with only the label.\"\"\"\n", - "\n", - "\n", - "def make_judge(prompt):\n", - " \"\"\"Bind one judge prompt into an experiment task. The task reads the receipt\n", - " image and the model's report from the dataset row and returns one rail label.\"\"\"\n", - "\n", - " def task_function(dataset_row):\n", - " response = client.chat.completions.create(\n", - " model=JUDGE_MODEL,\n", - " temperature=0,\n", - " messages=[\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": [\n", - " {\"type\": \"text\", \"text\": prompt.format(output=dataset_row[\"response\"])},\n", - " {\"type\": \"image_url\", \"image_url\": {\"url\": dataset_row[\"image\"]}},\n", - " ],\n", - " }\n", - " ],\n", - " )\n", - " verdict = response.choices[0].message.content.strip().lower()\n", - " # Map the reply onto one of the rails (check the longer labels first, since\n", - " # \"accurate\" is a substring of \"almost accurate\").\n", - " for rail in [\"almost accurate\", \"inaccurate\", \"accurate\"]:\n", - " if rail in verdict:\n", - " return rail\n", - " return verdict\n", - "\n", - " return task_function" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "evaluator", - "metadata": {}, - "outputs": [], - "source": [ - "from arize.experiments import EvaluationResult\n", - "\n", - "\n", - "def matches_annotation(output, dataset_row) -> EvaluationResult:\n", - " \"\"\"Score the judge: does its label match the human annotation for this example?\"\"\"\n", - " expected = dataset_row[\"accuracy\"]\n", - " if output == expected:\n", - " return EvaluationResult(\n", - " score=1.0, label=\"correct\", explanation=\"Judge label matches the human annotation\"\n", - " )\n", - " return EvaluationResult(\n", - " score=0.0,\n", - " label=\"incorrect\",\n", - " explanation=f\"Judge said '{output}', annotation was '{expected}'\",\n", - " )" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "run-v1", - "metadata": {}, - "outputs": [], - "source": [ - "experiment_v1, results_v1 = ax_client.experiments.run(\n", - " space=os.environ[\"SPACE_ID\"],\n", - " dataset=DATASET_NAME,\n", - " task=make_judge(JUDGE_PROMPT_V1),\n", - " evaluators=[matches_annotation],\n", - " name=\"Initial Experiment\",\n", - ")\n", - "results_v1[\"eval.matches_annotation.score\"].mean()" - ] - }, - { - "cell_type": "markdown", - "id": "iter1-hdr", - "metadata": {}, - "source": [ - "# Iteration 1 to improve the evaluator prompt" - ] - }, - { - "cell_type": "markdown", - "id": "iter1-desc", - "metadata": {}, - "source": [ - "Next, we'll refine our judge prompt by adding more specific classification rules. We can add these rules based on gaps we saw in the previous iteration. This additional guidance helps improve accuracy and ensures the evaluator's judgments better align with human expectations." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "judge-v2", - "metadata": {}, - "outputs": [], - "source": [ - "JUDGE_PROMPT_V2 = \"\"\"You are an evaluator tasked with assessing the quality of a model-generated expense report based on a receipt.\n", - "\n", - "MODEL OUTPUT (Expense Report):\n", - "{output}\n", - "\n", - "The input receipt image is attached. Evaluate the report and assign exactly one label:\n", - "- \"accurate\" - Total price, itemized list, and expense category are all accurate. All three must be correct to get this label.\n", - "- \"almost accurate\" - Mostly correct but with small issues. For example, the expense category is too vague.\n", - "- \"inaccurate\" - Substantially wrong or missing information. For example, an incorrect total price.\n", - "\n", - "Respond with only the label.\"\"\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "run-v2", - "metadata": {}, - "outputs": [], - "source": [ - "experiment_v2, results_v2 = ax_client.experiments.run(\n", - " space=os.environ[\"SPACE_ID\"],\n", - " dataset=DATASET_NAME,\n", - " task=make_judge(JUDGE_PROMPT_V2),\n", - " evaluators=[matches_annotation],\n", - " name=\"Stronger Prompt Experiment\",\n", - ")\n", - "results_v2[\"eval.matches_annotation.score\"].mean()" - ] - }, - { - "cell_type": "markdown", - "id": "iter2-hdr", - "metadata": {}, - "source": [ - "# Iteration 2 to improve the evaluator prompt" - ] - }, - { - "cell_type": "markdown", - "id": "iter2-desc", - "metadata": {}, - "source": [ - "To further improve our evaluator, we'll add few-shot guidance to the prompt. These examples help highlight common failure cases and guide the evaluator toward more consistent and generalized judgments." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "judge-v3", - "metadata": {}, - "outputs": [], - "source": [ - "JUDGE_PROMPT_V3 = \"\"\"You are an evaluator tasked with assessing the quality of a model-generated expense report based on a receipt.\n", - "\n", - "MODEL OUTPUT (Expense Report):\n", - "{output}\n", - "\n", - "The input receipt image is attached. Evaluate the report and assign exactly one label:\n", - "- \"accurate\" - Total price, itemized list, and expense category are accurate. All three must be correct to get this label.\n", - " An incorrect category is one that is overly vague (e.g., \"Miscellaneous\", \"Supplies\") or does not accurately reflect the itemized list.\n", - " For example, \"Dining and Entertainment\" should not be grouped together if the itemized list only includes food.\n", - " Reasonable general categories like \"Office Supplies\" or \"Groceries\" are acceptable if they align with the listed items.\n", - "- \"almost accurate\" - Mostly correct but with small issues. For example, the expense category is too vague.\n", - " If a category includes extra fields (e.g., \"Dining and Entertainment\" but the receipt only includes food) mark this as almost accurate.\n", - "- \"inaccurate\" - Substantially wrong or missing. For example, an incorrect total price, or one or more missing items, makes the result inaccurate.\n", - "\n", - "Respond with only the label.\"\"\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "run-v3", - "metadata": {}, - "outputs": [], - "source": [ - "experiment_v3, results_v3 = ax_client.experiments.run(\n", - " space=os.environ[\"SPACE_ID\"],\n", - " dataset=DATASET_NAME,\n", - " task=make_judge(JUDGE_PROMPT_V3),\n", - " evaluators=[matches_annotation],\n", - " name=\"Few Shot Experiment\",\n", - ")\n", - "results_v3[\"eval.matches_annotation.score\"].mean()" - ] - }, - { - "cell_type": "markdown", - "id": "final-hdr", - "metadata": {}, - "source": [ - "# Final Results" - ] - }, - { - "cell_type": "markdown", - "id": "final-desc", - "metadata": {}, - "source": [ - "Each experiment reports the share of examples where the judge's label matched the human annotation — that agreement rate is how you know whether to trust the judge. Compare the three runs in the **Experiments** tab of your dataset (or from the `results_*` dataframes above) and watch the agreement climb as the prompt is refined.\n", - "\n", - "Once your evaluator reaches a performance level you're satisfied with, it's ready for use. The target score will depend on your benchmark dataset and specific use case. That said, you can continue applying the techniques from this tutorial to refine and iterate until the evaluator meets your desired level of quality." - ] - }, - { - "cell_type": "markdown", - "id": "final-img", - "metadata": {}, - "source": [ - "![Final Results](https://storage.googleapis.com/arize-phoenix-assets/assets/images/eval-tutorial-compare-experiment.png)" - ] - } - ], - "metadata": { - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -}