From 8e4acf5df993452da52bd912190e50f1ac78a164 Mon Sep 17 00:00:00 2001 From: Alexander Song Date: Thu, 27 Aug 2026 17:16:52 -0400 Subject: [PATCH 1/2] fix(agents): record the tool name on pxi llm tool schemas Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01UuBCzjRLsRa9cLCJQVw2yu --- .../pages/trace/span/LLMToolSchemasList.tsx | 4 ++- .../pages/trace/span/__tests__/utils.test.ts | 31 +++++++++++++++++++ js/app/src/pages/trace/span/utils.ts | 26 +++++++++++++++- .../openinference_model_wrapper.py | 8 ++--- .../test_openinference_model_wrapper.py | 14 +++++---- 5 files changed, 71 insertions(+), 12 deletions(-) diff --git a/js/app/src/pages/trace/span/LLMToolSchemasList.tsx b/js/app/src/pages/trace/span/LLMToolSchemasList.tsx index 3efd9e96309..1450bdb148e 100644 --- a/js/app/src/pages/trace/span/LLMToolSchemasList.tsx +++ b/js/app/src/pages/trace/span/LLMToolSchemasList.tsx @@ -11,6 +11,7 @@ import { SpanKindIcon } from "@phoenix/components/trace"; import { defaultCardProps } from "./constants"; import { MimeTypeCodeBlock } from "./MimeTypeCodeBlock"; +import { getToolSchemaName } from "./utils"; /** * A card displaying a single tool JSON schema available to the LLM. @@ -22,10 +23,11 @@ function LLMToolSchema({ toolSchema: string; index: number; }) { + const name = getToolSchemaName(toolSchema); const titleEl = ( - Tool + {name == null ? "Tool" : `Tool: ${name}`} ); diff --git a/js/app/src/pages/trace/span/__tests__/utils.test.ts b/js/app/src/pages/trace/span/__tests__/utils.test.ts index df929a61b22..15ed143317e 100644 --- a/js/app/src/pages/trace/span/__tests__/utils.test.ts +++ b/js/app/src/pages/trace/span/__tests__/utils.test.ts @@ -10,6 +10,7 @@ import { getRerankerAttributes, getRetrieverAttributes, getToolAttributes, + getToolSchemaName, groupDocumentEvaluationsByPosition, parseSpanAttributes, } from "../utils"; @@ -382,6 +383,36 @@ describe("getToolAttributes", () => { }); }); +describe("getToolSchemaName", () => { + it("reads the name of an openai style tool definition", () => { + expect( + getToolSchemaName( + '{"type":"function","function":{"name":"get_weather","parameters":{"type":"object"}}}' + ) + ).toBe("get_weather"); + }); + + it("reads the name of an anthropic style tool definition", () => { + expect( + getToolSchemaName( + '{"name":"get_weather","input_schema":{"type":"object"}}' + ) + ).toBe("get_weather"); + }); + + it("falls back to the title of a bare parameter schema", () => { + expect(getToolSchemaName('{"title":"get_weather","type":"object"}')).toBe( + "get_weather" + ); + }); + + it("returns undefined when the schema names no tool", () => { + expect(getToolSchemaName('{"type":"object"}')).toBeUndefined(); + expect(getToolSchemaName('{"name":""}')).toBeUndefined(); + expect(getToolSchemaName("not json")).toBeUndefined(); + }); +}); + describe("groupDocumentEvaluationsByPosition", () => { const makeEvaluation = ( documentPosition: number, diff --git a/js/app/src/pages/trace/span/utils.ts b/js/app/src/pages/trace/span/utils.ts index 408c134ab0b..d9273f172d4 100644 --- a/js/app/src/pages/trace/span/utils.ts +++ b/js/app/src/pages/trace/span/utils.ts @@ -23,7 +23,10 @@ import { toRecordPreview, toToolCallsPreview, } from "@phoenix/utils/contentPreviewUtils"; -import { safelyParseJSON } from "@phoenix/utils/jsonUtils"; +import { + safelyParseJSON, + safelyParseJSONObjectString, +} from "@phoenix/utils/jsonUtils"; import type { AttributeObject, @@ -290,6 +293,27 @@ export function getEmbeddingAttributes(spanAttributes: AttributeObject): { }; } +/** + * The name of the tool a JSON schema recorded on an LLM span describes. + * + * The schema is the tool definition as the model provider's SDK received it, + * so where the name sits depends on the provider: nested under `function` for + * OpenAI-style definitions, at the top level for Anthropic-style ones, and in + * the JSON Schema `title` when an instrumentation recorded only the parameter + * schema. + */ +export function getToolSchemaName(toolSchema: string): string | undefined { + const schema = safelyParseJSONObjectString(toolSchema) as + | Record + | undefined; + if (schema == null) { + return undefined; + } + const fn = (schema.function ?? {}) as Record; + const name = fn.name ?? schema.name ?? schema.title; + return typeof name === "string" && name !== "" ? name : undefined; +} + /** * The attributes describing the tool of a tool span. */ diff --git a/src/phoenix/server/agents/pydantic_ai/openinference_model_wrapper.py b/src/phoenix/server/agents/pydantic_ai/openinference_model_wrapper.py index acaf10db0ca..cafe0470130 100644 --- a/src/phoenix/server/agents/pydantic_ai/openinference_model_wrapper.py +++ b/src/phoenix/server/agents/pydantic_ai/openinference_model_wrapper.py @@ -258,11 +258,11 @@ def _response_to_oi_message(msg: ModelResponse) -> Message: def _to_oi_tools(params: ModelRequestParameters) -> list[Tool]: tools: list[Tool] = [] for tool_def in params.function_tools or []: - schema: dict[str, Any] = {**tool_def.parameters_json_schema} - schema.setdefault("title", tool_def.name) + function: dict[str, Any] = {"name": tool_def.name} if tool_def.description: - schema.setdefault("description", tool_def.description) - tools.append({"json_schema": schema}) + function["description"] = tool_def.description + function["parameters"] = tool_def.parameters_json_schema + tools.append({"json_schema": {"type": "function", "function": function}}) return tools diff --git a/tests/unit/server/agents/pydantic_ai/test_openinference_model_wrapper.py b/tests/unit/server/agents/pydantic_ai/test_openinference_model_wrapper.py index 826d6158d45..599284e96fe 100644 --- a/tests/unit/server/agents/pydantic_ai/test_openinference_model_wrapper.py +++ b/tests/unit/server/agents/pydantic_ai/test_openinference_model_wrapper.py @@ -293,9 +293,10 @@ async def test_request_emits_llm_span_for_tool_call_response( tool_schema_attr = attributes.pop(f"{LLM_TOOLS}.0.{TOOL_JSON_SCHEMA}") assert isinstance(tool_schema_attr, str) tool_schema = json.loads(tool_schema_attr) - assert tool_schema["title"] == "get_weather" - assert tool_schema["description"] == "Look up the current weather for a city." - assert tool_schema["required"] == ["city"] + assert tool_schema["type"] == "function" + assert tool_schema["function"]["name"] == "get_weather" + assert tool_schema["function"]["description"] == "Look up the current weather for a city." + assert tool_schema["function"]["parameters"] == weather_tool.parameters_json_schema assert attributes.pop(f"{LLM_INPUT_MESSAGES}.0.{MESSAGE_ROLE}") == "system" assert isinstance(attributes.pop(f"{LLM_INPUT_MESSAGES}.0.{MESSAGE_CONTENT}"), str) @@ -581,9 +582,10 @@ async def test_request_emits_tool_return_message_in_history( tool_schema_attr = attributes.pop(f"{LLM_TOOLS}.0.{TOOL_JSON_SCHEMA}") assert isinstance(tool_schema_attr, str) tool_schema = json.loads(tool_schema_attr) - assert tool_schema["title"] == "get_weather" - assert tool_schema["description"] == "Look up the current weather for a city." - assert tool_schema["required"] == ["city"] + assert tool_schema["type"] == "function" + assert tool_schema["function"]["name"] == "get_weather" + assert tool_schema["function"]["description"] == "Look up the current weather for a city." + assert tool_schema["function"]["parameters"] == weather_tool.parameters_json_schema # Message 0: system prompt. assert attributes.pop(f"{LLM_INPUT_MESSAGES}.0.{MESSAGE_ROLE}") == "system" From 2510a48ee0cde72f949c734459751a1b93f8da72 Mon Sep 17 00:00:00 2001 From: Alexander Song Date: Fri, 28 Aug 2026 11:00:29 -0400 Subject: [PATCH 2/2] clean --- js/app/src/pages/trace/span/utils.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/js/app/src/pages/trace/span/utils.ts b/js/app/src/pages/trace/span/utils.ts index d9273f172d4..e9390e2a645 100644 --- a/js/app/src/pages/trace/span/utils.ts +++ b/js/app/src/pages/trace/span/utils.ts @@ -296,11 +296,7 @@ export function getEmbeddingAttributes(spanAttributes: AttributeObject): { /** * The name of the tool a JSON schema recorded on an LLM span describes. * - * The schema is the tool definition as the model provider's SDK received it, - * so where the name sits depends on the provider: nested under `function` for - * OpenAI-style definitions, at the top level for Anthropic-style ones, and in - * the JSON Schema `title` when an instrumentation recorded only the parameter - * schema. + * Makes a best-effort attempt to parse the name out of the schema. */ export function getToolSchemaName(toolSchema: string): string | undefined { const schema = safelyParseJSONObjectString(toolSchema) as