diff --git a/js/app/src/pages/playground/__tests__/playgroundUtils.test.ts b/js/app/src/pages/playground/__tests__/playgroundUtils.test.ts index cf502a121a2..8f49e189f6a 100644 --- a/js/app/src/pages/playground/__tests__/playgroundUtils.test.ts +++ b/js/app/src/pages/playground/__tests__/playgroundUtils.test.ts @@ -31,6 +31,7 @@ import { getToolsFromAttributes, getResponseFormatFromAttributes, getToolChoiceFromAttributes, + getToolDefinitionDisplay, getToolName, getVariablesMapFromInstances, inferOpenAIApiTypeFromRawToolDefinitions, @@ -38,6 +39,7 @@ import { isOpenAIResponsesSpan, processAttributeToolCalls, promptToolFromGraphQL, + toCanonicalToolDefinition, toolFromEditorJSON, toolToPromptToolInput, transformSpanAttributesToPlaygroundInstance, @@ -2214,6 +2216,62 @@ describe("toolFromEditorJSON", () => { }) ).toBeNull(); }); + + it("should convert Anthropic-shaped editor JSON with strict into a function tool", () => { + const value = { + name: "get_weather", + description: "Get weather", + input_schema: { + type: "object", + properties: { location: { type: "string" } }, + required: ["location"], + }, + strict: true, + }; + + expect(toolFromEditorJSON({ value, id: 1, editorType: "json" })).toEqual({ + kind: "function", + id: 1, + editorType: "json", + definition: { + name: "get_weather", + description: "Get weather", + parameters: { + type: "object", + properties: { location: { type: "string" } }, + required: ["location"], + }, + strict: true, + }, + }); + }); +}); + +describe("tool definition strict round-trip", () => { + const buildCanonicalTool = (strict: boolean): CanonicalToolDefinition => ({ + name: "get_weather", + description: "Get weather", + parameters: { + type: "object", + properties: { city: { type: "string" } }, + required: ["city"], + }, + strict, + }); + + it.each([ + ["ANTHROPIC", true], + ["ANTHROPIC", false], + ["AWS", true], + ["AWS", false], + ] as const)( + "should preserve strict through the %s editor display and back (strict: %s)", + (provider, strict) => { + const canonicalTool = buildCanonicalTool(strict); + const display = getToolDefinitionDisplay(canonicalTool, provider); + expect(toCanonicalToolDefinition(display)).toEqual(canonicalTool); + } + ); }); describe("getPromptTemplateVariablesFromAttributes", () => { diff --git a/js/app/src/pages/playground/playgroundUtils.ts b/js/app/src/pages/playground/playgroundUtils.ts index 290a16ba506..2f1a76e3d17 100644 --- a/js/app/src/pages/playground/playgroundUtils.ts +++ b/js/app/src/pages/playground/playgroundUtils.ts @@ -1730,14 +1730,11 @@ export function toCanonicalToolDefinition( // OpenAI Chat Completions: { type: "function", function: { name, description?, parameters, strict? } } const openai = openAIChatCompletionsToolDefinitionSchema.safeParse(raw); if (openai.success) { - const fn = openai.data.function as Record; return { name: openai.data.function.name, description: openai.data.function.description ?? null, parameters: canonicalParameters(openai.data.function.parameters), - // strict lives at the function level in the actual API but isn't in - // our looseObject schema — extract safely. - strict: typeof fn.strict === "boolean" ? fn.strict : null, + strict: openai.data.function.strict ?? null, }; } // OpenAI Responses API: flat { type: "function", name, parameters, strict, description? } @@ -1757,7 +1754,7 @@ export function toCanonicalToolDefinition( name: anthropic.data.name, description: anthropic.data.description ?? null, parameters: canonicalParameters(anthropic.data.input_schema), - strict: null, + strict: anthropic.data.strict ?? null, }; } // AWS: { toolSpec: { name, description, inputSchema: { json } } } @@ -1769,7 +1766,7 @@ export function toCanonicalToolDefinition( name: spec.name, description: spec.description ?? null, parameters: canonicalParameters(spec.inputSchema.json), - strict: null, + strict: spec.strict ?? null, }; } // Gemini: { name, description?, parameters? | parameters_json_schema? } @@ -1825,6 +1822,7 @@ export function getToolDefinitionDisplay( description: toolDefinition.description, }), input_schema: parametersSchemaWithObjectType(toolDefinition.parameters), + ...(toolDefinition.strict != null && { strict: toolDefinition.strict }), }; } if (provider === "AWS") { @@ -1837,6 +1835,7 @@ export function getToolDefinitionDisplay( inputSchema: { json: parametersSchemaWithObjectType(toolDefinition.parameters), }, + ...(toolDefinition.strict != null && { strict: toolDefinition.strict }), }, }; } diff --git a/js/app/src/schemas/__tests__/toolSchemas.test.ts b/js/app/src/schemas/__tests__/toolSchemas.test.ts index 484f8a59ea6..bc9c390ad1f 100644 --- a/js/app/src/schemas/__tests__/toolSchemas.test.ts +++ b/js/app/src/schemas/__tests__/toolSchemas.test.ts @@ -137,6 +137,24 @@ describe("toolSchemas", () => { }; expect(anthropicToolDefinitionSchema.safeParse(tool).success).toBe(false); }); + + it("should parse an Anthropic tool with strict", () => { + const tool = { + name: "get_weather", + description: "Get weather", + input_schema: { + type: "object", + properties: { city: { type: "string" } }, + required: ["city"], + }, + strict: true, + }; + const result = anthropicToolDefinitionSchema.safeParse(tool); + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.strict).toBe(true); + } + }); }); describe("geminiToolDefinitionSchema", () => { @@ -195,6 +213,28 @@ describe("toolSchemas", () => { } }); + it("should parse an AWS tool with strict", () => { + const tool = { + toolSpec: { + name: "get_weather", + description: "Get weather", + inputSchema: { + json: { + type: "object", + properties: { city: { type: "string" } }, + required: ["city"], + }, + }, + strict: true, + }, + }; + const result = awsToolDefinitionSchema.safeParse(tool); + expect(result.success).toBe(true); + if (result.success && "toolSpec" in result.data) { + expect(result.data.toolSpec.strict).toBe(true); + } + }); + it("should parse an unwrapped AWS tool definition (without toolSpec)", () => { const tool = { name: "get_weather", diff --git a/js/app/src/schemas/toolSchemas.ts b/js/app/src/schemas/toolSchemas.ts index c61db34189e..fab5fa31c8c 100644 --- a/js/app/src/schemas/toolSchemas.ts +++ b/js/app/src/schemas/toolSchemas.ts @@ -201,6 +201,10 @@ export const anthropicToolDefinitionSchema = z.strictObject({ name: z.string(), description: z.string().optional(), input_schema: requiredToolParametersJsonSchema, + strict: z + .boolean() + .optional() + .describe("Whether the tool input must exactly match the input schema"), }); /** @@ -223,6 +227,10 @@ const awsToolSpecSchema = z.strictObject({ inputSchema: z.strictObject({ json: parametersSchemaWithDefaultObjectType, }), + strict: z + .boolean() + .optional() + .describe("Whether the tool input must exactly match the input schema"), }); /** diff --git a/packages/phoenix-client/src/phoenix/client/helpers/sdk/anthropic/messages.py b/packages/phoenix-client/src/phoenix/client/helpers/sdk/anthropic/messages.py index ccd7037197b..db7ea2363f7 100644 --- a/packages/phoenix-client/src/phoenix/client/helpers/sdk/anthropic/messages.py +++ b/packages/phoenix-client/src/phoenix/client/helpers/sdk/anthropic/messages.py @@ -608,6 +608,8 @@ def to_anthropic( } if "description" in function: param["description"] = function["description"] + if "strict" in function and isinstance(function["strict"], bool): + param["strict"] = function["strict"] yield param elif tool["type"] == "raw": # Vendor passthrough: forward the raw dict as a ToolUnionParam @@ -634,6 +636,8 @@ def from_anthropic( if "description" in tool_param: function["description"] = tool_param["description"] function["parameters"] = tool_param["input_schema"] + if "strict" in tool_param and isinstance(tool_param["strict"], bool): + function["strict"] = tool_param["strict"] yield v1.PromptToolFunction(type="function", function=function) else: yield v1.PromptToolRaw(type="raw", raw=dict(cast(Mapping[str, Any], tool))) diff --git a/packages/phoenix-client/tests/canary/sdk/anthropic/test_messages.py b/packages/phoenix-client/tests/canary/sdk/anthropic/test_messages.py index db745686692..b3d0ce85294 100644 --- a/packages/phoenix-client/tests/canary/sdk/anthropic/test_messages.py +++ b/packages/phoenix-client/tests/canary/sdk/anthropic/test_messages.py @@ -65,8 +65,8 @@ def _tool_result() -> ToolResultBlockParam: } -def _tool(name: Optional[str] = None) -> ToolParam: - return { +def _tool(name: Optional[str] = None, strict: Optional[bool] = None) -> ToolParam: + tool: ToolParam = { "name": name or _str(), "description": _str(), "input_schema": { @@ -79,6 +79,9 @@ def _tool(name: Optional[str] = None) -> ToolParam: "additionalProperties": False, }, } + if strict is not None: + tool["strict"] = strict + return tool class TestMessageConversion: @@ -102,7 +105,7 @@ def test_round_trip(self, obj: MessageParam) -> None: class TestToolConversion: @pytest.mark.parametrize( "tools", - [[_tool() for _ in range(3)]], + [[_tool(), _tool(strict=True), _tool(strict=False)]], ) def test_round_trip(self, tools: Iterable[ToolParam]) -> None: new_tools = list(_ToolConversion.to_anthropic(_ToolConversion.from_anthropic(tools))) diff --git a/pyproject.toml b/pyproject.toml index 74c44b3dd20..b75a1848525 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -88,7 +88,7 @@ phoenix = "phoenix.server.main:main" dev = [ "check-wheel-contents", "twine", - "aiobotocore>=3.1.1", + "aiobotocore>=3.2.0", # first release whose botocore floor includes Converse ToolSpecification "strict"; older botocore rejects it client-side "aioboto3", "aiosqlite>=0.22.1", "anthropic>=1,<2", @@ -208,6 +208,7 @@ pg = [ ] aws = [ "aioboto3", + "aiobotocore>=3.2.0", # first release whose botocore floor includes Converse ToolSpecification "strict"; older botocore rejects it client-side "types-aiobotocore-bedrock-runtime>=3.6.0", ] azure = [ @@ -235,6 +236,7 @@ container = [ "strawberry-graphql[opentelemetry]==0.320.4", "uvloop; platform_system != 'Windows'", "aioboto3", + "aiobotocore>=3.2.0", # first release whose botocore floor includes Converse ToolSpecification "strict"; older botocore rejects it client-side "types-aiobotocore-bedrock-runtime>=3.6.0", # Sandbox provider SDKs — bundled so the shipped container can use any # configured provider without a rebuild. Each pin matches the matching diff --git a/src/phoenix/server/api/helpers/playground_clients.py b/src/phoenix/server/api/helpers/playground_clients.py index 739481aff57..c07b971d98a 100644 --- a/src/phoenix/server/api/helpers/playground_clients.py +++ b/src/phoenix/server/api/helpers/playground_clients.py @@ -1616,13 +1616,20 @@ def _converse_build_request( ) if fn.description: tool_spec["description"] = fn.description + if isinstance(fn.strict, bool): + tool_spec["strict"] = fn.strict tool_list.append(ToolTypeDef(toolSpec=tool_spec)) tool_config = ToolConfigurationTypeDef(tools=tool_list) + # The Converse API has no disable-parallel-tool-use setting, so + # tools.disable_parallel_tool_calls cannot be honored here. + send_tools = True if tc := tools.tool_choice: if tc.type == "none": - pass + # Converse has no "none" tool choice; withholding the + # tools entirely is the only way to prevent tool calls. + send_tools = False elif tc.type == "zero_or_more": tool_config["toolChoice"] = ToolChoiceTypeDef(auto={}) elif tc.type == "one_or_more": @@ -1634,7 +1641,8 @@ def _converse_build_request( elif TYPE_CHECKING: assert_never(tc.type) - request["toolConfig"] = tool_config + if send_tools: + request["toolConfig"] = tool_config if response_format: json_schema = JsonSchemaDefinitionTypeDef( @@ -2212,7 +2220,7 @@ def _anthropic_message_params( params["tool_choice"] = choice_tool else: assert_never(tc.type) - if tools.disable_parallel_tool_calls: + elif tools.disable_parallel_tool_calls: params["tool_choice"] = ToolChoiceAutoParam( type="auto", disable_parallel_tool_use=True ) @@ -2230,6 +2238,8 @@ def _anthropic_message_params( ) if f.description: t["description"] = f.description + if isinstance(f.strict, bool): + t["strict"] = f.strict tool_list.append(t) params["tools"] = tool_list extra_headers = _anthropic_beta_headers_for_tools(tool_list) diff --git a/tests/unit/server/api/helpers/test_playground_clients.py b/tests/unit/server/api/helpers/test_playground_clients.py index 2951635e34f..45d8c1dc245 100644 --- a/tests/unit/server/api/helpers/test_playground_clients.py +++ b/tests/unit/server/api/helpers/test_playground_clients.py @@ -20,13 +20,17 @@ from pydantic import SecretStr from phoenix.db import models +from phoenix.db.types.db_helper_types import UNDEFINED from phoenix.db.types.experiment_config import OpenAIConnectionConfig from phoenix.db.types.model_provider import LLMClientFactory, ModelProvider from phoenix.db.types.prompts import ( PromptAnthropicInvocationParameters, PromptAnthropicInvocationParametersContent, + PromptAwsInvocationParameters, + PromptAwsInvocationParametersContent, PromptOpenAIInvocationParameters, PromptOpenAIInvocationParametersContent, + PromptToolChoiceNone, PromptToolChoiceSpecificFunctionTool, PromptToolChoiceZeroOrMore, PromptToolFunction, @@ -42,6 +46,7 @@ AnthropicClient, AzureOpenAIChatCompletionsClient, AzureOpenAIResponsesClient, + BedrockClient, GoogleClient, OpenAIChatCompletionsClient, OpenAICompatibleClient, @@ -64,6 +69,26 @@ from tests.unit.vcr import CustomVCR +def _null_client_factory(provider: str) -> LLMClientFactory[Any]: + @asynccontextmanager + async def create_client() -> AsyncIterator[Any]: + yield None + + return LLMClientFactory(create_client, (provider, "test")) + + +def _function_tool(strict: bool = UNDEFINED) -> PromptToolFunction: + return PromptToolFunction( + type="function", + function=PromptToolFunctionDefinition( + name="correctness", + description="Evaluate correctness", + parameters={"type": "object", "properties": {"label": {"type": "string"}}}, + strict=strict, + ), + ) + + class TestGoogleStreamingClient: @pytest.fixture def client(self) -> GoogleClient: @@ -493,12 +518,8 @@ async def test_authentication_error_records_error_status_on_span( class TestAnthropicStreamingClient: def test_specific_tool_choice_includes_tool_definitions(self) -> None: - @asynccontextmanager - async def create_client() -> AsyncIterator[Any]: - yield None - client: Any = AnthropicClient( - client_factory=LLMClientFactory(create_client, ("anthropic", "test")), + client_factory=_null_client_factory("anthropic"), model_name="claude-3-5-sonnet-latest", provider="anthropic", ) @@ -561,13 +582,90 @@ async def create_client() -> AsyncIterator[Any]: ] assert extra_headers is None - def test_raw_computer_tools_add_anthropic_beta_header(self) -> None: - @asynccontextmanager - async def create_client() -> AsyncIterator[Any]: - yield None + def _anthropic_client(self) -> Any: + return AnthropicClient( + client_factory=_null_client_factory("anthropic"), + model_name="claude-3-5-sonnet-latest", + provider="anthropic", + ) + def _anthropic_params(self, tools: PromptTools) -> dict[str, Any]: + params, _, _ = self._anthropic_client()._anthropic_message_params( + messages=[ + create_playground_message(ChatCompletionMessageRole.USER, "Evaluate this answer.") + ], + tools=tools, + response_format=None, + invocation_parameters=PromptAnthropicInvocationParameters( + type="anthropic", + anthropic=PromptAnthropicInvocationParametersContent(max_tokens=1024), + ), + ) + return dict(params) + + def test_specific_tool_choice_survives_disable_parallel_tool_calls(self) -> None: + """Disabling parallel tool use must not downgrade a specific tool choice to `auto`.""" + params = self._anthropic_params( + PromptTools( + type="tools", + tool_choice=PromptToolChoiceSpecificFunctionTool( + type="specific_function", function_name="correctness" + ), + disable_parallel_tool_calls=True, + tools=[_function_tool()], + ) + ) + assert params["tool_choice"] == { + "type": "tool", + "name": "correctness", + "disable_parallel_tool_use": True, + } + + def test_tool_choice_none_survives_disable_parallel_tool_calls(self) -> None: + """`none` means "do not call tools" and must not be turned into `auto`.""" + params = self._anthropic_params( + PromptTools( + type="tools", + tool_choice=PromptToolChoiceNone(type="none"), + disable_parallel_tool_calls=True, + tools=[_function_tool()], + ) + ) + assert params["tool_choice"] == {"type": "none"} + + def test_disable_parallel_tool_calls_without_explicit_choice_falls_back_to_auto(self) -> None: + """With no explicit choice, the flag still yields `auto` + `disable_parallel_tool_use`.""" + params = self._anthropic_params( + PromptTools( + type="tools", + disable_parallel_tool_calls=True, + tools=[_function_tool()], + ) + ) + assert params["tool_choice"] == {"type": "auto", "disable_parallel_tool_use": True} + + def test_function_tool_strict_is_forwarded(self) -> None: + """The stored prompt tool's `strict` setting must reach the Anthropic request.""" + params = self._anthropic_params( + PromptTools(type="tools", tools=[_function_tool(strict=True)]) + ) + assert params["tools"][0]["strict"] is True + + def test_function_tool_strict_false_is_forwarded(self) -> None: + """An explicit `strict=False` must be sent, not treated as unset.""" + params = self._anthropic_params( + PromptTools(type="tools", tools=[_function_tool(strict=False)]) + ) + assert params["tools"][0]["strict"] is False + + def test_function_tool_omits_strict_when_unset(self) -> None: + """`strict` is optional; an unset value must not be sent.""" + params = self._anthropic_params(PromptTools(type="tools", tools=[_function_tool()])) + assert "strict" not in params["tools"][0] + + def test_raw_computer_tools_add_anthropic_beta_header(self) -> None: client: Any = AnthropicClient( - client_factory=LLMClientFactory(create_client, ("anthropic", "test")), + client_factory=_null_client_factory("anthropic"), model_name="claude-3-5-sonnet-latest", provider="anthropic", ) @@ -658,6 +756,61 @@ async def create_client() -> AsyncIterator[Any]: TOOL_JSON_SCHEMA = ToolAttributes.TOOL_JSON_SCHEMA +class TestBedrockClient: + def _converse_request(self, tools: PromptTools) -> dict[str, Any]: + client: Any = BedrockClient( + client_factory=_null_client_factory("aws"), + model_name="anthropic.claude-3-5-sonnet-20240620-v1:0", + provider="aws", + ) + request = client._converse_build_request( + messages=[ + create_playground_message(ChatCompletionMessageRole.USER, "Evaluate this answer.") + ], + tools=tools, + response_format=None, + invocation_parameters=PromptAwsInvocationParameters( + type="aws", + aws=PromptAwsInvocationParametersContent(max_tokens=1024), + ), + span=INVALID_SPAN, + ) + return dict(request) + + def test_tool_choice_none_withholds_tool_config(self) -> None: + """Converse has no `none` tool choice; the tools must be withheld entirely.""" + request = self._converse_request( + PromptTools( + type="tools", + tool_choice=PromptToolChoiceNone(type="none"), + tools=[_function_tool()], + ) + ) + assert "toolConfig" not in request + + def test_function_tool_strict_is_forwarded(self) -> None: + """The stored prompt tool's `strict` setting must reach the Bedrock toolSpec.""" + request = self._converse_request( + PromptTools(type="tools", tools=[_function_tool(strict=True)]) + ) + tool_spec = request["toolConfig"]["tools"][0]["toolSpec"] + assert tool_spec["strict"] is True + + def test_function_tool_strict_false_is_forwarded(self) -> None: + """An explicit `strict=False` must be sent, not treated as unset.""" + request = self._converse_request( + PromptTools(type="tools", tools=[_function_tool(strict=False)]) + ) + tool_spec = request["toolConfig"]["tools"][0]["toolSpec"] + assert tool_spec["strict"] is False + + def test_function_tool_omits_strict_when_unset(self) -> None: + """`strict` is optional; an unset value must not be sent.""" + request = self._converse_request(PromptTools(type="tools", tools=[_function_tool()])) + tool_spec = request["toolConfig"]["tools"][0]["toolSpec"] + assert "strict" not in tool_spec + + class TestGetOpenAIClientClass: """Tests for the get_openai_client_class helper function.""" diff --git a/uv.lock b/uv.lock index 20b41f5db27..7d474f34006 100644 --- a/uv.lock +++ b/uv.lock @@ -542,6 +542,7 @@ dependencies = [ [package.optional-dependencies] aws = [ { name = "aioboto3" }, + { name = "aiobotocore" }, { name = "types-aiobotocore-bedrock-runtime" }, ] azure = [ @@ -550,6 +551,7 @@ azure = [ ] container = [ { name = "aioboto3" }, + { name = "aiobotocore" }, { name = "aiohttp" }, { name = "anthropic" }, { name = "azure-identity" }, @@ -702,6 +704,8 @@ dev = [ requires-dist = [ { name = "aioboto3", marker = "extra == 'aws'" }, { name = "aioboto3", marker = "extra == 'container'" }, + { name = "aiobotocore", marker = "extra == 'aws'", specifier = ">=3.2.0" }, + { name = "aiobotocore", marker = "extra == 'container'", specifier = ">=3.2.0" }, { name = "aiohttp", marker = "extra == 'azure'" }, { name = "aiohttp", marker = "extra == 'container'" }, { name = "aioitertools" }, @@ -794,7 +798,7 @@ provides-extras = ["aws", "azure", "container", "daytona", "e2b", "evals", "expe [package.metadata.requires-dev] dev = [ { name = "aioboto3" }, - { name = "aiobotocore", specifier = ">=3.1.1" }, + { name = "aiobotocore", specifier = ">=3.2.0" }, { name = "aiosqlite", specifier = ">=0.22.1" }, { name = "anthropic", specifier = ">=1,<2" }, { name = "arize", specifier = ">=8.1.0" },