From d5ebb516f0f6f9f38b41ba2611da67b2af16d467 Mon Sep 17 00:00:00 2001 From: Jeremy Drouillard Date: Tue, 13 Oct 2026 14:40:37 -0700 Subject: [PATCH 1/6] Support Pi in LLM gateway setup --- cmd/thv/app/llm.go | 8 +- docs/cli/thv_llm_setup.md | 9 +- pkg/client/config.go | 11 ++ pkg/client/llm_gateway.go | 12 +- pkg/client/llm_gateway_pi.go | 184 +++++++++++++++++++++++++++ pkg/client/llm_gateway_pi_test.go | 140 +++++++++++++++++++++ pkg/llm/setup.go | 201 +++++++++++++++++++++++------- pkg/llm/setup_test.go | 153 ++++++++++++++++++++--- pkg/llmgateway/config.go | 9 ++ 9 files changed, 659 insertions(+), 68 deletions(-) create mode 100644 pkg/client/llm_gateway_pi.go create mode 100644 pkg/client/llm_gateway_pi_test.go diff --git a/cmd/thv/app/llm.go b/cmd/thv/app/llm.go index 8931049adb..6b15393059 100644 --- a/cmd/thv/app/llm.go +++ b/cmd/thv/app/llm.go @@ -289,7 +289,7 @@ func newLLMSetupCommand() *cobra.Command { cmd := &cobra.Command{ Use: "setup", Short: "Configure detected AI tools to use the LLM gateway", - Long: `Detect installed AI tools (Claude Code, Gemini CLI, Cursor, VS Code, Xcode, + Long: `Detect installed AI tools (Claude Code, Gemini CLI, Pi, Cursor, VS Code, Xcode, Claude Desktop, Codex) and patch each tool's configuration to route through the LLM gateway. @@ -309,6 +309,11 @@ untouched. Desktop app detection is supported on macOS and Windows; the canonica --client target remains "codex". If the desktop app is running, fully quit and reopen it after setup. +Pi is configured with OpenAI-compatible and Anthropic providers in +~/.pi/agent/models.json. It runs "thv llm token" for each request; models are +discovered from both gateway APIs unless --models is provided. With --lazy, +--models is required for Pi. + Proxy-mode tools (Cursor, VS Code, Xcode) are configured to send requests to the localhost reverse proxy started by "thv llm proxy start". @@ -370,6 +375,7 @@ Run "thv llm teardown" to revert all changes.`, cmd.Flags().StringSliceVar(&models, "models", nil, "Model IDs to configure, comma-separated or by repeating the flag, e.g. "+ "--models=us.anthropic.claude-opus-4-8,us.anthropic.claude-sonnet-5. "+ + "For Pi these define its custom provider models (required with --lazy). "+ "For credential-helper clients (Claude Desktop) these become inferenceModels. "+ "With --bedrock-compat, each ID is also mapped to a Claude Code tier by matching "+ "'haiku', 'opus', or 'sonnet' in the ID (IDs matching no tier are ignored with a "+ diff --git a/docs/cli/thv_llm_setup.md b/docs/cli/thv_llm_setup.md index dfdf7a02aa..22f9639f5b 100644 --- a/docs/cli/thv_llm_setup.md +++ b/docs/cli/thv_llm_setup.md @@ -15,7 +15,7 @@ Configure detected AI tools to use the LLM gateway ### Synopsis -Detect installed AI tools (Claude Code, Gemini CLI, Cursor, VS Code, Xcode, +Detect installed AI tools (Claude Code, Gemini CLI, Pi, Cursor, VS Code, Xcode, Claude Desktop, Codex) and patch each tool's configuration to route through the LLM gateway. @@ -35,6 +35,11 @@ untouched. Desktop app detection is supported on macOS and Windows; the canonica --client target remains "codex". If the desktop app is running, fully quit and reopen it after setup. +Pi is configured with OpenAI-compatible and Anthropic providers in +~/.pi/agent/models.json. It runs "thv llm token" for each request; models are +discovered from both gateway APIs unless --models is provided. With --lazy, +--models is required for Pi. + Proxy-mode tools (Cursor, VS Code, Xcode) are configured to send requests to the localhost reverse proxy started by "thv llm proxy start". @@ -75,7 +80,7 @@ thv llm setup [flags] -h, --help help for setup --issuer string OIDC issuer URL --lazy Skip the interactive OIDC login and defer it until the first time a configured tool accesses the gateway. Tool config and persisted settings are written normally. Useful for unattended provisioning (e.g. an MDM profile). - --models strings Model IDs to configure, comma-separated or by repeating the flag, e.g. --models=us.anthropic.claude-opus-4-8,us.anthropic.claude-sonnet-5. For credential-helper clients (Claude Desktop) these become inferenceModels. With --bedrock-compat, each ID is also mapped to a Claude Code tier by matching 'haiku', 'opus', or 'sonnet' in the ID (IDs matching no tier are ignored with a warning). Omit to use the built-in Bedrock defaults / gateway model discovery. + --models strings Model IDs to configure, comma-separated or by repeating the flag, e.g. --models=us.anthropic.claude-opus-4-8,us.anthropic.claude-sonnet-5. For Pi these define its custom provider models (required with --lazy). For credential-helper clients (Claude Desktop) these become inferenceModels. With --bedrock-compat, each ID is also mapped to a Claude Code tier by matching 'haiku', 'opus', or 'sonnet' in the ID (IDs matching no tier are ignored with a warning). Omit to use the built-in Bedrock defaults / gateway model discovery. --proxy-port int Localhost proxy listen port (omit to keep current; default: 14000) --short-prompt-cache Use Claude Code's five-minute prompt cache instead of ToolHive's one-hour default. Persisted, so a later plain "thv llm setup" keeps it; use --short-prompt-cache=false to restore one-hour caching. --skip-browser Print the OIDC authorization URL instead of opening a browser, then wait for the callback. Use in headless/SSH/CI environments where no system browser is available. diff --git a/pkg/client/config.go b/pkg/client/config.go index 81a6b0d4c8..e3773997ec 100644 --- a/pkg/client/config.go +++ b/pkg/client/config.go @@ -98,6 +98,8 @@ const ( ) const ( + // Pi represents the Pi coding agent's LLM gateway integration. + Pi LLMClientApp = "pi" // Xcode represents GitHub Copilot for Xcode. // Xcode does not support MCP; it is an LLM-gateway-only tool. // It is declared as LLMClientApp (not ClientApp) so that code generators @@ -911,6 +913,15 @@ var supportedClientIntegrations = []clientAppConfig{ SkillsGlobalPath: []string{".vibe", skillsDirName}, SkillsProjectPath: []string{".vibe", skillsDirName}, }, + { + ClientType: ClientApp(Pi), + Description: "Pi coding agent", + LLMGatewayOnly: true, + LLMGatewayMode: llmgateway.ModePi, + LLMBinaryName: "pi", + LLMSettingsFile: "models.json", + LLMSettingsRelPath: []string{".pi", "agent"}, + }, { ClientType: Codex, Description: "OpenAI Codex CLI", diff --git a/pkg/client/llm_gateway.go b/pkg/client/llm_gateway.go index 8ec942ed22..daf0837c14 100644 --- a/pkg/client/llm_gateway.go +++ b/pkg/client/llm_gateway.go @@ -53,6 +53,9 @@ func (cm *ClientManager) ConfigureLLMGateway(clientType ClientApp, cfg llmgatewa if appCfg.LLMGatewayMode == llmgateway.ModeCodexAuth { return cm.configureCodexAuth(appCfg, cfg) } + if appCfg.LLMGatewayMode == llmgateway.ModePi { + return cm.configurePi(appCfg, cfg) + } if appCfg.LLMGatewayMode == llmgateway.ModeVSCode { return cm.configureVSCode(appCfg, cfg) } @@ -192,6 +195,9 @@ func (cm *ClientManager) RevertLLMGateway(clientType ClientApp, configPath strin if appCfg.LLMGatewayMode == llmgateway.ModeCodexAuth { return cm.revertCodexAuth(appCfg, configPath) } + if appCfg.LLMGatewayMode == llmgateway.ModePi { + return cm.revertPi(appCfg, configPath) + } if appCfg.LLMGatewayMode == llmgateway.ModeVSCode { return cm.revertVSCode(appCfg, configPath) } @@ -509,8 +515,10 @@ func (cm *ClientManager) sharedCLIDetection(cfg *clientAppConfig) bool { // exist until first configured (e.g. Claude Desktop's configLibrary), // LLMDetectRelPath points at a directory that exists once the app is // installed. Otherwise fall back to the settings directory. - if _, err := os.Stat(cm.llmDetectDir(cfg)); err != nil { - return false + if cfg.ClientType != ClientApp(Pi) { + if _, err := os.Stat(cm.llmDetectDir(cfg)); err != nil { + return false + } } if cfg.LLMBinaryName != "" { if cm.lookPath == nil { diff --git a/pkg/client/llm_gateway_pi.go b/pkg/client/llm_gateway_pi.go new file mode 100644 index 0000000000..196305a231 --- /dev/null +++ b/pkg/client/llm_gateway_pi.go @@ -0,0 +1,184 @@ +// SPDX-FileCopyrightText: Copyright 2026 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package client + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "slices" + "strings" + + "github.com/stacklok/toolhive/pkg/fileutils" + "github.com/stacklok/toolhive/pkg/llmgateway" +) + +const piProviderName = "toolhive" +const piAnthropicProviderName = "toolhive-anthropic" +const piTokenCommand = "!thv llm token" //nolint:gosec // command, not a credential + +// configurePi adds a named provider without disturbing the user's other providers. +func (cm *ClientManager) configurePi(appCfg *clientAppConfig, cfg llmgateway.ApplyConfig) (string, error) { + openAIModels, anthropicModels := selectPiModels(cfg) + if len(openAIModels) == 0 && len(anthropicModels) == 0 { + return "", fmt.Errorf("pi requires a model list; pass --models or run setup without --lazy for gateway model discovery") + } + path := cm.buildLLMSettingsPath(appCfg) + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return "", fmt.Errorf("creating Pi config directory: %w", err) + } + err := fileutils.WithFileLock(path, func() error { + root, providers, err := readPiConfig(path) + if err != nil { + return err + } + for _, name := range []string{piProviderName, piAnthropicProviderName} { + if existing, ok := providers[name]; ok && !piProviderManaged(existing) { + return fmt.Errorf("pi provider %q already exists and is not managed by ToolHive", name) + } + } + baseURL := strings.TrimRight(cfg.GatewayURL, "/") + if len(openAIModels) > 0 { + provider, err := encodePiGatewayProvider(baseURL+"/v1", "openai-completions", openAIModels, + cfg.DiscoveredModelContextWindows) + if err != nil { + return err + } + providers[piProviderName] = provider + } else { + delete(providers, piProviderName) + } + if len(anthropicModels) > 0 { + provider, err := encodePiGatewayProvider(baseURL+"/anthropic", "anthropic-messages", anthropicModels, + cfg.AnthropicModelContextWindows) + if err != nil { + return err + } + providers[piAnthropicProviderName] = provider + } else { + delete(providers, piAnthropicProviderName) + } + return writePiConfig(path, root, providers) + }) + if err != nil { + return "", err + } + return path, nil +} + +func selectPiModels(cfg llmgateway.ApplyConfig) ([]string, []string) { + if len(cfg.Models) == 0 { + return cfg.DiscoveredModels, cfg.AnthropicModels + } + var openAIModels, anthropicModels []string + for _, id := range cfg.Models { + if slices.Contains(cfg.DiscoveredModels, id) || !slices.Contains(cfg.AnthropicModels, id) { + openAIModels = append(openAIModels, id) + } + if slices.Contains(cfg.AnthropicModels, id) { + anthropicModels = append(anthropicModels, id) + } + } + return openAIModels, anthropicModels +} + +func piProviderManaged(existing json.RawMessage) bool { + var provider struct { + APIKey string `json:"apiKey"` + } + return json.Unmarshal(existing, &provider) == nil && provider.APIKey == piTokenCommand +} + +func encodePiGatewayProvider(baseURL, api string, models []string, contextWindows map[string]int) (json.RawMessage, error) { + type piModel struct { + ID string `json:"id"` + ContextWindow int `json:"contextWindow,omitempty"` + } + entries := make([]piModel, 0, len(models)) + for _, id := range models { + entries = append(entries, piModel{ID: id, ContextWindow: contextWindows[id]}) + } + provider, err := json.Marshal(struct { + BaseURL string `json:"baseUrl"` + API string `json:"api"` + APIKey string `json:"apiKey"` + AuthHeader bool `json:"authHeader"` + Models []piModel `json:"models"` + }{baseURL, api, piTokenCommand, true, entries}) + if err != nil { + return nil, fmt.Errorf("encoding Pi provider: %w", err) + } + return provider, nil +} + +func (cm *ClientManager) revertPi(appCfg *clientAppConfig, path string) error { + if path != cm.buildLLMSettingsPath(appCfg) { + return fmt.Errorf("refusing to revert Pi config at unexpected path %q", path) + } + if _, err := os.Stat(path); os.IsNotExist(err) { + return nil + } else if err != nil { + return fmt.Errorf("checking Pi config: %w", err) + } + return fileutils.WithFileLock(path, func() error { + root, providers, err := readPiConfig(path) + if err != nil { + return err + } + owned := false + for _, name := range []string{piProviderName, piAnthropicProviderName} { + existing, ok := providers[name] + if !ok { + continue + } + owned = true + if !piProviderManaged(existing) { + return fmt.Errorf("pi provider %q is no longer managed by ToolHive", name) + } + } + if !owned { + return nil + } + delete(providers, piProviderName) + delete(providers, piAnthropicProviderName) + return writePiConfig(path, root, providers) + }) +} + +func readPiConfig(path string) (map[string]json.RawMessage, map[string]json.RawMessage, error) { + root := make(map[string]json.RawMessage) + content, err := os.ReadFile(path) // #nosec G304 -- path is the registered Pi config location + if err != nil && !os.IsNotExist(err) { + return nil, nil, fmt.Errorf("reading Pi config: %w", err) + } + if len(content) != 0 { + if err := json.Unmarshal(content, &root); err != nil || root == nil { + return nil, nil, fmt.Errorf("decoding Pi config %s: expected a JSON object: %v", path, err) + } + } + providers := make(map[string]json.RawMessage) + if raw, ok := root["providers"]; ok { + if err := json.Unmarshal(raw, &providers); err != nil || providers == nil { + return nil, nil, fmt.Errorf("decoding Pi providers in %s: expected a JSON object: %v", path, err) + } + } + return root, providers, nil +} + +func writePiConfig(path string, root, providers map[string]json.RawMessage) error { + encoded, err := json.Marshal(providers) + if err != nil { + return fmt.Errorf("encoding Pi providers: %w", err) + } + root["providers"] = encoded + content, err := json.MarshalIndent(root, "", " ") + if err != nil { + return fmt.Errorf("encoding Pi config: %w", err) + } + if err := fileutils.AtomicWriteFile(path, append(content, '\n'), 0o600); err != nil { + return fmt.Errorf("writing Pi config: %w", err) + } + return nil +} diff --git a/pkg/client/llm_gateway_pi_test.go b/pkg/client/llm_gateway_pi_test.go new file mode 100644 index 0000000000..5c175513f9 --- /dev/null +++ b/pkg/client/llm_gateway_pi_test.go @@ -0,0 +1,140 @@ +// SPDX-FileCopyrightText: Copyright 2026 Stacklok, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package client + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/stacklok/toolhive/pkg/llmgateway" +) + +func TestPiGatewaySetupAndTeardown(t *testing.T) { + t.Parallel() + home := t.TempDir() + cm := NewTestClientManager(home, nil, supportedClientIntegrations, nil) + path := filepath.Join(home, ".pi", "agent", "models.json") + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o700)) + require.NoError(t, os.WriteFile(path, []byte(`{"theme":"dark","providers":{"personal":{"models":[{"id":"local"}]}}}`), 0o600)) + cfg := llmgateway.ApplyConfig{ + GatewayURL: "https://gateway.example/", + DiscoveredModels: []string{"anthropic/claude-sonnet-5", "gpt-5"}, + DiscoveredModelContextWindows: map[string]int{"anthropic/claude-sonnet-5": 1000000}, + AnthropicModels: []string{"claude-opus-4-8", "claude-haiku-5-5"}, + AnthropicModelContextWindows: map[string]int{"claude-opus-4-8": 1000000}, + } + for range 2 { + got, err := cm.ConfigureLLMGateway(ClientApp(Pi), cfg) + require.NoError(t, err) + require.Equal(t, path, got) + } + content, err := os.ReadFile(path) + require.NoError(t, err) + var root struct { + Theme string `json:"theme"` + Providers map[string]struct { + BaseURL string `json:"baseUrl"` + API string `json:"api"` + APIKey string `json:"apiKey"` + AuthHeader bool `json:"authHeader"` + Models []struct { + ID string `json:"id"` + ContextWindow *int `json:"contextWindow"` + } `json:"models"` + } `json:"providers"` + } + require.NoError(t, json.Unmarshal(content, &root)) + require.Equal(t, "dark", root.Theme) + require.Equal(t, "local", root.Providers["personal"].Models[0].ID) + require.Equal(t, "https://gateway.example/v1", root.Providers[piProviderName].BaseURL) + require.Equal(t, "openai-completions", root.Providers[piProviderName].API) + require.Equal(t, piTokenCommand, root.Providers[piProviderName].APIKey) + require.True(t, root.Providers[piProviderName].AuthHeader) + require.Equal(t, "anthropic/claude-sonnet-5", root.Providers[piProviderName].Models[0].ID) + require.NotNil(t, root.Providers[piProviderName].Models[0].ContextWindow) + require.Equal(t, 1000000, *root.Providers[piProviderName].Models[0].ContextWindow) + require.Equal(t, "gpt-5", root.Providers[piProviderName].Models[1].ID) + require.Nil(t, root.Providers[piProviderName].Models[1].ContextWindow) + require.Equal(t, "https://gateway.example/anthropic", root.Providers[piAnthropicProviderName].BaseURL) + require.Equal(t, "anthropic-messages", root.Providers[piAnthropicProviderName].API) + require.Equal(t, piTokenCommand, root.Providers[piAnthropicProviderName].APIKey) + require.True(t, root.Providers[piAnthropicProviderName].AuthHeader) + require.Equal(t, "claude-opus-4-8", root.Providers[piAnthropicProviderName].Models[0].ID) + require.NotNil(t, root.Providers[piAnthropicProviderName].Models[0].ContextWindow) + require.Equal(t, 1000000, *root.Providers[piAnthropicProviderName].Models[0].ContextWindow) + require.Nil(t, root.Providers[piAnthropicProviderName].Models[1].ContextWindow) + + require.NoError(t, cm.RevertLLMGateway(ClientApp(Pi), path)) + content, err = os.ReadFile(path) + require.NoError(t, err) + root.Providers = nil + require.NoError(t, json.Unmarshal(content, &root)) + require.NotContains(t, root.Providers, piProviderName) + require.NotContains(t, root.Providers, piAnthropicProviderName) + require.Contains(t, root.Providers, "personal") +} + +func TestPiGatewayAnthropicOnly(t *testing.T) { + t.Parallel() + home := t.TempDir() + cm := NewTestClientManager(home, nil, supportedClientIntegrations, nil) + cfg := llmgateway.ApplyConfig{ + GatewayURL: "https://gateway.example", + Models: []string{"claude-opus-4-8"}, + DiscoveredModels: []string{"gpt-5"}, + AnthropicModels: []string{"claude-opus-4-8"}, + AnthropicModelContextWindows: map[string]int{"claude-opus-4-8": 1000000}, + } + path, err := cm.ConfigureLLMGateway(ClientApp(Pi), cfg) + require.NoError(t, err) + _, providers, err := readPiConfig(path) + require.NoError(t, err) + require.NotContains(t, providers, piProviderName) + require.Contains(t, providers, piAnthropicProviderName) + require.NoError(t, cm.RevertLLMGateway(ClientApp(Pi), path)) + _, providers, err = readPiConfig(path) + require.NoError(t, err) + require.NotContains(t, providers, piAnthropicProviderName) +} + +func TestPiGatewayRejectsUnmanagedAnthropicProvider(t *testing.T) { + t.Parallel() + home := t.TempDir() + cm := NewTestClientManager(home, nil, supportedClientIntegrations, nil) + path := filepath.Join(home, ".pi", "agent", "models.json") + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o700)) + original := []byte(`{"providers":{"toolhive-anthropic":{"apiKey":"personal","models":[{"id":"my-model"}]}}}`) + require.NoError(t, os.WriteFile(path, original, 0o600)) + cfg := llmgateway.ApplyConfig{GatewayURL: "https://gateway.example", DiscoveredModels: []string{"gpt-5"}} + _, err := cm.ConfigureLLMGateway(ClientApp(Pi), cfg) + require.ErrorContains(t, err, "not managed") + require.ErrorContains(t, cm.RevertLLMGateway(ClientApp(Pi), path), "no longer managed") + content, err := os.ReadFile(path) + require.NoError(t, err) + require.Equal(t, original, content) +} + +func TestPiGatewayRejectsUnownedProviderAndMissingModels(t *testing.T) { + t.Parallel() + home := t.TempDir() + cm := NewTestClientManager(home, nil, supportedClientIntegrations, nil) + cfg := llmgateway.ApplyConfig{GatewayURL: "https://gateway.example"} + _, err := cm.ConfigureLLMGateway(ClientApp(Pi), cfg) + require.ErrorContains(t, err, "requires a model list") + path := filepath.Join(home, ".pi", "agent", "models.json") + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o700)) + original := []byte(`{"providers":{"toolhive":{"apiKey":"personal","models":[{"id":"my-model"}]}}}`) + require.NoError(t, os.WriteFile(path, original, 0o600)) + cfg.Models = []string{"claude-sonnet-5"} + _, err = cm.ConfigureLLMGateway(ClientApp(Pi), cfg) + require.ErrorContains(t, err, "not managed") + require.ErrorContains(t, cm.RevertLLMGateway(ClientApp(Pi), path), "no longer managed") + content, err := os.ReadFile(path) + require.NoError(t, err) + require.Equal(t, original, content) +} diff --git a/pkg/llm/setup.go b/pkg/llm/setup.go index 5e2b54b289..362b85a7bd 100644 --- a/pkg/llm/setup.go +++ b/pkg/llm/setup.go @@ -13,6 +13,7 @@ import ( "net/http" "net/url" "os" + "slices" "sort" "strings" "time" @@ -100,7 +101,7 @@ func Setup( // there is nothing to configure. In non-lazy mode login still runs before any // files are patched, preserving the guarantee that a failed login leaves no // state. - detected, done, err := setupClients(gm, targetClient, lazy, out, errOut) + detected, done, err := setupClients(gm, targetClient, lazy, llmCfg.Models, out, errOut) if err != nil { return err } @@ -132,7 +133,7 @@ func Setup( _, _ = fmt.Fprintln(out, "Login successful.") } - detected, discoveredModels, err := discoverVSCodeModelsForSetup( + detected, discoveredModels, err := discoverSetupModels( ctx, errOut, detected, targetClient, llmCfg, ) if err != nil { @@ -463,6 +464,9 @@ func warnTLSSkipVerify(errOut io.Writer, skip bool, configured []ToolConfig) { "Warning: %s uses proxy mode — TLS certificate verification is disabled for the "+ "proxy's upstream gateway connection only. Use only in isolated local environments.\n", tc.Tool) } + case llmgateway.ModePi: + _, _ = fmt.Fprintf(errOut, + "Warning: --tls-skip-verify was NOT applied to %s — trust the gateway certificate in the system store.\n", tc.Tool) case llmgateway.ModeCodexAuth: _, _ = fmt.Fprintf(errOut, "Warning: --tls-skip-verify was NOT applied to %s — its config.toml has no "+ @@ -492,7 +496,7 @@ func filterDetectedClients(gm GatewayManager, targetClient string) ([]string, er } func setupClients( - gm GatewayManager, targetClient string, lazy bool, out, errOut io.Writer, + gm GatewayManager, targetClient string, lazy bool, models []string, out, errOut io.Writer, ) ([]string, bool, error) { detected, err := filterDetectedClients(gm, targetClient) if err != nil { @@ -505,6 +509,9 @@ func setupClients( if !lazy { return detected, false, nil } + if slices.Contains(detected, "pi") && len(models) == 0 { + return nil, false, fmt.Errorf("pi requires --models with --lazy; omit --lazy to discover gateway models") + } detected, err = filterLazyVSCodeClients(detected, targetClient, errOut) if err != nil { return nil, false, err @@ -558,90 +565,187 @@ func filterLazyVSCodeClients(clients []string, targetClient string, errOut io.Wr return filtered, nil } -func discoverVSCodeModelsForSetup( +type discoveredGatewayModels struct { + openAI []string + openAIContextWindows map[string]int + anthropic []string + anthropicContextWindows map[string]int +} + +func discoverSetupModels( ctx context.Context, errOut io.Writer, clients []string, targetClient string, cfg Config, -) ([]string, []string, error) { - if !hasVSCodeClient(clients) { - return clients, nil, nil +) ([]string, discoveredGatewayModels, error) { + hasPi := slices.Contains(clients, "pi") + hasVSCode := hasVSCodeClient(clients) + catalog, openErr, anthropicErr := fetchSetupModelCatalog(ctx, cfg, hasPi, hasVSCode) + piMissing := hasPi && len(cfg.Models) == 0 && len(catalog.openAI) == 0 && len(catalog.anthropic) == 0 + if isVSCodeClient(targetClient) && openErr != nil { + return nil, catalog, openErr + } + if targetClient == "pi" && piMissing { + return nil, catalog, fmt.Errorf("discovering Pi models: %w", errors.Join(openErr, anthropicErr)) + } + warnModelDiscovery(errOut, hasVSCode, hasPi, piMissing, catalog, openErr, anthropicErr) + filtered := make([]string, 0, len(clients)) + for _, clientType := range clients { + if (isVSCodeClient(clientType) && openErr != nil) || (clientType == "pi" && piMissing) { + continue + } + filtered = append(filtered, clientType) } - models, err := discoverGatewayModels(ctx, cfg) - if err == nil { - return clients, models, nil + return filtered, catalog, nil +} + +func fetchSetupModelCatalog( + ctx context.Context, cfg Config, hasPi, hasVSCode bool, +) (discoveredGatewayModels, error, error) { + var catalog discoveredGatewayModels + var openErr, anthropicErr error + if hasVSCode || (hasPi && (len(cfg.Models) == 0 || cfg.discoveryAccessToken != "")) { + catalog.openAI, catalog.openAIContextWindows, openErr = discoverGatewayModels(ctx, cfg) } - if isVSCodeClient(targetClient) { - return nil, nil, err + if hasPi && cfg.discoveryAccessToken != "" { + catalog.anthropic, catalog.anthropicContextWindows, anthropicErr = discoverAnthropicGatewayModels(ctx, cfg) } + return catalog, openErr, anthropicErr +} - _, _ = fmt.Fprintf( - errOut, - "Warning: skipping auto-detected VS Code clients because gateway model discovery failed: %v\n", - err, - ) - filtered := make([]string, 0, len(clients)) - for _, clientType := range clients { - if !isVSCodeClient(clientType) { - filtered = append(filtered, clientType) - } +func warnModelDiscovery( + errOut io.Writer, hasVSCode, hasPi, piMissing bool, catalog discoveredGatewayModels, openErr, anthropicErr error, +) { + if hasVSCode && openErr != nil { + _, _ = fmt.Fprintf(errOut, + "Warning: skipping auto-detected VS Code clients because gateway model discovery failed: %v\n", openErr) + } + if piMissing { + _, _ = fmt.Fprintf(errOut, + "Warning: skipping auto-detected Pi because gateway model discovery failed: %v\n", + errors.Join(openErr, anthropicErr)) + } + if hasPi && len(catalog.openAI) > 0 && anthropicErr != nil { + _, _ = fmt.Fprintf(errOut, "Warning: skipping Pi Anthropic models: %v\n", anthropicErr) + } + if hasPi && len(catalog.anthropic) > 0 && openErr != nil { + _, _ = fmt.Fprintf(errOut, "Warning: skipping Pi OpenAI models: %v\n", openErr) } - return filtered, nil, nil } type gatewayModelsResponse struct { - Object string `json:"object"` - Data []struct { - ID string `json:"id"` + Object string `json:"object"` + HasMore *bool `json:"has_more"` + LastID string `json:"last_id"` + Data []struct { + ID string `json:"id"` + ContextWindow int `json:"context_window"` + MaxInputTokens int `json:"max_input_tokens"` } `json:"data"` } -func discoverGatewayModels(ctx context.Context, cfg Config) ([]string, error) { +func requestGatewayModels(ctx context.Context, cfg Config, path, afterID string) (gatewayModelsResponse, error) { if cfg.discoveryAccessToken == "" { - return nil, fmt.Errorf("discovering gateway models: authenticated setup did not provide an access token") + return gatewayModelsResponse{}, fmt.Errorf("discovering gateway models: authenticated setup did not provide an access token") } - modelsURL, err := url.JoinPath(cfg.GatewayURL, "v1/models") + modelsURL, err := url.JoinPath(cfg.GatewayURL, path) if err != nil { - return nil, fmt.Errorf("building gateway models URL: %w", err) + return gatewayModelsResponse{}, fmt.Errorf("building gateway models URL: %w", err) } requestCtx, cancel := context.WithTimeout(ctx, 15*time.Second) defer cancel() req, err := http.NewRequestWithContext(requestCtx, http.MethodGet, modelsURL, nil) if err != nil { - return nil, fmt.Errorf("creating gateway model-discovery request: %w", err) + return gatewayModelsResponse{}, fmt.Errorf("creating gateway model-discovery request: %w", err) + } + if afterID != "" { + query := req.URL.Query() + query.Set("after_id", afterID) + req.URL.RawQuery = query.Encode() } req.Header.Set("Authorization", "Bearer "+cfg.discoveryAccessToken) resp, err := gatewayHTTPClient(cfg.TLSSkipVerify).Do(req) if err != nil { - return nil, fmt.Errorf("discovering gateway models: %w", err) + return gatewayModelsResponse{}, fmt.Errorf("discovering gateway models: %w", err) } defer func() { _ = resp.Body.Close() }() if resp.StatusCode != http.StatusOK { _, _ = io.Copy(io.Discard, resp.Body) - return nil, fmt.Errorf("discovering gateway models: gateway returned %s", resp.Status) + return gatewayModelsResponse{}, fmt.Errorf("discovering gateway models: gateway returned %s", resp.Status) } var payload gatewayModelsResponse if err := json.NewDecoder(resp.Body).Decode(&payload); err != nil { _, _ = io.Copy(io.Discard, resp.Body) - return nil, fmt.Errorf("decoding gateway model response: %w", err) + return gatewayModelsResponse{}, fmt.Errorf("decoding gateway model response: %w", err) + } + return payload, nil +} + +func discoverGatewayModels(ctx context.Context, cfg Config) ([]string, map[string]int, error) { + payload, err := requestGatewayModels(ctx, cfg, "v1/models", "") + if err != nil { + return nil, nil, err } if payload.Object != "list" || len(payload.Data) == 0 { - return nil, fmt.Errorf("decoding gateway model response: expected a non-empty OpenAI model list") + return nil, nil, fmt.Errorf("decoding gateway model response: expected a non-empty OpenAI model list") } seen := make(map[string]struct{}, len(payload.Data)) models := make([]string, 0, len(payload.Data)) + contextWindows := make(map[string]int, len(payload.Data)) for _, model := range payload.Data { modelID := strings.TrimSpace(model.ID) if modelID == "" { - return nil, fmt.Errorf("decoding gateway model response: model ID must not be empty") + return nil, nil, fmt.Errorf("decoding gateway model response: model ID must not be empty") } if _, exists := seen[modelID]; exists { continue } seen[modelID] = struct{}{} models = append(models, modelID) + if model.ContextWindow > 0 { + contextWindows[modelID] = model.ContextWindow + } } sort.Strings(models) - return models, nil + return models, contextWindows, nil +} + +func discoverAnthropicGatewayModels(ctx context.Context, cfg Config) ([]string, map[string]int, error) { + seen := make(map[string]struct{}) + models := make([]string, 0) + contextWindows := make(map[string]int) + afterID := "" + for range 100 { + payload, err := requestGatewayModels(ctx, cfg, "anthropic/v1/models", afterID) + if err != nil { + return nil, nil, err + } + if payload.HasMore == nil || len(payload.Data) == 0 { + return nil, nil, fmt.Errorf("decoding gateway model response: expected a non-empty Anthropic model page") + } + for _, model := range payload.Data { + id := strings.TrimSpace(model.ID) + if id == "" { + return nil, nil, fmt.Errorf("decoding gateway model response: model ID must not be empty") + } + if _, exists := seen[id]; exists { + continue + } + seen[id] = struct{}{} + models = append(models, id) + if model.MaxInputTokens > 0 { + contextWindows[id] = model.MaxInputTokens + } + } + if !*payload.HasMore { + sort.Strings(models) + return models, contextWindows, nil + } + if payload.LastID == "" || payload.LastID == afterID { + return nil, nil, fmt.Errorf("decoding gateway model response: invalid Anthropic pagination cursor") + } + afterID = payload.LastID + } + return nil, nil, fmt.Errorf("decoding gateway model response: too many Anthropic model pages") } // Default Bedrock inference-profile model IDs written for Claude Code in @@ -762,7 +866,7 @@ func configureDetectedToolsWithDiscovery( out, errOut io.Writer, gm GatewayManager, detected []string, gatewayURL, proxyBaseURL, tokenHelperPath string, tokenHelperArgs []string, tlsSkipVerify bool, anthropicPathPrefix string, - models, discoveredModels []string, shortPromptCache bool, bedrock BedrockConfig, + models []string, discovered discoveredGatewayModels, shortPromptCache bool, bedrock BedrockConfig, ) ([]ToolConfig, error) { var configured []ToolConfig for _, clientType := range detected { @@ -782,15 +886,18 @@ func configureDetectedToolsWithDiscovery( } applyCfg := llmgateway.ApplyConfig{ - GatewayURL: gatewayURL, - AnthropicBaseURL: anthropicBaseURL, - ProxyBaseURL: proxyBaseURL, - TokenHelperCommand: tokenHelperShellCommand, - TokenHelperPath: tokenHelperPath, - TokenHelperArgs: tokenHelperArgs, - TLSSkipVerify: tlsSkipVerify, - Models: models, - DiscoveredModels: discoveredModels, + GatewayURL: gatewayURL, + AnthropicBaseURL: anthropicBaseURL, + ProxyBaseURL: proxyBaseURL, + TokenHelperCommand: tokenHelperShellCommand, + TokenHelperPath: tokenHelperPath, + TokenHelperArgs: tokenHelperArgs, + TLSSkipVerify: tlsSkipVerify, + Models: models, + DiscoveredModels: discovered.openAI, + DiscoveredModelContextWindows: discovered.openAIContextWindows, + AnthropicModels: discovered.anthropic, + AnthropicModelContextWindows: discovered.anthropicContextWindows, } if clientType == claudeCodeClient { diff --git a/pkg/llm/setup_test.go b/pkg/llm/setup_test.go index 59fbedd0d5..2c3a219707 100644 --- a/pkg/llm/setup_test.go +++ b/pkg/llm/setup_test.go @@ -190,7 +190,7 @@ func TestConfigureDetectedTools_BedrockClaudeCode(t *testing.T) { []string{"claude-code"}, "https://gw.example.com", "http://localhost:14000/v1", "/usr/local/bin/thv", []string{"llm", "token", "--skip-browser"}, - false, "/anthropic", nil, nil, false, + false, "/anthropic", nil, discoveredGatewayModels{}, false, BedrockConfig{Compat: true, Enable1M: true}, ) require.NoError(t, err) @@ -214,7 +214,7 @@ func TestConfigureDetectedTools_BedrockSkippedForNonClaudeCode(t *testing.T) { []string{"cursor"}, "https://gw.example.com", "http://localhost:14000/v1", "/usr/local/bin/thv", []string{"llm", "token", "--skip-browser"}, - false, "", nil, nil, false, + false, "", nil, discoveredGatewayModels{}, false, BedrockConfig{Compat: true}, ) require.NoError(t, err) @@ -288,7 +288,7 @@ func TestConfigureDetectedTools_PromptCache(t *testing.T) { &out, &errOut, gm, []string{tt.clientType}, "https://gw.example.com", "http://localhost:14000/v1", "/usr/local/bin/thv", []string{"llm", "token", "--skip-browser"}, - false, "", nil, nil, tt.shortPromptCache, BedrockConfig{}, + false, "", nil, discoveredGatewayModels{}, tt.shortPromptCache, BedrockConfig{}, ) require.NoError(t, err) require.Len(t, configured, 1) @@ -923,21 +923,141 @@ func TestSetup_Lazy_VSCodeSelectionSemantics(t *testing.T) { }) } +func TestSetupPiModels(t *testing.T) { + t.Parallel() + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "Bearer discovery-token", r.Header.Get("Authorization")) + switch r.URL.Path { + case "/v1/models": + _, _ = w.Write([]byte(`{"object":"list","data":[{"id":"gpt-5","context_window":400000}]}`)) + case "/anthropic/v1/models": + _, _ = w.Write([]byte(`{"data":[{"id":"claude-sonnet-5","max_input_tokens":1000000}],"has_more":false}`)) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + t.Cleanup(server.Close) + for _, tt := range []struct { + name string + lazy bool + models []string + wantErr string + }{ + {name: "discover models", models: nil}, + {name: "lazy requires models", lazy: true, wantErr: "pi requires --models"}, + {name: "lazy explicit models", lazy: true, models: []string{"claude-opus-4-8"}}, + } { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + gm := &setupGatewayManager{detected: []string{"pi"}, mode: llmgateway.ModePi} + provider := configuredSetupProvider() + provider.cfg.GatewayURL = server.URL + provider.cfg.TLSSkipVerify = true + var out, errOut bytes.Buffer + err := Setup(context.Background(), &out, &errOut, gm, provider, + func(_ context.Context, cfg *Config) error { + cfg.SetDiscoveryAccessToken("discovery-token") + return nil + }, SetOptions{Models: tt.models}, "/anthropic", true, "pi", tt.lazy) + if tt.wantErr != "" { + require.ErrorContains(t, err, tt.wantErr) + assert.Zero(t, provider.updateCalls) + return + } + require.NoError(t, err) + require.Len(t, gm.applied, 1) + assert.Equal(t, tt.models, gm.applied[0].Models) + if !tt.lazy { + assert.Equal(t, []string{"gpt-5"}, gm.applied[0].DiscoveredModels) + assert.Equal(t, map[string]int{"gpt-5": 400000}, gm.applied[0].DiscoveredModelContextWindows) + assert.Equal(t, []string{"claude-sonnet-5"}, gm.applied[0].AnthropicModels) + assert.Equal(t, map[string]int{"claude-sonnet-5": 1000000}, gm.applied[0].AnthropicModelContextWindows) + } else { + assert.Empty(t, gm.applied[0].DiscoveredModelContextWindows) + } + assert.Empty(t, gm.applied[0].AnthropicBaseURL) + }) + } +} + +func TestDiscoverSetupModelsPiAnthropicFallback(t *testing.T) { + t.Parallel() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/v1/models" { + w.WriteHeader(http.StatusNotFound) + return + } + assert.Equal(t, "/anthropic/v1/models", r.URL.Path) + _, _ = w.Write([]byte(`{"data":[{"id":"claude-opus-4-8","max_input_tokens":1000000}],"has_more":false}`)) + })) + t.Cleanup(server.Close) + cfg := Config{GatewayURL: server.URL} + cfg.SetDiscoveryAccessToken("discovery-token") + var warnings strings.Builder + clients, catalog, err := discoverSetupModels(context.Background(), &warnings, []string{"pi"}, "pi", cfg) + require.NoError(t, err) + assert.Equal(t, []string{"pi"}, clients) + assert.Empty(t, catalog.openAI) + assert.Equal(t, []string{"claude-opus-4-8"}, catalog.anthropic) + assert.Equal(t, map[string]int{"claude-opus-4-8": 1000000}, catalog.anthropicContextWindows) + assert.Contains(t, warnings.String(), "skipping Pi OpenAI models") +} + +func TestDiscoverAnthropicGatewayModels(t *testing.T) { + t.Parallel() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/anthropic/v1/models", r.URL.Path) + assert.Equal(t, "Bearer discovery-token", r.Header.Get("Authorization")) + switch r.URL.Query().Get("after_id") { + case "": + _, _ = w.Write([]byte(`{"data":[{"id":" claude-opus-4-8 ","max_input_tokens":1000000},` + + `{"id":"claude-haiku-5-5"}],"has_more":true,"last_id":"claude-haiku-5-5"}`)) + case "claude-haiku-5-5": + _, _ = w.Write([]byte(`{"data":[{"id":"claude-sonnet-5","max_input_tokens":200000},` + + `{"id":"claude-opus-4-8","max_input_tokens":10}],"has_more":false}`)) + default: + w.WriteHeader(http.StatusBadRequest) + } + })) + t.Cleanup(server.Close) + cfg := Config{GatewayURL: server.URL} + cfg.SetDiscoveryAccessToken("discovery-token") + models, contexts, err := discoverAnthropicGatewayModels(context.Background(), cfg) + require.NoError(t, err) + assert.Equal(t, []string{"claude-haiku-5-5", "claude-opus-4-8", "claude-sonnet-5"}, models) + assert.Equal(t, map[string]int{"claude-opus-4-8": 1000000, "claude-sonnet-5": 200000}, contexts) +} + +func TestDiscoverAnthropicGatewayModelsRejectsInvalidPagination(t *testing.T) { + t.Parallel() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{"data":[{"id":"model"}],"has_more":true}`)) + })) + t.Cleanup(server.Close) + cfg := Config{GatewayURL: server.URL} + cfg.SetDiscoveryAccessToken("discovery-token") + _, _, err := discoverAnthropicGatewayModels(context.Background(), cfg) + require.ErrorContains(t, err, "invalid Anthropic pagination cursor") +} + func TestDiscoverGatewayModels(t *testing.T) { t.Parallel() tests := []struct { - name string - status int - body string - setToken bool - want []string - wantErr string + name string + status int + body string + setToken bool + want []string + wantContext map[string]int + wantErr string }{ {name: "valid OpenAI list is trimmed, deduplicated, and sorted", status: http.StatusOK, - body: `{"object":"list","data":[{"id":" z-model "},{"id":"a-model"},{"id":"z-model"},` + - `{"id":" a-model "}]}`, - setToken: true, want: []string{"a-model", "z-model"}}, + body: `{"object":"list","data":[{"id":" z-model ","context_window":1000000},` + + `{"id":"a-model"},{"id":"z-model","context_window":-5},` + + `{"id":" a-model ","context_window":0}]}`, + setToken: true, want: []string{"a-model", "z-model"}, + wantContext: map[string]int{"z-model": 1000000}}, {name: "missing authentication", status: http.StatusOK, body: `{"object":"list","data":[{"id":"model"}]}`, wantErr: "did not provide an access token"}, {name: "authentication failure", status: http.StatusUnauthorized, body: `{"error":"unauthorized"}`, @@ -966,13 +1086,14 @@ func TestDiscoverGatewayModels(t *testing.T) { if tt.setToken { cfg.SetDiscoveryAccessToken("discovery-token") } - got, err := discoverGatewayModels(context.Background(), cfg) + got, contexts, err := discoverGatewayModels(context.Background(), cfg) if tt.wantErr != "" { require.ErrorContains(t, err, tt.wantErr) return } require.NoError(t, err) assert.Equal(t, tt.want, got) + assert.Equal(t, tt.wantContext, contexts) }) } } @@ -1325,7 +1446,7 @@ func TestConfigureDetectedTools_PathPrefixAppendedForDirectMode(t *testing.T) { []string{"claude-code"}, "https://gw.example.com", "http://localhost:14000/v1", "/usr/local/bin/thv", []string{"llm", "token", "--skip-browser"}, - false, "/anthropic", nil, nil, false, + false, "/anthropic", nil, discoveredGatewayModels{}, false, BedrockConfig{}, ) require.NoError(t, err) @@ -1347,7 +1468,7 @@ func TestConfigureDetectedTools_NoPrefixWhenEmpty(t *testing.T) { []string{"claude-code"}, "https://gw.example.com", "http://localhost:14000/v1", "/usr/local/bin/thv", []string{"llm", "token", "--skip-browser"}, - false, "", nil, nil, false, // no prefix + false, "", nil, discoveredGatewayModels{}, false, // no prefix BedrockConfig{}, ) require.NoError(t, err) @@ -1368,7 +1489,7 @@ func TestConfigureDetectedTools_PrefixNotAppliedForProxyMode(t *testing.T) { []string{"cursor"}, "https://gw.example.com", "http://localhost:14000/v1", "/usr/local/bin/thv", []string{"llm", "token", "--skip-browser"}, - false, "/anthropic", nil, nil, false, + false, "/anthropic", nil, discoveredGatewayModels{}, false, BedrockConfig{}, ) require.NoError(t, err) diff --git a/pkg/llmgateway/config.go b/pkg/llmgateway/config.go index 6d6e8c1197..2d916d3abc 100644 --- a/pkg/llmgateway/config.go +++ b/pkg/llmgateway/config.go @@ -65,6 +65,8 @@ const ( // shell), rather than the JSON-Pointer LLMGatewayKeys mechanism the other // modes share. ModeCodexAuth = "codex-auth" //nolint:gosec // G101: mode identifier, not a credential + // ModePi configures Pi's OpenAI-compatible and Anthropic providers in models.json. + ModePi = "pi" // ModeVSCode configures VS Code's chatLanguageModels.json provider groups. ModeVSCode = "vscode" ) @@ -114,6 +116,13 @@ type ApplyConfig struct { // discovery request. It is used only by integrations that require an explicit // model catalogue, such as VS Code's customendpoint provider. DiscoveredModels []string + // DiscoveredModelContextWindows contains positive context sizes from gateway + // discovery for Pi's model catalogue. Missing entries use Pi's default. + DiscoveredModelContextWindows map[string]int + // AnthropicModels and AnthropicModelContextWindows contain the separately + // discovered Anthropic model catalogue for Pi. Sizes are max_input_tokens. + AnthropicModels []string + AnthropicModelContextWindows map[string]int // ShortPromptCache opts supported clients out of ToolHive's one-hour // prompt-cache lifetime and restores their shorter default. ShortPromptCache bool From 70f96b030bdefec74ab901aa8d2ed6e2ae773cb9 Mon Sep 17 00:00:00 2001 From: Jeremy Drouillard Date: Tue, 13 Oct 2026 15:14:43 -0700 Subject: [PATCH 2/6] Address Pi setup review feedback --- cmd/thv/app/llm.go | 7 ++- docs/cli/thv_llm_setup.md | 7 ++- pkg/client/llm_gateway.go | 15 +++++ pkg/client/llm_gateway_pi.go | 101 +++++++++++++++++++++++------- pkg/client/llm_gateway_pi_test.go | 74 ++++++++++++++++++++++ pkg/llm/setup.go | 23 ++++--- pkg/llm/setup_test.go | 24 ++++--- 7 files changed, 199 insertions(+), 52 deletions(-) diff --git a/cmd/thv/app/llm.go b/cmd/thv/app/llm.go index 6b15393059..d3f71659af 100644 --- a/cmd/thv/app/llm.go +++ b/cmd/thv/app/llm.go @@ -310,9 +310,10 @@ untouched. Desktop app detection is supported on macOS and Windows; the canonica reopen it after setup. Pi is configured with OpenAI-compatible and Anthropic providers in -~/.pi/agent/models.json. It runs "thv llm token" for each request; models are -discovered from both gateway APIs unless --models is provided. With --lazy, ---models is required for Pi. +~/.pi/agent/models.json (or $PI_CODING_AGENT_DIR/models.json). It runs +"thv llm token" for each request; authenticated setup discovers models and +context sizes from both gateway APIs. --models filters the discovered models. +Pi does not support --lazy because model discovery determines each model's API. Proxy-mode tools (Cursor, VS Code, Xcode) are configured to send requests to the localhost reverse proxy started by "thv llm proxy start". diff --git a/docs/cli/thv_llm_setup.md b/docs/cli/thv_llm_setup.md index 22f9639f5b..67dbb15131 100644 --- a/docs/cli/thv_llm_setup.md +++ b/docs/cli/thv_llm_setup.md @@ -36,9 +36,10 @@ untouched. Desktop app detection is supported on macOS and Windows; the canonica reopen it after setup. Pi is configured with OpenAI-compatible and Anthropic providers in -~/.pi/agent/models.json. It runs "thv llm token" for each request; models are -discovered from both gateway APIs unless --models is provided. With --lazy, ---models is required for Pi. +~/.pi/agent/models.json (or $PI_CODING_AGENT_DIR/models.json). It runs +"thv llm token" for each request; authenticated setup discovers models and +context sizes from both gateway APIs. --models filters the discovered models. +Pi does not support --lazy because model discovery determines each model's API. Proxy-mode tools (Cursor, VS Code, Xcode) are configured to send requests to the localhost reverse proxy started by "thv llm proxy start". diff --git a/pkg/client/llm_gateway.go b/pkg/client/llm_gateway.go index daf0837c14..8b0d117b1e 100644 --- a/pkg/client/llm_gateway.go +++ b/pkg/client/llm_gateway.go @@ -551,6 +551,21 @@ func (cm *ClientManager) LLMClientDetectionHint(clientType ClientApp) string { // buildLLMSettingsPath resolves the absolute path to the LLM settings file // for the given client using the same PlatformPrefix logic as MCP config paths. func (cm *ClientManager) buildLLMSettingsPath(cfg *clientAppConfig) string { + if cfg.ClientType == ClientApp(Pi) { + if dir := os.Getenv("PI_CODING_AGENT_DIR"); dir != "" { + if dir == "~" { + dir = cm.homeDir + } else if strings.HasPrefix(dir, "~/") { + dir = filepath.Join(cm.homeDir, dir[2:]) + } + if !filepath.IsAbs(dir) { + if absolute, err := filepath.Abs(dir); err == nil { + dir = absolute + } + } + return filepath.Join(dir, cfg.LLMSettingsFile) + } + } return buildConfigFilePath( cfg.LLMSettingsFile, cfg.LLMSettingsRelPath, diff --git a/pkg/client/llm_gateway_pi.go b/pkg/client/llm_gateway_pi.go index 196305a231..ed96823b9c 100644 --- a/pkg/client/llm_gateway_pi.go +++ b/pkg/client/llm_gateway_pi.go @@ -4,6 +4,7 @@ package client import ( + "bytes" "encoding/json" "fmt" "os" @@ -11,6 +12,8 @@ import ( "slices" "strings" + "github.com/tailscale/hujson" + "github.com/stacklok/toolhive/pkg/fileutils" "github.com/stacklok/toolhive/pkg/llmgateway" ) @@ -21,15 +24,18 @@ const piTokenCommand = "!thv llm token" //nolint:gosec // command, not a credent // configurePi adds a named provider without disturbing the user's other providers. func (cm *ClientManager) configurePi(appCfg *clientAppConfig, cfg llmgateway.ApplyConfig) (string, error) { - openAIModels, anthropicModels := selectPiModels(cfg) + openAIModels, anthropicModels, err := selectPiModels(cfg) + if err != nil { + return "", err + } if len(openAIModels) == 0 && len(anthropicModels) == 0 { - return "", fmt.Errorf("pi requires a model list; pass --models or run setup without --lazy for gateway model discovery") + return "", fmt.Errorf("pi requires a model list from authenticated gateway discovery") } path := cm.buildLLMSettingsPath(appCfg) if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { return "", fmt.Errorf("creating Pi config directory: %w", err) } - err := fileutils.WithFileLock(path, func() error { + err = fileutils.WithFileLock(path, func() error { root, providers, err := readPiConfig(path) if err != nil { return err @@ -68,20 +74,25 @@ func (cm *ClientManager) configurePi(appCfg *clientAppConfig, cfg llmgateway.App return path, nil } -func selectPiModels(cfg llmgateway.ApplyConfig) ([]string, []string) { +func selectPiModels(cfg llmgateway.ApplyConfig) ([]string, []string, error) { if len(cfg.Models) == 0 { - return cfg.DiscoveredModels, cfg.AnthropicModels + return cfg.DiscoveredModels, cfg.AnthropicModels, nil } var openAIModels, anthropicModels []string for _, id := range cfg.Models { - if slices.Contains(cfg.DiscoveredModels, id) || !slices.Contains(cfg.AnthropicModels, id) { + openAI := slices.Contains(cfg.DiscoveredModels, id) + anthropic := slices.Contains(cfg.AnthropicModels, id) + if !openAI && !anthropic { + return nil, nil, fmt.Errorf("pi model %q was not found by authenticated model discovery; cannot determine its API", id) + } + if openAI { openAIModels = append(openAIModels, id) } - if slices.Contains(cfg.AnthropicModels, id) { + if anthropic { anthropicModels = append(anthropicModels, id) } } - return openAIModels, anthropicModels + return openAIModels, anthropicModels, nil } func piProviderManaged(existing json.RawMessage) bool { @@ -147,37 +158,79 @@ func (cm *ClientManager) revertPi(appCfg *clientAppConfig, path string) error { }) } -func readPiConfig(path string) (map[string]json.RawMessage, map[string]json.RawMessage, error) { - root := make(map[string]json.RawMessage) +type piConfig struct { + value hujson.Value + hasBOM bool +} + +var piBOM = []byte{0xef, 0xbb, 0xbf} + +func readPiConfig(path string) (piConfig, map[string]json.RawMessage, error) { content, err := os.ReadFile(path) // #nosec G304 -- path is the registered Pi config location if err != nil && !os.IsNotExist(err) { - return nil, nil, fmt.Errorf("reading Pi config: %w", err) + return piConfig{}, nil, fmt.Errorf("reading Pi config: %w", err) } - if len(content) != 0 { - if err := json.Unmarshal(content, &root); err != nil || root == nil { - return nil, nil, fmt.Errorf("decoding Pi config %s: expected a JSON object: %v", path, err) - } + hasBOM := bytes.HasPrefix(content, piBOM) + content = bytes.TrimPrefix(content, piBOM) + if len(content) == 0 { + content = []byte("{}") + } + value, err := parseJSONC(content, path) + if err != nil { + return piConfig{}, nil, err + } + standardized, err := standardizeJSONC(&value, path) + if err != nil { + return piConfig{}, nil, err + } + var root map[string]json.RawMessage + if err := json.Unmarshal(standardized, &root); err != nil || root == nil { + return piConfig{}, nil, fmt.Errorf("decoding Pi config %s: expected a JSON object: %v", path, err) } providers := make(map[string]json.RawMessage) if raw, ok := root["providers"]; ok { if err := json.Unmarshal(raw, &providers); err != nil || providers == nil { - return nil, nil, fmt.Errorf("decoding Pi providers in %s: expected a JSON object: %v", path, err) + return piConfig{}, nil, fmt.Errorf("decoding Pi providers in %s: expected a JSON object: %v", path, err) } } - return root, providers, nil + return piConfig{value: value, hasBOM: hasBOM}, providers, nil } -func writePiConfig(path string, root, providers map[string]json.RawMessage) error { - encoded, err := json.Marshal(providers) +func writePiConfig(path string, root piConfig, providers map[string]json.RawMessage) error { + standardized, err := standardizeJSONC(&root.value, path) if err != nil { - return fmt.Errorf("encoding Pi providers: %w", err) + return err + } + if !jsonPointerExists(standardized, "/providers") && len(providers) != 0 { + if err := root.value.Patch([]byte(`[{"op":"add","path":"/providers","value":{}}]`)); err != nil { + return fmt.Errorf("adding Pi providers in %s: %w", path, err) + } + } + for _, name := range []string{piProviderName, piAnthropicProviderName} { + ptr := "/providers/" + name + value, ok := providers[name] + if !ok { + if err := removeLLMKey(&root.value, ptr, path, standardized); err != nil { + return err + } + continue + } + patch, err := json.Marshal([]llmPatchOp{{Op: "add", Path: ptr, Value: value}}) + if err != nil { + return fmt.Errorf("encoding Pi provider %q: %w", name, err) + } + if err := root.value.Patch(patch); err != nil { + return fmt.Errorf("patching Pi provider %q in %s: %w", name, path, err) + } } - root["providers"] = encoded - content, err := json.MarshalIndent(root, "", " ") + content, err := hujson.Format(root.value.Pack()) if err != nil { - return fmt.Errorf("encoding Pi config: %w", err) + return fmt.Errorf("formatting Pi config %s: %w", path, err) + } + if root.hasBOM { + content = append(append([]byte{}, piBOM...), content...) } - if err := fileutils.AtomicWriteFile(path, append(content, '\n'), 0o600); err != nil { + if err := fileutils.AtomicWriteFile(path, content, 0o600); err != nil { return fmt.Errorf("writing Pi config: %w", err) } return nil diff --git a/pkg/client/llm_gateway_pi_test.go b/pkg/client/llm_gateway_pi_test.go index 5c175513f9..74390a136a 100644 --- a/pkg/client/llm_gateway_pi_test.go +++ b/pkg/client/llm_gateway_pi_test.go @@ -4,6 +4,7 @@ package client import ( + "bytes" "encoding/json" "os" "path/filepath" @@ -79,6 +80,70 @@ func TestPiGatewaySetupAndTeardown(t *testing.T) { require.Contains(t, root.Providers, "personal") } +func TestPiGatewayPreservesJSONCAndBOM(t *testing.T) { + t.Parallel() + home := t.TempDir() + cm := NewTestClientManager(home, nil, supportedClientIntegrations, nil) + path := filepath.Join(home, ".pi", "agent", "models.json") + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o700)) + original := append(append([]byte{}, piBOM...), []byte(`{ + // Keep this comment on the root object. + "theme": "dark", + "providers": { + // Keep this personal provider comment. + "personal": { + "models": [ + { "id": "local" }, // Keep this model comment. + ], + }, + }, +}`)...) + require.NoError(t, os.WriteFile(path, original, 0o600)) + cfg := llmgateway.ApplyConfig{GatewayURL: "https://gateway.example", DiscoveredModels: []string{"gpt-5"}} + _, err := cm.ConfigureLLMGateway(ClientApp(Pi), cfg) + require.NoError(t, err) + assertPreserved := func() { + content, err := os.ReadFile(path) + require.NoError(t, err) + require.True(t, bytes.HasPrefix(content, piBOM)) + for _, comment := range []string{"Keep this comment on the root object.", + "Keep this personal provider comment.", "Keep this model comment."} { + require.Contains(t, string(content), comment) + } + _, providers, err := readPiConfig(path) + require.NoError(t, err) + require.Contains(t, providers, "personal") + } + assertPreserved() + require.NoError(t, cm.RevertLLMGateway(ClientApp(Pi), path)) + assertPreserved() +} + +func TestPiGatewayAgentDirOverride(t *testing.T) { + home, agentDir := t.TempDir(), t.TempDir() + t.Setenv("PI_CODING_AGENT_DIR", agentDir) + cm := NewTestClientManager(home, nil, supportedClientIntegrations, nil) + cm.lookPath = func(string) (string, error) { return "/usr/local/bin/pi", nil } + require.Contains(t, cm.DetectedLLMGatewayClients(), ClientApp(Pi)) + path, err := cm.ConfigureLLMGateway(ClientApp(Pi), llmgateway.ApplyConfig{ + GatewayURL: "https://gateway.example", DiscoveredModels: []string{"gpt-5"}, + }) + require.NoError(t, err) + require.Equal(t, filepath.Join(agentDir, "models.json"), path) + require.NoFileExists(t, filepath.Join(home, ".pi", "agent", "models.json")) + require.NoError(t, cm.RevertLLMGateway(ClientApp(Pi), path)) + _, providers, err := readPiConfig(path) + require.NoError(t, err) + require.NotContains(t, providers, piProviderName) + t.Setenv("PI_CODING_AGENT_DIR", "~/custom-pi-agent") + require.Equal(t, filepath.Join(home, "custom-pi-agent", "models.json"), + cm.buildLLMSettingsPath(cm.lookupClientAppConfig(ClientApp(Pi)))) + t.Setenv("PI_CODING_AGENT_DIR", "relative-pi-agent") + relativePath, err := filepath.Abs(filepath.Join("relative-pi-agent", "models.json")) + require.NoError(t, err) + require.Equal(t, relativePath, cm.buildLLMSettingsPath(cm.lookupClientAppConfig(ClientApp(Pi)))) +} + func TestPiGatewayAnthropicOnly(t *testing.T) { t.Parallel() home := t.TempDir() @@ -119,6 +184,14 @@ func TestPiGatewayRejectsUnmanagedAnthropicProvider(t *testing.T) { require.Equal(t, original, content) } +func TestPiGatewayRejectsUnclassifiedModel(t *testing.T) { + t.Parallel() + cm := NewTestClientManager(t.TempDir(), nil, supportedClientIntegrations, nil) + cfg := llmgateway.ApplyConfig{GatewayURL: "https://gateway.example", Models: []string{"claude-opus-4-8"}} + _, err := cm.ConfigureLLMGateway(ClientApp(Pi), cfg) + require.ErrorContains(t, err, "cannot determine its API") +} + func TestPiGatewayRejectsUnownedProviderAndMissingModels(t *testing.T) { t.Parallel() home := t.TempDir() @@ -131,6 +204,7 @@ func TestPiGatewayRejectsUnownedProviderAndMissingModels(t *testing.T) { original := []byte(`{"providers":{"toolhive":{"apiKey":"personal","models":[{"id":"my-model"}]}}}`) require.NoError(t, os.WriteFile(path, original, 0o600)) cfg.Models = []string{"claude-sonnet-5"} + cfg.DiscoveredModels = []string{"claude-sonnet-5"} _, err = cm.ConfigureLLMGateway(ClientApp(Pi), cfg) require.ErrorContains(t, err, "not managed") require.ErrorContains(t, cm.RevertLLMGateway(ClientApp(Pi), path), "no longer managed") diff --git a/pkg/llm/setup.go b/pkg/llm/setup.go index 362b85a7bd..0ff744cee1 100644 --- a/pkg/llm/setup.go +++ b/pkg/llm/setup.go @@ -101,7 +101,7 @@ func Setup( // there is nothing to configure. In non-lazy mode login still runs before any // files are patched, preserving the guarantee that a failed login leaves no // state. - detected, done, err := setupClients(gm, targetClient, lazy, llmCfg.Models, out, errOut) + detected, done, err := setupClients(gm, targetClient, lazy, out, errOut) if err != nil { return err } @@ -496,7 +496,7 @@ func filterDetectedClients(gm GatewayManager, targetClient string) ([]string, er } func setupClients( - gm GatewayManager, targetClient string, lazy bool, models []string, out, errOut io.Writer, + gm GatewayManager, targetClient string, lazy bool, out, errOut io.Writer, ) ([]string, bool, error) { detected, err := filterDetectedClients(gm, targetClient) if err != nil { @@ -509,10 +509,7 @@ func setupClients( if !lazy { return detected, false, nil } - if slices.Contains(detected, "pi") && len(models) == 0 { - return nil, false, fmt.Errorf("pi requires --models with --lazy; omit --lazy to discover gateway models") - } - detected, err = filterLazyVSCodeClients(detected, targetClient, errOut) + detected, err = filterLazyModelDiscoveryClients(detected, targetClient, errOut) if err != nil { return nil, false, err } @@ -544,7 +541,11 @@ func hasVSCodeClient(clients []string) bool { return false } -func filterLazyVSCodeClients(clients []string, targetClient string, errOut io.Writer) ([]string, error) { +func filterLazyModelDiscoveryClients(clients []string, targetClient string, errOut io.Writer) ([]string, error) { + if targetClient == "pi" { + return nil, fmt.Errorf("client %q does not support --lazy: Pi needs authenticated model discovery "+ + "to select each model's API; rerun without --lazy", targetClient) + } if targetClient != "" && isVSCodeClient(targetClient) { return nil, fmt.Errorf( "client %q does not support --lazy: VS Code requires non-lazy setup for authenticated model discovery; "+ @@ -552,6 +553,10 @@ func filterLazyVSCodeClients(clients []string, targetClient string, errOut io.Wr } filtered := make([]string, 0, len(clients)) for _, clientType := range clients { + if clientType == "pi" { + _, _ = fmt.Fprintln(errOut, "Warning: skipping pi in lazy mode; Pi needs authenticated model discovery.") + continue + } if isVSCodeClient(clientType) { _, _ = fmt.Fprintf( errOut, @@ -578,7 +583,7 @@ func discoverSetupModels( hasPi := slices.Contains(clients, "pi") hasVSCode := hasVSCodeClient(clients) catalog, openErr, anthropicErr := fetchSetupModelCatalog(ctx, cfg, hasPi, hasVSCode) - piMissing := hasPi && len(cfg.Models) == 0 && len(catalog.openAI) == 0 && len(catalog.anthropic) == 0 + piMissing := hasPi && len(catalog.openAI) == 0 && len(catalog.anthropic) == 0 if isVSCodeClient(targetClient) && openErr != nil { return nil, catalog, openErr } @@ -601,7 +606,7 @@ func fetchSetupModelCatalog( ) (discoveredGatewayModels, error, error) { var catalog discoveredGatewayModels var openErr, anthropicErr error - if hasVSCode || (hasPi && (len(cfg.Models) == 0 || cfg.discoveryAccessToken != "")) { + if hasVSCode || hasPi { catalog.openAI, catalog.openAIContextWindows, openErr = discoverGatewayModels(ctx, cfg) } if hasPi && cfg.discoveryAccessToken != "" { diff --git a/pkg/llm/setup_test.go b/pkg/llm/setup_test.go index 2c3a219707..39f60a407a 100644 --- a/pkg/llm/setup_test.go +++ b/pkg/llm/setup_test.go @@ -904,9 +904,9 @@ func TestSetup_Lazy_VSCodeSelectionSemantics(t *testing.T) { assert.Empty(t, gm.configured) }) - t.Run("multi-client setup warns, skips VS Code, and continues", func(t *testing.T) { + t.Run("multi-client setup warns, skips model-discovery clients, and continues", func(t *testing.T) { t.Parallel() - gm := &setupGatewayManager{detected: []string{"vscode", "cursor", "vscode-insider"}, mode: llmgateway.ModeProxy} + gm := &setupGatewayManager{detected: []string{"vscode", "pi", "cursor", "vscode-insider"}, mode: llmgateway.ModeProxy} provider := configuredSetupProvider() loginCalled := false var stdout, stderr bytes.Buffer @@ -918,6 +918,7 @@ func TestSetup_Lazy_VSCodeSelectionSemantics(t *testing.T) { assert.Equal(t, []string{"cursor"}, gm.configured) assert.Contains(t, stderr.String(), "skipping vscode in lazy mode") assert.Contains(t, stderr.String(), "skipping vscode-insider in lazy mode") + assert.Contains(t, stderr.String(), "skipping pi in lazy mode") require.Len(t, provider.cfg.ConfiguredTools, 1) assert.Equal(t, "cursor", provider.cfg.ConfiguredTools[0].Tool) }) @@ -943,9 +944,10 @@ func TestSetupPiModels(t *testing.T) { models []string wantErr string }{ - {name: "discover models", models: nil}, - {name: "lazy requires models", lazy: true, wantErr: "pi requires --models"}, - {name: "lazy explicit models", lazy: true, models: []string{"claude-opus-4-8"}}, + {name: "discover models"}, + {name: "explicit Anthropic model", models: []string{"claude-sonnet-5"}}, + {name: "lazy no models", lazy: true, wantErr: "does not support --lazy"}, + {name: "lazy explicit models", lazy: true, models: []string{"claude-sonnet-5"}, wantErr: "does not support --lazy"}, } { t.Run(tt.name, func(t *testing.T) { t.Parallel() @@ -967,14 +969,10 @@ func TestSetupPiModels(t *testing.T) { require.NoError(t, err) require.Len(t, gm.applied, 1) assert.Equal(t, tt.models, gm.applied[0].Models) - if !tt.lazy { - assert.Equal(t, []string{"gpt-5"}, gm.applied[0].DiscoveredModels) - assert.Equal(t, map[string]int{"gpt-5": 400000}, gm.applied[0].DiscoveredModelContextWindows) - assert.Equal(t, []string{"claude-sonnet-5"}, gm.applied[0].AnthropicModels) - assert.Equal(t, map[string]int{"claude-sonnet-5": 1000000}, gm.applied[0].AnthropicModelContextWindows) - } else { - assert.Empty(t, gm.applied[0].DiscoveredModelContextWindows) - } + assert.Equal(t, []string{"gpt-5"}, gm.applied[0].DiscoveredModels) + assert.Equal(t, map[string]int{"gpt-5": 400000}, gm.applied[0].DiscoveredModelContextWindows) + assert.Equal(t, []string{"claude-sonnet-5"}, gm.applied[0].AnthropicModels) + assert.Equal(t, map[string]int{"claude-sonnet-5": 1000000}, gm.applied[0].AnthropicModelContextWindows) assert.Empty(t, gm.applied[0].AnthropicBaseURL) }) } From df2918e0e2f8fe28ccf2f400f205635ab95bacdf Mon Sep 17 00:00:00 2001 From: Jeremy Drouillard Date: Wed, 14 Oct 2026 14:59:49 -0700 Subject: [PATCH 3/6] Address remaining Pi setup review feedback Keep Anthropic discovery and requests on the same gateway path, and explain Pi's environment-only cache setting so users can opt into long retention. Tighten installation detection and document model metadata limits. --- cmd/thv/app/llm.go | 12 ++++-- docs/cli/thv_llm_setup.md | 10 ++++- pkg/client/llm_gateway.go | 6 +-- pkg/client/llm_gateway_pi.go | 11 +++++- pkg/client/llm_gateway_pi_test.go | 16 +++++++- pkg/llm/setup.go | 65 ++++++++++++++++++------------- pkg/llm/setup_test.go | 62 ++++++++++++++++++++++++++--- pkg/llmgateway/config.go | 3 ++ 8 files changed, 139 insertions(+), 46 deletions(-) diff --git a/cmd/thv/app/llm.go b/cmd/thv/app/llm.go index d3f71659af..bde016e56d 100644 --- a/cmd/thv/app/llm.go +++ b/cmd/thv/app/llm.go @@ -314,6 +314,12 @@ Pi is configured with OpenAI-compatible and Anthropic providers in "thv llm token" for each request; authenticated setup discovers models and context sizes from both gateway APIs. --models filters the discovered models. Pi does not support --lazy because model discovery determines each model's API. +Pi defaults to short prompt caching: start Pi with PI_CACHE_RETENTION=long +in its environment for long retention (Anthropic: 1h; OpenAI: 24h). +This cannot be persisted in models.json. Pi defaults models without discovered +capability metadata to text-only, no reasoning, and 16K max output tokens. +If Pi cannot refresh its gateway token, run "thv llm token" in a terminal to +log in again. Proxy-mode tools (Cursor, VS Code, Xcode) are configured to send requests to the localhost reverse proxy started by "thv llm proxy start". @@ -364,8 +370,8 @@ Run "thv llm teardown" to revert all changes.`, "Use Claude Code's five-minute prompt cache instead of ToolHive's one-hour default. Persisted, so a later "+ "plain \"thv llm setup\" keeps it; use --short-prompt-cache=false to restore one-hour caching.") cmd.Flags().StringVar(&anthropicPathPrefix, "anthropic-path-prefix", "", - "Path prefix appended to the gateway URL when writing ANTHROPIC_BASE_URL for direct-mode tools "+ - "(e.g. /anthropic). When omitted, the gateway is probed automatically.") + "Path prefix appended to the gateway URL for Anthropic API tools (e.g. /anthropic). "+ + "Also used for Pi model discovery. When omitted, the gateway is probed automatically.") cmd.Flags().StringVar(&targetClient, "client", "", "Configure only this AI tool by name (e.g. claude-code, cursor, codex). Omit to configure all detected tools.") cmd.Flags().BoolVar(&lazy, "lazy", false, @@ -376,7 +382,7 @@ Run "thv llm teardown" to revert all changes.`, cmd.Flags().StringSliceVar(&models, "models", nil, "Model IDs to configure, comma-separated or by repeating the flag, e.g. "+ "--models=us.anthropic.claude-opus-4-8,us.anthropic.claude-sonnet-5. "+ - "For Pi these define its custom provider models (required with --lazy). "+ + "For Pi these filter its discovered models (Pi does not support --lazy). "+ "For credential-helper clients (Claude Desktop) these become inferenceModels. "+ "With --bedrock-compat, each ID is also mapped to a Claude Code tier by matching "+ "'haiku', 'opus', or 'sonnet' in the ID (IDs matching no tier are ignored with a "+ diff --git a/docs/cli/thv_llm_setup.md b/docs/cli/thv_llm_setup.md index 67dbb15131..88c33340df 100644 --- a/docs/cli/thv_llm_setup.md +++ b/docs/cli/thv_llm_setup.md @@ -40,6 +40,12 @@ Pi is configured with OpenAI-compatible and Anthropic providers in "thv llm token" for each request; authenticated setup discovers models and context sizes from both gateway APIs. --models filters the discovered models. Pi does not support --lazy because model discovery determines each model's API. +Pi defaults to short prompt caching: start Pi with PI_CACHE_RETENTION=long +in its environment for long retention (Anthropic: 1h; OpenAI: 24h). +This cannot be persisted in models.json. Pi defaults models without discovered +capability metadata to text-only, no reasoning, and 16K max output tokens. +If Pi cannot refresh its gateway token, run "thv llm token" in a terminal to +log in again. Proxy-mode tools (Cursor, VS Code, Xcode) are configured to send requests to the localhost reverse proxy started by "thv llm proxy start". @@ -70,7 +76,7 @@ thv llm setup [flags] ### Options ``` - --anthropic-path-prefix string Path prefix appended to the gateway URL when writing ANTHROPIC_BASE_URL for direct-mode tools (e.g. /anthropic). When omitted, the gateway is probed automatically. + --anthropic-path-prefix string Path prefix appended to the gateway URL for Anthropic API tools (e.g. /anthropic). Also used for Pi model discovery. When omitted, the gateway is probed automatically. --audience string OIDC audience (optional) --bedrock-compat Configure Claude Code for a gateway that forwards to AWS Bedrock: write CLAUDE_CODE_DISABLE_EXPERIMENTAL_BETAS=1 (Bedrock rejects the experimental anthropic-beta headers) and pin per-tier Bedrock model IDs (override with --models). Persisted, so a later plain "thv llm setup" keeps it; clear with --bedrock-compat=false. Only affects Claude Code. --callback-port int OIDC callback port (omit to keep current; default: ephemeral) @@ -81,7 +87,7 @@ thv llm setup [flags] -h, --help help for setup --issuer string OIDC issuer URL --lazy Skip the interactive OIDC login and defer it until the first time a configured tool accesses the gateway. Tool config and persisted settings are written normally. Useful for unattended provisioning (e.g. an MDM profile). - --models strings Model IDs to configure, comma-separated or by repeating the flag, e.g. --models=us.anthropic.claude-opus-4-8,us.anthropic.claude-sonnet-5. For Pi these define its custom provider models (required with --lazy). For credential-helper clients (Claude Desktop) these become inferenceModels. With --bedrock-compat, each ID is also mapped to a Claude Code tier by matching 'haiku', 'opus', or 'sonnet' in the ID (IDs matching no tier are ignored with a warning). Omit to use the built-in Bedrock defaults / gateway model discovery. + --models strings Model IDs to configure, comma-separated or by repeating the flag, e.g. --models=us.anthropic.claude-opus-4-8,us.anthropic.claude-sonnet-5. For Pi these filter its discovered models (Pi does not support --lazy). For credential-helper clients (Claude Desktop) these become inferenceModels. With --bedrock-compat, each ID is also mapped to a Claude Code tier by matching 'haiku', 'opus', or 'sonnet' in the ID (IDs matching no tier are ignored with a warning). Omit to use the built-in Bedrock defaults / gateway model discovery. --proxy-port int Localhost proxy listen port (omit to keep current; default: 14000) --short-prompt-cache Use Claude Code's five-minute prompt cache instead of ToolHive's one-hour default. Persisted, so a later plain "thv llm setup" keeps it; use --short-prompt-cache=false to restore one-hour caching. --skip-browser Print the OIDC authorization URL instead of opening a browser, then wait for the callback. Use in headless/SSH/CI environments where no system browser is available. diff --git a/pkg/client/llm_gateway.go b/pkg/client/llm_gateway.go index 8b0d117b1e..2985437483 100644 --- a/pkg/client/llm_gateway.go +++ b/pkg/client/llm_gateway.go @@ -515,10 +515,8 @@ func (cm *ClientManager) sharedCLIDetection(cfg *clientAppConfig) bool { // exist until first configured (e.g. Claude Desktop's configLibrary), // LLMDetectRelPath points at a directory that exists once the app is // installed. Otherwise fall back to the settings directory. - if cfg.ClientType != ClientApp(Pi) { - if _, err := os.Stat(cm.llmDetectDir(cfg)); err != nil { - return false - } + if _, err := os.Stat(cm.llmDetectDir(cfg)); err != nil { + return false } if cfg.LLMBinaryName != "" { if cm.lookPath == nil { diff --git a/pkg/client/llm_gateway_pi.go b/pkg/client/llm_gateway_pi.go index ed96823b9c..f562d4969c 100644 --- a/pkg/client/llm_gateway_pi.go +++ b/pkg/client/llm_gateway_pi.go @@ -20,7 +20,10 @@ import ( const piProviderName = "toolhive" const piAnthropicProviderName = "toolhive-anthropic" -const piTokenCommand = "!thv llm token" //nolint:gosec // command, not a credential + +// Pi runs !-prefixed API keys on every request with its own fixed timeout; +// it has no configurable helper TTL or timeout to write into models.json. +const piTokenCommand = "!" + llmgateway.TokenHelperShellCommand //nolint:gosec // command, not a credential // configurePi adds a named provider without disturbing the user's other providers. func (cm *ClientManager) configurePi(appCfg *clientAppConfig, cfg llmgateway.ApplyConfig) (string, error) { @@ -57,7 +60,11 @@ func (cm *ClientManager) configurePi(appCfg *clientAppConfig, cfg llmgateway.App delete(providers, piProviderName) } if len(anthropicModels) > 0 { - provider, err := encodePiGatewayProvider(baseURL+"/anthropic", "anthropic-messages", anthropicModels, + anthropicBaseURL := baseURL + if cfg.AnthropicBaseURL != "" { + anthropicBaseURL = strings.TrimRight(cfg.AnthropicBaseURL, "/") + } + provider, err := encodePiGatewayProvider(anthropicBaseURL, "anthropic-messages", anthropicModels, cfg.AnthropicModelContextWindows) if err != nil { return err diff --git a/pkg/client/llm_gateway_pi_test.go b/pkg/client/llm_gateway_pi_test.go index 74390a136a..46756fa9b4 100644 --- a/pkg/client/llm_gateway_pi_test.go +++ b/pkg/client/llm_gateway_pi_test.go @@ -24,6 +24,7 @@ func TestPiGatewaySetupAndTeardown(t *testing.T) { require.NoError(t, os.WriteFile(path, []byte(`{"theme":"dark","providers":{"personal":{"models":[{"id":"local"}]}}}`), 0o600)) cfg := llmgateway.ApplyConfig{ GatewayURL: "https://gateway.example/", + AnthropicBaseURL: "https://gateway.example/custom", DiscoveredModels: []string{"anthropic/claude-sonnet-5", "gpt-5"}, DiscoveredModelContextWindows: map[string]int{"anthropic/claude-sonnet-5": 1000000}, AnthropicModels: []string{"claude-opus-4-8", "claude-haiku-5-5"}, @@ -61,7 +62,7 @@ func TestPiGatewaySetupAndTeardown(t *testing.T) { require.Equal(t, 1000000, *root.Providers[piProviderName].Models[0].ContextWindow) require.Equal(t, "gpt-5", root.Providers[piProviderName].Models[1].ID) require.Nil(t, root.Providers[piProviderName].Models[1].ContextWindow) - require.Equal(t, "https://gateway.example/anthropic", root.Providers[piAnthropicProviderName].BaseURL) + require.Equal(t, "https://gateway.example/custom", root.Providers[piAnthropicProviderName].BaseURL) require.Equal(t, "anthropic-messages", root.Providers[piAnthropicProviderName].API) require.Equal(t, piTokenCommand, root.Providers[piAnthropicProviderName].APIKey) require.True(t, root.Providers[piAnthropicProviderName].AuthHeader) @@ -120,10 +121,12 @@ func TestPiGatewayPreservesJSONCAndBOM(t *testing.T) { } func TestPiGatewayAgentDirOverride(t *testing.T) { - home, agentDir := t.TempDir(), t.TempDir() + home, agentDir := t.TempDir(), filepath.Join(t.TempDir(), "agent") t.Setenv("PI_CODING_AGENT_DIR", agentDir) cm := NewTestClientManager(home, nil, supportedClientIntegrations, nil) cm.lookPath = func(string) (string, error) { return "/usr/local/bin/pi", nil } + require.NotContains(t, cm.DetectedLLMGatewayClients(), ClientApp(Pi)) + require.NoError(t, os.MkdirAll(agentDir, 0o700)) require.Contains(t, cm.DetectedLLMGatewayClients(), ClientApp(Pi)) path, err := cm.ConfigureLLMGateway(ClientApp(Pi), llmgateway.ApplyConfig{ GatewayURL: "https://gateway.example", DiscoveredModels: []string{"gpt-5"}, @@ -135,6 +138,10 @@ func TestPiGatewayAgentDirOverride(t *testing.T) { _, providers, err := readPiConfig(path) require.NoError(t, err) require.NotContains(t, providers, piProviderName) + t.Setenv("PI_CODING_AGENT_DIR", "") + require.NotContains(t, cm.DetectedLLMGatewayClients(), ClientApp(Pi)) + require.NoError(t, os.MkdirAll(filepath.Join(home, ".pi", "agent"), 0o700)) + require.Contains(t, cm.DetectedLLMGatewayClients(), ClientApp(Pi)) t.Setenv("PI_CODING_AGENT_DIR", "~/custom-pi-agent") require.Equal(t, filepath.Join(home, "custom-pi-agent", "models.json"), cm.buildLLMSettingsPath(cm.lookupClientAppConfig(ClientApp(Pi)))) @@ -161,6 +168,11 @@ func TestPiGatewayAnthropicOnly(t *testing.T) { require.NoError(t, err) require.NotContains(t, providers, piProviderName) require.Contains(t, providers, piAnthropicProviderName) + var anthropic struct { + BaseURL string `json:"baseUrl"` + } + require.NoError(t, json.Unmarshal(providers[piAnthropicProviderName], &anthropic)) + require.Equal(t, cfg.GatewayURL, anthropic.BaseURL) require.NoError(t, cm.RevertLLMGateway(ClientApp(Pi), path)) _, providers, err = readPiConfig(path) require.NoError(t, err) diff --git a/pkg/llm/setup.go b/pkg/llm/setup.go index 0ff744cee1..8e1b927e85 100644 --- a/pkg/llm/setup.go +++ b/pkg/llm/setup.go @@ -133,8 +133,12 @@ func Setup( _, _ = fmt.Fprintln(out, "Login successful.") } + // Resolve the effective path prefix for the Anthropic API before discovery + // so Pi uses the same endpoint to list models and serve requests. + anthropicPrefix := resolveAnthropicPrefix(ctx, gm, detected, llmCfg, anthropicPathPrefix, anthropicPathPrefixSet) + detected, discoveredModels, err := discoverSetupModels( - ctx, errOut, detected, targetClient, llmCfg, + ctx, errOut, detected, targetClient, llmCfg, anthropicPrefix, ) if err != nil { return err @@ -143,15 +147,6 @@ func Setup( return nil } - // Resolve the effective path prefix for ANTHROPIC_BASE_URL. - // If the caller supplied --anthropic-path-prefix, use it directly. - // Otherwise auto-probe: a HEAD request to /anthropic/v1/messages - // that returns 401 (rather than 404) indicates the gateway uses the - // /anthropic prefix, so we apply it automatically. - // Only probe if at least one detected tool uses direct mode; proxy-mode - // tools ignore the Anthropic prefix entirely. - anthropicPrefix := resolveAnthropicPrefix(ctx, gm, detected, llmCfg, anthropicPathPrefix, anthropicPathPrefixSet) - configured, err := configureDetectedToolsWithDiscovery( out, errOut, gm, detected, llmCfg.GatewayURL, proxyBaseURL, tokenHelperPath, tokenHelperArgs, llmCfg.TLSSkipVerify, anthropicPrefix, @@ -191,6 +186,7 @@ func Setup( return fmt.Errorf("persisting tool configuration: %w", err) } + printPiLongCacheHint(out, configured, llmCfg.ShortPromptCache) if hasProxyMode(configured) { _, _ = fmt.Fprintln(out, "One or more tools use proxy mode. Start the proxy with: thv llm proxy start") } @@ -578,11 +574,11 @@ type discoveredGatewayModels struct { } func discoverSetupModels( - ctx context.Context, errOut io.Writer, clients []string, targetClient string, cfg Config, + ctx context.Context, errOut io.Writer, clients []string, targetClient string, cfg Config, anthropicPrefix string, ) ([]string, discoveredGatewayModels, error) { hasPi := slices.Contains(clients, "pi") hasVSCode := hasVSCodeClient(clients) - catalog, openErr, anthropicErr := fetchSetupModelCatalog(ctx, cfg, hasPi, hasVSCode) + catalog, openErr, anthropicErr := fetchSetupModelCatalog(ctx, cfg, hasPi, hasVSCode, anthropicPrefix) piMissing := hasPi && len(catalog.openAI) == 0 && len(catalog.anthropic) == 0 if isVSCodeClient(targetClient) && openErr != nil { return nil, catalog, openErr @@ -602,7 +598,7 @@ func discoverSetupModels( } func fetchSetupModelCatalog( - ctx context.Context, cfg Config, hasPi, hasVSCode bool, + ctx context.Context, cfg Config, hasPi, hasVSCode bool, anthropicPrefix string, ) (discoveredGatewayModels, error, error) { var catalog discoveredGatewayModels var openErr, anthropicErr error @@ -610,7 +606,8 @@ func fetchSetupModelCatalog( catalog.openAI, catalog.openAIContextWindows, openErr = discoverGatewayModels(ctx, cfg) } if hasPi && cfg.discoveryAccessToken != "" { - catalog.anthropic, catalog.anthropicContextWindows, anthropicErr = discoverAnthropicGatewayModels(ctx, cfg) + catalog.anthropic, catalog.anthropicContextWindows, anthropicErr = + discoverAnthropicGatewayModels(ctx, cfg, anthropicPrefix) } return catalog, openErr, anthropicErr } @@ -714,13 +711,18 @@ func discoverGatewayModels(ctx context.Context, cfg Config) ([]string, map[strin return models, contextWindows, nil } -func discoverAnthropicGatewayModels(ctx context.Context, cfg Config) ([]string, map[string]int, error) { +func discoverAnthropicGatewayModels(ctx context.Context, cfg Config, anthropicPrefix string) ([]string, map[string]int, error) { + path := strings.Trim(anthropicPrefix, "/") + if path != "" { + path += "/" + } + path += "v1/models" seen := make(map[string]struct{}) models := make([]string, 0) contextWindows := make(map[string]int) afterID := "" for range 100 { - payload, err := requestGatewayModels(ctx, cfg, "anthropic/v1/models", afterID) + payload, err := requestGatewayModels(ctx, cfg, path, afterID) if err != nil { return nil, nil, err } @@ -844,6 +846,13 @@ func warnBedrockNoEffect(errOut io.Writer, opts SetOptions, effectiveCompat bool } } +func printPiLongCacheHint(out io.Writer, configured []ToolConfig, shortPromptCache bool) { + if isTarget(configured, "pi") && !shortPromptCache { + _, _ = fmt.Fprintln(out, "Pi uses short prompt caching by default. To enable long retention, "+ + "start Pi with PI_CACHE_RETENTION=long in its environment (Anthropic: 1h; OpenAI: 24h).") + } +} + func warnShortPromptCacheNoEffect(errOut io.Writer, opts SetOptions, configured []ToolConfig) { if opts.ShortPromptCache != nil && *opts.ShortPromptCache && !isTarget(configured, claudeCodeClient) { _, _ = fmt.Fprintln(errOut, @@ -974,12 +983,12 @@ func warnPromptCacheConflict(errOut io.Writer, gm GatewayManager, clientType str // resolveAnthropicPrefix returns the effective Anthropic path prefix. When the // caller explicitly set the flag (anthropicPathPrefixSet), the provided value is // returned as-is (including empty string, which disables the prefix). Otherwise -// the gateway is auto-probed when at least one direct-mode client is present. +// the gateway is auto-probed when at least one client uses its Anthropic API. func resolveAnthropicPrefix( ctx context.Context, gm GatewayManager, detected []string, llmCfg Config, anthropicPathPrefix string, anthropicPathPrefixSet bool, ) string { - if anthropicPathPrefixSet || !hasDirectModeClient(gm, detected) { + if anthropicPathPrefixSet || !hasAnthropicBaseURLClient(gm, detected) { return anthropicPathPrefix } return probeAnthropicPrefix(ctx, llmCfg.GatewayURL, llmCfg.TLSSkipVerify) @@ -987,8 +996,8 @@ func resolveAnthropicPrefix( // probeAnthropicPrefix performs a HEAD request to /anthropic/v1/messages. // If the server responds with HTTP 401 (Unauthorized) — meaning the path exists but -// requires authentication — it returns "/anthropic" as the path prefix so that -// ANTHROPIC_BASE_URL is constructed as /anthropic rather than . +// requires authentication — it returns "/anthropic" as the path prefix for +// clients using the Anthropic API. // Any other status code (including 404 Not Found) or any network error is treated // as "no prefix needed" and the function returns "". func probeAnthropicPrefix(ctx context.Context, gatewayURL string, tlsSkipVerify bool) string { @@ -1053,7 +1062,7 @@ func gatewayHTTPClient(tlsSkipVerify bool) *http.Client { // Claude Desktop cannot accept them — it is only ever GUI-launched — so its // credential-helper shim uses the absolute TokenHelperPath instead (see // writeCredentialHelperShim in pkg/client). -const tokenHelperShellCommand = "thv llm token" //nolint:gosec // G101: a command line, not a credential +const tokenHelperShellCommand = llmgateway.TokenHelperShellCommand // buildTokenHelperArgv returns the argv-form of the token helper, for config // formats that invoke an executable directly (no shell) — e.g. Codex's @@ -1070,16 +1079,16 @@ func buildTokenHelperArgv() (string, []string, error) { } // usesAnthropicBaseURL reports whether a client mode consumes the Anthropic base -// URL (gateway + /anthropic prefix): direct mode via ANTHROPIC_BASE_URL, and -// credential-helper mode (Claude Desktop) via inferenceGatewayBaseUrl. +// URL (gateway plus its probed or configured prefix): direct mode via +// ANTHROPIC_BASE_URL, credential-helper mode via inferenceGatewayBaseUrl, +// and Pi via its Anthropic provider. func usesAnthropicBaseURL(mode string) bool { - return mode == llmgateway.ModeDirect || mode == llmgateway.ModeCredentialHelper + return mode == llmgateway.ModeDirect || mode == llmgateway.ModeCredentialHelper || mode == llmgateway.ModePi } -// hasDirectModeClient reports whether any client in the detected list uses a -// mode that needs the Anthropic base URL. Used to skip the Anthropic-prefix -// probe when no such tools are present (proxy-mode tools ignore it entirely). -func hasDirectModeClient(gm GatewayManager, detected []string) bool { +// hasAnthropicBaseURLClient reports whether any detected client needs the +// Anthropic base URL. Proxy-mode tools ignore it entirely. +func hasAnthropicBaseURLClient(gm GatewayManager, detected []string) bool { for _, clientType := range detected { if usesAnthropicBaseURL(gm.LLMGatewayModeFor(clientType)) { return true diff --git a/pkg/llm/setup_test.go b/pkg/llm/setup_test.go index 39f60a407a..2676b21d49 100644 --- a/pkg/llm/setup_test.go +++ b/pkg/llm/setup_test.go @@ -941,11 +941,13 @@ func TestSetupPiModels(t *testing.T) { for _, tt := range []struct { name string lazy bool + short bool models []string wantErr string }{ {name: "discover models"}, {name: "explicit Anthropic model", models: []string{"claude-sonnet-5"}}, + {name: "short cache does not suggest long", short: true}, {name: "lazy no models", lazy: true, wantErr: "does not support --lazy"}, {name: "lazy explicit models", lazy: true, models: []string{"claude-sonnet-5"}, wantErr: "does not support --lazy"}, } { @@ -956,24 +958,74 @@ func TestSetupPiModels(t *testing.T) { provider.cfg.GatewayURL = server.URL provider.cfg.TLSSkipVerify = true var out, errOut bytes.Buffer + opts := SetOptions{Models: tt.models} + if tt.short { + opts.ShortPromptCache = &tt.short + } err := Setup(context.Background(), &out, &errOut, gm, provider, func(_ context.Context, cfg *Config) error { cfg.SetDiscoveryAccessToken("discovery-token") return nil - }, SetOptions{Models: tt.models}, "/anthropic", true, "pi", tt.lazy) + }, opts, "/anthropic", true, "pi", tt.lazy) if tt.wantErr != "" { require.ErrorContains(t, err, tt.wantErr) assert.Zero(t, provider.updateCalls) return } require.NoError(t, err) + assert.Equal(t, !tt.short, strings.Contains(out.String(), "PI_CACHE_RETENTION=long")) require.Len(t, gm.applied, 1) assert.Equal(t, tt.models, gm.applied[0].Models) assert.Equal(t, []string{"gpt-5"}, gm.applied[0].DiscoveredModels) assert.Equal(t, map[string]int{"gpt-5": 400000}, gm.applied[0].DiscoveredModelContextWindows) assert.Equal(t, []string{"claude-sonnet-5"}, gm.applied[0].AnthropicModels) assert.Equal(t, map[string]int{"claude-sonnet-5": 1000000}, gm.applied[0].AnthropicModelContextWindows) - assert.Empty(t, gm.applied[0].AnthropicBaseURL) + assert.Equal(t, server.URL+"/anthropic", gm.applied[0].AnthropicBaseURL) + }) + } +} + +func TestSetupPiAnthropicPrefix(t *testing.T) { + t.Parallel() + for _, tt := range []struct { + name, flag, wantPath, wantBase string + flagSet bool + }{ + {name: "root", flagSet: true, wantPath: "/v1/models"}, + {name: "custom", flag: "/custom", flagSet: true, wantPath: "/custom/v1/models", wantBase: "/custom"}, + {name: "probed", wantPath: "/anthropic/v1/models", wantBase: "/anthropic"}, + } { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodHead && r.URL.Path == "/anthropic/v1/messages": + w.WriteHeader(http.StatusUnauthorized) + case r.URL.Path == "/v1/models" && r.Method == http.MethodGet && tt.wantPath != "/v1/models": + w.WriteHeader(http.StatusNotFound) + case r.URL.Path == tt.wantPath && r.Method == http.MethodGet: + _, _ = w.Write([]byte(`{"data":[{"id":"claude-test"}],"has_more":false}`)) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + t.Cleanup(server.Close) + gm := &setupGatewayManager{detected: []string{"pi"}, mode: llmgateway.ModePi} + provider := configuredSetupProvider() + provider.cfg.GatewayURL = server.URL + provider.cfg.TLSSkipVerify = true + var out, errOut bytes.Buffer + err := Setup(context.Background(), &out, &errOut, gm, provider, + func(_ context.Context, cfg *Config) error { cfg.SetDiscoveryAccessToken("token"); return nil }, + SetOptions{}, tt.flag, tt.flagSet, "pi", false) + require.NoError(t, err) + require.Len(t, gm.applied, 1) + assert.Equal(t, []string{"claude-test"}, gm.applied[0].AnthropicModels) + wantBase := "" + if tt.wantBase != "" { + wantBase = server.URL + tt.wantBase + } + assert.Equal(t, wantBase, gm.applied[0].AnthropicBaseURL) }) } } @@ -992,7 +1044,7 @@ func TestDiscoverSetupModelsPiAnthropicFallback(t *testing.T) { cfg := Config{GatewayURL: server.URL} cfg.SetDiscoveryAccessToken("discovery-token") var warnings strings.Builder - clients, catalog, err := discoverSetupModels(context.Background(), &warnings, []string{"pi"}, "pi", cfg) + clients, catalog, err := discoverSetupModels(context.Background(), &warnings, []string{"pi"}, "pi", cfg, "/anthropic") require.NoError(t, err) assert.Equal(t, []string{"pi"}, clients) assert.Empty(t, catalog.openAI) @@ -1020,7 +1072,7 @@ func TestDiscoverAnthropicGatewayModels(t *testing.T) { t.Cleanup(server.Close) cfg := Config{GatewayURL: server.URL} cfg.SetDiscoveryAccessToken("discovery-token") - models, contexts, err := discoverAnthropicGatewayModels(context.Background(), cfg) + models, contexts, err := discoverAnthropicGatewayModels(context.Background(), cfg, "/anthropic") require.NoError(t, err) assert.Equal(t, []string{"claude-haiku-5-5", "claude-opus-4-8", "claude-sonnet-5"}, models) assert.Equal(t, map[string]int{"claude-opus-4-8": 1000000, "claude-sonnet-5": 200000}, contexts) @@ -1034,7 +1086,7 @@ func TestDiscoverAnthropicGatewayModelsRejectsInvalidPagination(t *testing.T) { t.Cleanup(server.Close) cfg := Config{GatewayURL: server.URL} cfg.SetDiscoveryAccessToken("discovery-token") - _, _, err := discoverAnthropicGatewayModels(context.Background(), cfg) + _, _, err := discoverAnthropicGatewayModels(context.Background(), cfg, "/anthropic") require.ErrorContains(t, err, "invalid Anthropic pagination cursor") } diff --git a/pkg/llmgateway/config.go b/pkg/llmgateway/config.go index 2d916d3abc..3fdad6640a 100644 --- a/pkg/llmgateway/config.go +++ b/pkg/llmgateway/config.go @@ -10,6 +10,9 @@ import ( "time" ) +// TokenHelperShellCommand is used by shell-invoked LLM gateway token helpers. +const TokenHelperShellCommand = "thv llm token" //nolint:gosec // command line, not a credential + // ClaudeCodeHelperTTL is written to settings.json as // CLAUDE_CODE_API_KEY_HELPER_TTL_MS: how often Claude Code re-invokes the // apiKeyHelper ("thv llm token"). Pinned to Claude Code's documented default so From 25a3eff50134c90b18d7658ce177648c9cd64485 Mon Sep 17 00:00:00 2001 From: Jeremy Drouillard Date: Wed, 14 Oct 2026 15:29:44 -0700 Subject: [PATCH 4/6] Enable Pi reasoning from gateway discovery Staging advertises Anthropic thinking support in model discovery. Propagate that metadata to Pi so supported models expose thinking, while leaving fields the gateway does not provide at Pi's defaults. --- cmd/thv/app/llm.go | 10 ++++++---- docs/cli/thv_llm_setup.md | 10 ++++++---- pkg/client/llm_gateway_pi.go | 11 +++++++---- pkg/client/llm_gateway_pi_test.go | 7 +++++++ pkg/llm/setup.go | 29 +++++++++++++++++++++-------- pkg/llm/setup_test.go | 16 +++++++++++----- pkg/llmgateway/config.go | 3 +++ 7 files changed, 61 insertions(+), 25 deletions(-) diff --git a/cmd/thv/app/llm.go b/cmd/thv/app/llm.go index bde016e56d..e020fcba0b 100644 --- a/cmd/thv/app/llm.go +++ b/cmd/thv/app/llm.go @@ -312,12 +312,14 @@ reopen it after setup. Pi is configured with OpenAI-compatible and Anthropic providers in ~/.pi/agent/models.json (or $PI_CODING_AGENT_DIR/models.json). It runs "thv llm token" for each request; authenticated setup discovers models and -context sizes from both gateway APIs. --models filters the discovered models. -Pi does not support --lazy because model discovery determines each model's API. +context sizes from both gateway APIs and Anthropic thinking support when +advertised. --models filters the discovered models. Pi does not support --lazy +because model discovery determines each model's API. Pi defaults to short prompt caching: start Pi with PI_CACHE_RETENTION=long in its environment for long retention (Anthropic: 1h; OpenAI: 24h). -This cannot be persisted in models.json. Pi defaults models without discovered -capability metadata to text-only, no reasoning, and 16K max output tokens. +This cannot be persisted in models.json. The gateway does not currently provide +image-input or max-output metadata, so Pi defaults to text-only input and 16K +max output tokens. OpenAI models without reasoning metadata default to no reasoning. If Pi cannot refresh its gateway token, run "thv llm token" in a terminal to log in again. diff --git a/docs/cli/thv_llm_setup.md b/docs/cli/thv_llm_setup.md index 88c33340df..3d9c68d35b 100644 --- a/docs/cli/thv_llm_setup.md +++ b/docs/cli/thv_llm_setup.md @@ -38,12 +38,14 @@ reopen it after setup. Pi is configured with OpenAI-compatible and Anthropic providers in ~/.pi/agent/models.json (or $PI_CODING_AGENT_DIR/models.json). It runs "thv llm token" for each request; authenticated setup discovers models and -context sizes from both gateway APIs. --models filters the discovered models. -Pi does not support --lazy because model discovery determines each model's API. +context sizes from both gateway APIs and Anthropic thinking support when +advertised. --models filters the discovered models. Pi does not support --lazy +because model discovery determines each model's API. Pi defaults to short prompt caching: start Pi with PI_CACHE_RETENTION=long in its environment for long retention (Anthropic: 1h; OpenAI: 24h). -This cannot be persisted in models.json. Pi defaults models without discovered -capability metadata to text-only, no reasoning, and 16K max output tokens. +This cannot be persisted in models.json. The gateway does not currently provide +image-input or max-output metadata, so Pi defaults to text-only input and 16K +max output tokens. OpenAI models without reasoning metadata default to no reasoning. If Pi cannot refresh its gateway token, run "thv llm token" in a terminal to log in again. diff --git a/pkg/client/llm_gateway_pi.go b/pkg/client/llm_gateway_pi.go index f562d4969c..d28c8bf8c4 100644 --- a/pkg/client/llm_gateway_pi.go +++ b/pkg/client/llm_gateway_pi.go @@ -51,7 +51,7 @@ func (cm *ClientManager) configurePi(appCfg *clientAppConfig, cfg llmgateway.App baseURL := strings.TrimRight(cfg.GatewayURL, "/") if len(openAIModels) > 0 { provider, err := encodePiGatewayProvider(baseURL+"/v1", "openai-completions", openAIModels, - cfg.DiscoveredModelContextWindows) + cfg.DiscoveredModelContextWindows, nil) if err != nil { return err } @@ -65,7 +65,7 @@ func (cm *ClientManager) configurePi(appCfg *clientAppConfig, cfg llmgateway.App anthropicBaseURL = strings.TrimRight(cfg.AnthropicBaseURL, "/") } provider, err := encodePiGatewayProvider(anthropicBaseURL, "anthropic-messages", anthropicModels, - cfg.AnthropicModelContextWindows) + cfg.AnthropicModelContextWindows, cfg.AnthropicModelReasoning) if err != nil { return err } @@ -109,14 +109,17 @@ func piProviderManaged(existing json.RawMessage) bool { return json.Unmarshal(existing, &provider) == nil && provider.APIKey == piTokenCommand } -func encodePiGatewayProvider(baseURL, api string, models []string, contextWindows map[string]int) (json.RawMessage, error) { +func encodePiGatewayProvider( + baseURL, api string, models []string, contextWindows map[string]int, reasoning map[string]bool, +) (json.RawMessage, error) { type piModel struct { ID string `json:"id"` ContextWindow int `json:"contextWindow,omitempty"` + Reasoning bool `json:"reasoning,omitempty"` } entries := make([]piModel, 0, len(models)) for _, id := range models { - entries = append(entries, piModel{ID: id, ContextWindow: contextWindows[id]}) + entries = append(entries, piModel{ID: id, ContextWindow: contextWindows[id], Reasoning: reasoning[id]}) } provider, err := json.Marshal(struct { BaseURL string `json:"baseUrl"` diff --git a/pkg/client/llm_gateway_pi_test.go b/pkg/client/llm_gateway_pi_test.go index 46756fa9b4..d0594d3e1d 100644 --- a/pkg/client/llm_gateway_pi_test.go +++ b/pkg/client/llm_gateway_pi_test.go @@ -29,6 +29,7 @@ func TestPiGatewaySetupAndTeardown(t *testing.T) { DiscoveredModelContextWindows: map[string]int{"anthropic/claude-sonnet-5": 1000000}, AnthropicModels: []string{"claude-opus-4-8", "claude-haiku-5-5"}, AnthropicModelContextWindows: map[string]int{"claude-opus-4-8": 1000000}, + AnthropicModelReasoning: map[string]bool{"claude-opus-4-8": true}, } for range 2 { got, err := cm.ConfigureLLMGateway(ClientApp(Pi), cfg) @@ -47,6 +48,7 @@ func TestPiGatewaySetupAndTeardown(t *testing.T) { Models []struct { ID string `json:"id"` ContextWindow *int `json:"contextWindow"` + Reasoning *bool `json:"reasoning"` } `json:"models"` } `json:"providers"` } @@ -62,6 +64,8 @@ func TestPiGatewaySetupAndTeardown(t *testing.T) { require.Equal(t, 1000000, *root.Providers[piProviderName].Models[0].ContextWindow) require.Equal(t, "gpt-5", root.Providers[piProviderName].Models[1].ID) require.Nil(t, root.Providers[piProviderName].Models[1].ContextWindow) + require.Nil(t, root.Providers[piProviderName].Models[0].Reasoning) + require.Nil(t, root.Providers[piProviderName].Models[1].Reasoning) require.Equal(t, "https://gateway.example/custom", root.Providers[piAnthropicProviderName].BaseURL) require.Equal(t, "anthropic-messages", root.Providers[piAnthropicProviderName].API) require.Equal(t, piTokenCommand, root.Providers[piAnthropicProviderName].APIKey) @@ -69,7 +73,10 @@ func TestPiGatewaySetupAndTeardown(t *testing.T) { require.Equal(t, "claude-opus-4-8", root.Providers[piAnthropicProviderName].Models[0].ID) require.NotNil(t, root.Providers[piAnthropicProviderName].Models[0].ContextWindow) require.Equal(t, 1000000, *root.Providers[piAnthropicProviderName].Models[0].ContextWindow) + require.NotNil(t, root.Providers[piAnthropicProviderName].Models[0].Reasoning) + require.True(t, *root.Providers[piAnthropicProviderName].Models[0].Reasoning) require.Nil(t, root.Providers[piAnthropicProviderName].Models[1].ContextWindow) + require.Nil(t, root.Providers[piAnthropicProviderName].Models[1].Reasoning) require.NoError(t, cm.RevertLLMGateway(ClientApp(Pi), path)) content, err = os.ReadFile(path) diff --git a/pkg/llm/setup.go b/pkg/llm/setup.go index 8e1b927e85..3dd77395d7 100644 --- a/pkg/llm/setup.go +++ b/pkg/llm/setup.go @@ -571,6 +571,7 @@ type discoveredGatewayModels struct { openAIContextWindows map[string]int anthropic []string anthropicContextWindows map[string]int + anthropicReasoning map[string]bool } func discoverSetupModels( @@ -606,7 +607,7 @@ func fetchSetupModelCatalog( catalog.openAI, catalog.openAIContextWindows, openErr = discoverGatewayModels(ctx, cfg) } if hasPi && cfg.discoveryAccessToken != "" { - catalog.anthropic, catalog.anthropicContextWindows, anthropicErr = + catalog.anthropic, catalog.anthropicContextWindows, catalog.anthropicReasoning, anthropicErr = discoverAnthropicGatewayModels(ctx, cfg, anthropicPrefix) } return catalog, openErr, anthropicErr @@ -640,6 +641,11 @@ type gatewayModelsResponse struct { ID string `json:"id"` ContextWindow int `json:"context_window"` MaxInputTokens int `json:"max_input_tokens"` + Capabilities *struct { + Thinking struct { + Supported bool `json:"supported"` + } `json:"thinking"` + } `json:"capabilities"` } `json:"data"` } @@ -711,7 +717,9 @@ func discoverGatewayModels(ctx context.Context, cfg Config) ([]string, map[strin return models, contextWindows, nil } -func discoverAnthropicGatewayModels(ctx context.Context, cfg Config, anthropicPrefix string) ([]string, map[string]int, error) { +func discoverAnthropicGatewayModels( + ctx context.Context, cfg Config, anthropicPrefix string, +) ([]string, map[string]int, map[string]bool, error) { path := strings.Trim(anthropicPrefix, "/") if path != "" { path += "/" @@ -720,19 +728,20 @@ func discoverAnthropicGatewayModels(ctx context.Context, cfg Config, anthropicPr seen := make(map[string]struct{}) models := make([]string, 0) contextWindows := make(map[string]int) + reasoning := make(map[string]bool) afterID := "" for range 100 { payload, err := requestGatewayModels(ctx, cfg, path, afterID) if err != nil { - return nil, nil, err + return nil, nil, nil, err } if payload.HasMore == nil || len(payload.Data) == 0 { - return nil, nil, fmt.Errorf("decoding gateway model response: expected a non-empty Anthropic model page") + return nil, nil, nil, fmt.Errorf("decoding gateway model response: expected a non-empty Anthropic model page") } for _, model := range payload.Data { id := strings.TrimSpace(model.ID) if id == "" { - return nil, nil, fmt.Errorf("decoding gateway model response: model ID must not be empty") + return nil, nil, nil, fmt.Errorf("decoding gateway model response: model ID must not be empty") } if _, exists := seen[id]; exists { continue @@ -742,17 +751,20 @@ func discoverAnthropicGatewayModels(ctx context.Context, cfg Config, anthropicPr if model.MaxInputTokens > 0 { contextWindows[id] = model.MaxInputTokens } + if model.Capabilities != nil && model.Capabilities.Thinking.Supported { + reasoning[id] = true + } } if !*payload.HasMore { sort.Strings(models) - return models, contextWindows, nil + return models, contextWindows, reasoning, nil } if payload.LastID == "" || payload.LastID == afterID { - return nil, nil, fmt.Errorf("decoding gateway model response: invalid Anthropic pagination cursor") + return nil, nil, nil, fmt.Errorf("decoding gateway model response: invalid Anthropic pagination cursor") } afterID = payload.LastID } - return nil, nil, fmt.Errorf("decoding gateway model response: too many Anthropic model pages") + return nil, nil, nil, fmt.Errorf("decoding gateway model response: too many Anthropic model pages") } // Default Bedrock inference-profile model IDs written for Claude Code in @@ -912,6 +924,7 @@ func configureDetectedToolsWithDiscovery( DiscoveredModelContextWindows: discovered.openAIContextWindows, AnthropicModels: discovered.anthropic, AnthropicModelContextWindows: discovered.anthropicContextWindows, + AnthropicModelReasoning: discovered.anthropicReasoning, } if clientType == claudeCodeClient { diff --git a/pkg/llm/setup_test.go b/pkg/llm/setup_test.go index 2676b21d49..96170c564f 100644 --- a/pkg/llm/setup_test.go +++ b/pkg/llm/setup_test.go @@ -932,7 +932,9 @@ func TestSetupPiModels(t *testing.T) { case "/v1/models": _, _ = w.Write([]byte(`{"object":"list","data":[{"id":"gpt-5","context_window":400000}]}`)) case "/anthropic/v1/models": - _, _ = w.Write([]byte(`{"data":[{"id":"claude-sonnet-5","max_input_tokens":1000000}],"has_more":false}`)) + _, _ = w.Write([]byte(`{"data":[{"id":"claude-sonnet-5","max_input_tokens":1000000,` + + `"capabilities":{"thinking":{"supported":true,"types":{"adaptive":{"supported":true}}}}}],` + + `"has_more":false}`)) default: w.WriteHeader(http.StatusNotFound) } @@ -980,6 +982,7 @@ func TestSetupPiModels(t *testing.T) { assert.Equal(t, map[string]int{"gpt-5": 400000}, gm.applied[0].DiscoveredModelContextWindows) assert.Equal(t, []string{"claude-sonnet-5"}, gm.applied[0].AnthropicModels) assert.Equal(t, map[string]int{"claude-sonnet-5": 1000000}, gm.applied[0].AnthropicModelContextWindows) + assert.Equal(t, map[string]bool{"claude-sonnet-5": true}, gm.applied[0].AnthropicModelReasoning) assert.Equal(t, server.URL+"/anthropic", gm.applied[0].AnthropicBaseURL) }) } @@ -1060,8 +1063,10 @@ func TestDiscoverAnthropicGatewayModels(t *testing.T) { assert.Equal(t, "Bearer discovery-token", r.Header.Get("Authorization")) switch r.URL.Query().Get("after_id") { case "": - _, _ = w.Write([]byte(`{"data":[{"id":" claude-opus-4-8 ","max_input_tokens":1000000},` + - `{"id":"claude-haiku-5-5"}],"has_more":true,"last_id":"claude-haiku-5-5"}`)) + _, _ = w.Write([]byte(`{"data":[{"id":" claude-opus-4-8 ","max_input_tokens":1000000,` + + `"capabilities":{"thinking":{"supported":true,"types":{"adaptive":{"supported":true}}}}},` + + `{"id":"claude-haiku-5-5","capabilities":{"thinking":{"supported":false}}}],` + + `"has_more":true,"last_id":"claude-haiku-5-5"}`)) case "claude-haiku-5-5": _, _ = w.Write([]byte(`{"data":[{"id":"claude-sonnet-5","max_input_tokens":200000},` + `{"id":"claude-opus-4-8","max_input_tokens":10}],"has_more":false}`)) @@ -1072,10 +1077,11 @@ func TestDiscoverAnthropicGatewayModels(t *testing.T) { t.Cleanup(server.Close) cfg := Config{GatewayURL: server.URL} cfg.SetDiscoveryAccessToken("discovery-token") - models, contexts, err := discoverAnthropicGatewayModels(context.Background(), cfg, "/anthropic") + models, contexts, reasoning, err := discoverAnthropicGatewayModels(context.Background(), cfg, "/anthropic") require.NoError(t, err) assert.Equal(t, []string{"claude-haiku-5-5", "claude-opus-4-8", "claude-sonnet-5"}, models) assert.Equal(t, map[string]int{"claude-opus-4-8": 1000000, "claude-sonnet-5": 200000}, contexts) + assert.Equal(t, map[string]bool{"claude-opus-4-8": true}, reasoning) } func TestDiscoverAnthropicGatewayModelsRejectsInvalidPagination(t *testing.T) { @@ -1086,7 +1092,7 @@ func TestDiscoverAnthropicGatewayModelsRejectsInvalidPagination(t *testing.T) { t.Cleanup(server.Close) cfg := Config{GatewayURL: server.URL} cfg.SetDiscoveryAccessToken("discovery-token") - _, _, err := discoverAnthropicGatewayModels(context.Background(), cfg, "/anthropic") + _, _, _, err := discoverAnthropicGatewayModels(context.Background(), cfg, "/anthropic") require.ErrorContains(t, err, "invalid Anthropic pagination cursor") } diff --git a/pkg/llmgateway/config.go b/pkg/llmgateway/config.go index 3fdad6640a..2d1854facc 100644 --- a/pkg/llmgateway/config.go +++ b/pkg/llmgateway/config.go @@ -126,6 +126,9 @@ type ApplyConfig struct { // discovered Anthropic model catalogue for Pi. Sizes are max_input_tokens. AnthropicModels []string AnthropicModelContextWindows map[string]int + // AnthropicModelReasoning is true for Anthropic models whose discovery + // capabilities advertise thinking support. Missing entries use Pi's default. + AnthropicModelReasoning map[string]bool // ShortPromptCache opts supported clients out of ToolHive's one-hour // prompt-cache lifetime and restores their shorter default. ShortPromptCache bool From 20986ceca755b1d484981cb1327475a0ef8076ae Mon Sep 17 00:00:00 2001 From: Jeremy Drouillard Date: Wed, 14 Oct 2026 15:40:58 -0700 Subject: [PATCH 5/6] Select Pi thinking format from gateway metadata Pi uses budget-based thinking by default for custom Anthropic models, but staging advertises adaptive-only models. Write the adaptive compatibility setting when discovery reports that mode so reasoning requests do not fail. --- pkg/client/llm_gateway_pi.go | 22 ++++++++++++------ pkg/client/llm_gateway_pi_test.go | 15 +++++++++++-- pkg/llm/setup.go | 37 ++++++++++++++++++++++--------- pkg/llm/setup_test.go | 14 ++++++++---- pkg/llmgateway/config.go | 7 ++++-- 5 files changed, 70 insertions(+), 25 deletions(-) diff --git a/pkg/client/llm_gateway_pi.go b/pkg/client/llm_gateway_pi.go index d28c8bf8c4..aa4c9ac0e3 100644 --- a/pkg/client/llm_gateway_pi.go +++ b/pkg/client/llm_gateway_pi.go @@ -51,7 +51,7 @@ func (cm *ClientManager) configurePi(appCfg *clientAppConfig, cfg llmgateway.App baseURL := strings.TrimRight(cfg.GatewayURL, "/") if len(openAIModels) > 0 { provider, err := encodePiGatewayProvider(baseURL+"/v1", "openai-completions", openAIModels, - cfg.DiscoveredModelContextWindows, nil) + cfg.DiscoveredModelContextWindows, nil, nil) if err != nil { return err } @@ -65,7 +65,7 @@ func (cm *ClientManager) configurePi(appCfg *clientAppConfig, cfg llmgateway.App anthropicBaseURL = strings.TrimRight(cfg.AnthropicBaseURL, "/") } provider, err := encodePiGatewayProvider(anthropicBaseURL, "anthropic-messages", anthropicModels, - cfg.AnthropicModelContextWindows, cfg.AnthropicModelReasoning) + cfg.AnthropicModelContextWindows, cfg.AnthropicModelReasoning, cfg.AnthropicModelAdaptive) if err != nil { return err } @@ -110,16 +110,24 @@ func piProviderManaged(existing json.RawMessage) bool { } func encodePiGatewayProvider( - baseURL, api string, models []string, contextWindows map[string]int, reasoning map[string]bool, + baseURL, api string, models []string, contextWindows map[string]int, reasoning, adaptive map[string]bool, ) (json.RawMessage, error) { + type piCompat struct { + ForceAdaptiveThinking bool `json:"forceAdaptiveThinking"` + } type piModel struct { - ID string `json:"id"` - ContextWindow int `json:"contextWindow,omitempty"` - Reasoning bool `json:"reasoning,omitempty"` + ID string `json:"id"` + ContextWindow int `json:"contextWindow,omitempty"` + Reasoning bool `json:"reasoning,omitempty"` + Compat *piCompat `json:"compat,omitempty"` } entries := make([]piModel, 0, len(models)) for _, id := range models { - entries = append(entries, piModel{ID: id, ContextWindow: contextWindows[id], Reasoning: reasoning[id]}) + entry := piModel{ID: id, ContextWindow: contextWindows[id], Reasoning: reasoning[id]} + if adaptive[id] { + entry.Compat = &piCompat{ForceAdaptiveThinking: true} + } + entries = append(entries, entry) } provider, err := json.Marshal(struct { BaseURL string `json:"baseUrl"` diff --git a/pkg/client/llm_gateway_pi_test.go b/pkg/client/llm_gateway_pi_test.go index d0594d3e1d..d0d6d78131 100644 --- a/pkg/client/llm_gateway_pi_test.go +++ b/pkg/client/llm_gateway_pi_test.go @@ -27,9 +27,10 @@ func TestPiGatewaySetupAndTeardown(t *testing.T) { AnthropicBaseURL: "https://gateway.example/custom", DiscoveredModels: []string{"anthropic/claude-sonnet-5", "gpt-5"}, DiscoveredModelContextWindows: map[string]int{"anthropic/claude-sonnet-5": 1000000}, - AnthropicModels: []string{"claude-opus-4-8", "claude-haiku-5-5"}, + AnthropicModels: []string{"claude-opus-4-8", "claude-haiku-5-5", "claude-haiku-4-5"}, AnthropicModelContextWindows: map[string]int{"claude-opus-4-8": 1000000}, - AnthropicModelReasoning: map[string]bool{"claude-opus-4-8": true}, + AnthropicModelReasoning: map[string]bool{"claude-opus-4-8": true, "claude-haiku-4-5": true}, + AnthropicModelAdaptive: map[string]bool{"claude-opus-4-8": true}, } for range 2 { got, err := cm.ConfigureLLMGateway(ClientApp(Pi), cfg) @@ -49,6 +50,9 @@ func TestPiGatewaySetupAndTeardown(t *testing.T) { ID string `json:"id"` ContextWindow *int `json:"contextWindow"` Reasoning *bool `json:"reasoning"` + Compat *struct { + ForceAdaptiveThinking bool `json:"forceAdaptiveThinking"` + } `json:"compat"` } `json:"models"` } `json:"providers"` } @@ -66,6 +70,7 @@ func TestPiGatewaySetupAndTeardown(t *testing.T) { require.Nil(t, root.Providers[piProviderName].Models[1].ContextWindow) require.Nil(t, root.Providers[piProviderName].Models[0].Reasoning) require.Nil(t, root.Providers[piProviderName].Models[1].Reasoning) + require.Nil(t, root.Providers[piProviderName].Models[0].Compat) require.Equal(t, "https://gateway.example/custom", root.Providers[piAnthropicProviderName].BaseURL) require.Equal(t, "anthropic-messages", root.Providers[piAnthropicProviderName].API) require.Equal(t, piTokenCommand, root.Providers[piAnthropicProviderName].APIKey) @@ -75,8 +80,14 @@ func TestPiGatewaySetupAndTeardown(t *testing.T) { require.Equal(t, 1000000, *root.Providers[piAnthropicProviderName].Models[0].ContextWindow) require.NotNil(t, root.Providers[piAnthropicProviderName].Models[0].Reasoning) require.True(t, *root.Providers[piAnthropicProviderName].Models[0].Reasoning) + require.NotNil(t, root.Providers[piAnthropicProviderName].Models[0].Compat) + require.True(t, root.Providers[piAnthropicProviderName].Models[0].Compat.ForceAdaptiveThinking) require.Nil(t, root.Providers[piAnthropicProviderName].Models[1].ContextWindow) require.Nil(t, root.Providers[piAnthropicProviderName].Models[1].Reasoning) + require.Nil(t, root.Providers[piAnthropicProviderName].Models[1].Compat) + require.NotNil(t, root.Providers[piAnthropicProviderName].Models[2].Reasoning) + require.True(t, *root.Providers[piAnthropicProviderName].Models[2].Reasoning) + require.Nil(t, root.Providers[piAnthropicProviderName].Models[2].Compat) require.NoError(t, cm.RevertLLMGateway(ClientApp(Pi), path)) content, err = os.ReadFile(path) diff --git a/pkg/llm/setup.go b/pkg/llm/setup.go index 3dd77395d7..f48a42e4fd 100644 --- a/pkg/llm/setup.go +++ b/pkg/llm/setup.go @@ -572,6 +572,7 @@ type discoveredGatewayModels struct { anthropic []string anthropicContextWindows map[string]int anthropicReasoning map[string]bool + anthropicAdaptive map[string]bool } func discoverSetupModels( @@ -607,7 +608,7 @@ func fetchSetupModelCatalog( catalog.openAI, catalog.openAIContextWindows, openErr = discoverGatewayModels(ctx, cfg) } if hasPi && cfg.discoveryAccessToken != "" { - catalog.anthropic, catalog.anthropicContextWindows, catalog.anthropicReasoning, anthropicErr = + catalog.anthropic, catalog.anthropicContextWindows, catalog.anthropicReasoning, catalog.anthropicAdaptive, anthropicErr = discoverAnthropicGatewayModels(ctx, cfg, anthropicPrefix) } return catalog, openErr, anthropicErr @@ -644,6 +645,14 @@ type gatewayModelsResponse struct { Capabilities *struct { Thinking struct { Supported bool `json:"supported"` + Types struct { + Adaptive struct { + Supported bool `json:"supported"` + } `json:"adaptive"` + Enabled struct { + Supported bool `json:"supported"` + } `json:"enabled"` + } `json:"types"` } `json:"thinking"` } `json:"capabilities"` } `json:"data"` @@ -719,7 +728,7 @@ func discoverGatewayModels(ctx context.Context, cfg Config) ([]string, map[strin func discoverAnthropicGatewayModels( ctx context.Context, cfg Config, anthropicPrefix string, -) ([]string, map[string]int, map[string]bool, error) { +) ([]string, map[string]int, map[string]bool, map[string]bool, error) { path := strings.Trim(anthropicPrefix, "/") if path != "" { path += "/" @@ -729,19 +738,20 @@ func discoverAnthropicGatewayModels( models := make([]string, 0) contextWindows := make(map[string]int) reasoning := make(map[string]bool) + adaptive := make(map[string]bool) afterID := "" for range 100 { payload, err := requestGatewayModels(ctx, cfg, path, afterID) if err != nil { - return nil, nil, nil, err + return nil, nil, nil, nil, err } if payload.HasMore == nil || len(payload.Data) == 0 { - return nil, nil, nil, fmt.Errorf("decoding gateway model response: expected a non-empty Anthropic model page") + return nil, nil, nil, nil, fmt.Errorf("decoding gateway model response: expected a non-empty Anthropic model page") } for _, model := range payload.Data { id := strings.TrimSpace(model.ID) if id == "" { - return nil, nil, nil, fmt.Errorf("decoding gateway model response: model ID must not be empty") + return nil, nil, nil, nil, fmt.Errorf("decoding gateway model response: model ID must not be empty") } if _, exists := seen[id]; exists { continue @@ -751,20 +761,26 @@ func discoverAnthropicGatewayModels( if model.MaxInputTokens > 0 { contextWindows[id] = model.MaxInputTokens } - if model.Capabilities != nil && model.Capabilities.Thinking.Supported { - reasoning[id] = true + if model.Capabilities != nil { + thinking := model.Capabilities.Thinking + reasoning[id], adaptive[id] = piThinkingModes( + thinking.Supported, thinking.Types.Adaptive.Supported, thinking.Types.Enabled.Supported) } } if !*payload.HasMore { sort.Strings(models) - return models, contextWindows, reasoning, nil + return models, contextWindows, reasoning, adaptive, nil } if payload.LastID == "" || payload.LastID == afterID { - return nil, nil, nil, fmt.Errorf("decoding gateway model response: invalid Anthropic pagination cursor") + return nil, nil, nil, nil, fmt.Errorf("decoding gateway model response: invalid Anthropic pagination cursor") } afterID = payload.LastID } - return nil, nil, nil, fmt.Errorf("decoding gateway model response: too many Anthropic model pages") + return nil, nil, nil, nil, fmt.Errorf("decoding gateway model response: too many Anthropic model pages") +} + +func piThinkingModes(supported, adaptive, enabled bool) (bool, bool) { + return supported && (adaptive || enabled), supported && adaptive } // Default Bedrock inference-profile model IDs written for Claude Code in @@ -925,6 +941,7 @@ func configureDetectedToolsWithDiscovery( AnthropicModels: discovered.anthropic, AnthropicModelContextWindows: discovered.anthropicContextWindows, AnthropicModelReasoning: discovered.anthropicReasoning, + AnthropicModelAdaptive: discovered.anthropicAdaptive, } if clientType == claudeCodeClient { diff --git a/pkg/llm/setup_test.go b/pkg/llm/setup_test.go index 96170c564f..ce41a110ef 100644 --- a/pkg/llm/setup_test.go +++ b/pkg/llm/setup_test.go @@ -983,6 +983,7 @@ func TestSetupPiModels(t *testing.T) { assert.Equal(t, []string{"claude-sonnet-5"}, gm.applied[0].AnthropicModels) assert.Equal(t, map[string]int{"claude-sonnet-5": 1000000}, gm.applied[0].AnthropicModelContextWindows) assert.Equal(t, map[string]bool{"claude-sonnet-5": true}, gm.applied[0].AnthropicModelReasoning) + assert.Equal(t, map[string]bool{"claude-sonnet-5": true}, gm.applied[0].AnthropicModelAdaptive) assert.Equal(t, server.URL+"/anthropic", gm.applied[0].AnthropicBaseURL) }) } @@ -1069,6 +1070,8 @@ func TestDiscoverAnthropicGatewayModels(t *testing.T) { `"has_more":true,"last_id":"claude-haiku-5-5"}`)) case "claude-haiku-5-5": _, _ = w.Write([]byte(`{"data":[{"id":"claude-sonnet-5","max_input_tokens":200000},` + + `{"id":"claude-haiku-4-5","capabilities":{"thinking":{"supported":true,` + + `"types":{"enabled":{"supported":true}}}}},` + `{"id":"claude-opus-4-8","max_input_tokens":10}],"has_more":false}`)) default: w.WriteHeader(http.StatusBadRequest) @@ -1077,11 +1080,14 @@ func TestDiscoverAnthropicGatewayModels(t *testing.T) { t.Cleanup(server.Close) cfg := Config{GatewayURL: server.URL} cfg.SetDiscoveryAccessToken("discovery-token") - models, contexts, reasoning, err := discoverAnthropicGatewayModels(context.Background(), cfg, "/anthropic") + models, contexts, reasoning, adaptive, err := discoverAnthropicGatewayModels(context.Background(), cfg, "/anthropic") require.NoError(t, err) - assert.Equal(t, []string{"claude-haiku-5-5", "claude-opus-4-8", "claude-sonnet-5"}, models) + assert.Equal(t, []string{"claude-haiku-4-5", "claude-haiku-5-5", "claude-opus-4-8", "claude-sonnet-5"}, models) assert.Equal(t, map[string]int{"claude-opus-4-8": 1000000, "claude-sonnet-5": 200000}, contexts) - assert.Equal(t, map[string]bool{"claude-opus-4-8": true}, reasoning) + assert.Equal(t, map[string]bool{"claude-opus-4-8": true, "claude-haiku-4-5": true, + "claude-haiku-5-5": false}, reasoning) + assert.Equal(t, map[string]bool{"claude-opus-4-8": true, "claude-haiku-4-5": false, + "claude-haiku-5-5": false}, adaptive) } func TestDiscoverAnthropicGatewayModelsRejectsInvalidPagination(t *testing.T) { @@ -1092,7 +1098,7 @@ func TestDiscoverAnthropicGatewayModelsRejectsInvalidPagination(t *testing.T) { t.Cleanup(server.Close) cfg := Config{GatewayURL: server.URL} cfg.SetDiscoveryAccessToken("discovery-token") - _, _, _, err := discoverAnthropicGatewayModels(context.Background(), cfg, "/anthropic") + _, _, _, _, err := discoverAnthropicGatewayModels(context.Background(), cfg, "/anthropic") require.ErrorContains(t, err, "invalid Anthropic pagination cursor") } diff --git a/pkg/llmgateway/config.go b/pkg/llmgateway/config.go index 2d1854facc..02c84ff949 100644 --- a/pkg/llmgateway/config.go +++ b/pkg/llmgateway/config.go @@ -126,9 +126,12 @@ type ApplyConfig struct { // discovered Anthropic model catalogue for Pi. Sizes are max_input_tokens. AnthropicModels []string AnthropicModelContextWindows map[string]int - // AnthropicModelReasoning is true for Anthropic models whose discovery - // capabilities advertise thinking support. Missing entries use Pi's default. + // AnthropicModelReasoning is true for Anthropic models with a supported + // thinking mode. Missing entries use Pi's default. AnthropicModelReasoning map[string]bool + // AnthropicModelAdaptive marks models that accept adaptive thinking; Pi + // otherwise sends budget-based thinking for reasoning models. + AnthropicModelAdaptive map[string]bool // ShortPromptCache opts supported clients out of ToolHive's one-hour // prompt-cache lifetime and restores their shorter default. ShortPromptCache bool From 08b175c4dab190d776604a25f3956e4bb49b325a Mon Sep 17 00:00:00 2001 From: Jeremy Drouillard Date: Wed, 14 Oct 2026 15:59:44 -0700 Subject: [PATCH 6/6] Combine Pi MCP and LLM integrations Main added Pi as an MCP and skills client after this branch added it as a gateway-only client. Reuse the existing Pi entry for both integrations to avoid a duplicate declaration and unblock CI. --- pkg/client/config.go | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/pkg/client/config.go b/pkg/client/config.go index 7222d9beb2..86af32ebd2 100644 --- a/pkg/client/config.go +++ b/pkg/client/config.go @@ -100,8 +100,6 @@ const ( ) const ( - // Pi represents the Pi coding agent's LLM gateway integration. - Pi LLMClientApp = "pi" // Xcode represents GitHub Copilot for Xcode. // Xcode does not support MCP; it is an LLM-gateway-only tool. // It is declared as LLMClientApp (not ClientApp) so that code generators @@ -915,15 +913,6 @@ var supportedClientIntegrations = []clientAppConfig{ SkillsGlobalPath: []string{".vibe", skillsDirName}, SkillsProjectPath: []string{".vibe", skillsDirName}, }, - { - ClientType: ClientApp(Pi), - Description: "Pi coding agent", - LLMGatewayOnly: true, - LLMGatewayMode: llmgateway.ModePi, - LLMBinaryName: "pi", - LLMSettingsFile: "models.json", - LLMSettingsRelPath: []string{".pi", "agent"}, - }, { ClientType: Codex, Description: "OpenAI Codex CLI", @@ -1063,9 +1052,13 @@ var supportedClientIntegrations = []clientAppConfig{ types.TransportTypeSSE: defaultURLFieldName, types.TransportTypeStreamableHTTP: defaultURLFieldName, }, - SupportsSkills: true, - SkillsGlobalPath: []string{".pi", "agent", skillsDirName}, - SkillsProjectPath: []string{".pi", skillsDirName}, + SupportsSkills: true, + SkillsGlobalPath: []string{".pi", "agent", skillsDirName}, + SkillsProjectPath: []string{".pi", skillsDirName}, + LLMGatewayMode: llmgateway.ModePi, + LLMBinaryName: "pi", + LLMSettingsFile: "models.json", + LLMSettingsRelPath: []string{".pi", "agent"}, }, { // Xcode does not support MCP; it is an LLM-gateway-only entry.