diff --git a/src/ipc/utils/fallback_ai_model.test.ts b/src/ipc/utils/fallback_ai_model.test.ts new file mode 100644 index 0000000000..69ec08b4d1 --- /dev/null +++ b/src/ipc/utils/fallback_ai_model.test.ts @@ -0,0 +1,238 @@ +import type { + LanguageModelV3, + LanguageModelV3CallOptions, + LanguageModelV3StreamPart, +} from "@ai-sdk/provider"; +import { describe, expect, it } from "vitest"; + +import { createFallback } from "./fallback_ai_model"; + +/** + * The regression under test: call options (temperature) are resolved for the + * PRIMARY model before the request; forwarding them verbatim to a fallback of + * a different provider produced a hard 400 (Anthropic rejects an explicit + * temperature for thinking models), turning a recoverable stream blip into a + * fatal error. On any non-primary model the fallback wrapper must drop + * `temperature` and let the provider default apply. + */ + +function textStream(): ReadableStream { + return new ReadableStream({ + start(controller) { + controller.enqueue({ type: "stream-start", warnings: [] } as any); + controller.enqueue({ type: "text-delta", id: "1", delta: "ok" } as any); + controller.close(); + }, + }); +} + +function fakeModel(params: { + modelId: string; + behavior: "succeed" | "reject-retryable"; + seen: LanguageModelV3CallOptions[]; +}): LanguageModelV3 { + return { + specificationVersion: "v3", + provider: "fake", + modelId: params.modelId, + supportedUrls: {}, + async doGenerate() { + throw new Error("not used"); + }, + async doStream(options: LanguageModelV3CallOptions) { + params.seen.push(options); + if (params.behavior === "reject-retryable") { + // Matches RETRYABLE_ERROR_PATTERNS ("service unavailable"). + throw new Error("service unavailable"); + } + return { stream: textStream() }; + }, + } as unknown as LanguageModelV3; +} + +async function drain(stream: ReadableStream) { + const reader = stream.getReader(); + while (!(await reader.read()).done) { + // consume + } +} + +describe("fallback model call options", () => { + it("passes temperature to the primary model untouched", async () => { + const seen: LanguageModelV3CallOptions[] = []; + const model = createFallback({ + models: [fakeModel({ modelId: "primary", behavior: "succeed", seen })], + }) as unknown as LanguageModelV3; + + const result = await model.doStream({ + prompt: [], + temperature: 1, + } as unknown as LanguageModelV3CallOptions); + await drain(result.stream); + + expect(seen).toHaveLength(1); + expect(seen[0].temperature).toBe(1); + }); + + it("drops temperature when failing over to a non-primary model", async () => { + const primarySeen: LanguageModelV3CallOptions[] = []; + const fallbackSeen: LanguageModelV3CallOptions[] = []; + const model = createFallback({ + models: [ + fakeModel({ + modelId: "primary", + behavior: "reject-retryable", + seen: primarySeen, + }), + fakeModel({ + modelId: "fallback", + behavior: "succeed", + seen: fallbackSeen, + }), + ], + }) as unknown as LanguageModelV3; + + const result = await model.doStream({ + prompt: [], + temperature: 1, + } as unknown as LanguageModelV3CallOptions); + await drain(result.stream); + + // Primary was tried with the caller's options... + expect(primarySeen.length).toBeGreaterThanOrEqual(1); + expect(primarySeen[0].temperature).toBe(1); + // ...the fallback's provider default must apply instead. + expect(fallbackSeen).toHaveLength(1); + expect(fallbackSeen[0].temperature).toBeUndefined(); + // Everything else survives the strip. + expect(fallbackSeen[0].prompt).toEqual([]); + }); + + it("applies a fallback model's own call options as if it were primary", async () => { + const primarySeen: LanguageModelV3CallOptions[] = []; + const fallbackSeen: LanguageModelV3CallOptions[] = []; + const model = createFallback({ + models: [ + fakeModel({ + modelId: "primary", + behavior: "reject-retryable", + seen: primarySeen, + }), + fakeModel({ + modelId: "fallback", + behavior: "succeed", + seen: fallbackSeen, + }), + ], + modelCallOptions: [ + undefined, // primary always gets the caller's options + { + temperature: 1, + maxOutputTokens: 32_000, + providerOptions: { + anthropic: { thinking: { type: "adaptive" } }, + }, + }, + ], + }) as unknown as LanguageModelV3; + + const result = await model.doStream({ + prompt: [], + temperature: 0.2, + maxOutputTokens: 128_000, + providerOptions: { + "dyad-engine": { dyadRequestId: "req-1" }, + openai: { reasoningEffort: "medium" }, + }, + } as unknown as LanguageModelV3CallOptions); + await drain(result.stream); + + expect(fallbackSeen).toHaveLength(1); + const seen = fallbackSeen[0]; + // The model-derived subset is the fallback's own... + expect(seen.temperature).toBe(1); + expect(seen.maxOutputTokens).toBe(32_000); + expect((seen.providerOptions as any).anthropic).toEqual({ + thinking: { type: "adaptive" }, + }); + // ...request-scoped options pass through untouched. + expect((seen.providerOptions as any)["dyad-engine"]).toEqual({ + dyadRequestId: "req-1", + }); + expect(seen.prompt).toEqual([]); + }); + + it("unsets temperature when the fallback's own options have none", async () => { + const fallbackSeen: LanguageModelV3CallOptions[] = []; + const model = createFallback({ + models: [ + fakeModel({ + modelId: "primary", + behavior: "reject-retryable", + seen: [], + }), + fakeModel({ + modelId: "fallback", + behavior: "succeed", + seen: fallbackSeen, + }), + ], + modelCallOptions: [ + undefined, + // catalog had no temperature/cap for this model: undefined means + // "unset", never "inherit the primary's" + { providerOptions: { anthropic: { thinking: { type: "adaptive" } } } }, + ], + }) as unknown as LanguageModelV3; + + const result = await model.doStream({ + prompt: [], + temperature: 0.2, + } as unknown as LanguageModelV3CallOptions); + await drain(result.stream); + + expect(fallbackSeen).toHaveLength(1); + expect(fallbackSeen[0].temperature).toBeUndefined(); + }); + + it("drops temperature on a sticky non-primary index without a same-request failover", async () => { + // After a failover the index stays on the fallback for modelResetInterval; + // a FRESH request's first call then already targets the fallback while its + // options were still computed for the primary selection. + const primarySeen: LanguageModelV3CallOptions[] = []; + const fallbackSeen: LanguageModelV3CallOptions[] = []; + const model = createFallback({ + models: [ + fakeModel({ + modelId: "primary", + behavior: "reject-retryable", + seen: primarySeen, + }), + fakeModel({ + modelId: "fallback", + behavior: "succeed", + seen: fallbackSeen, + }), + ], + }) as unknown as LanguageModelV3; + + // First request fails over primary -> fallback. + const first = await model.doStream({ + prompt: [], + temperature: 1, + } as unknown as LanguageModelV3CallOptions); + await drain(first.stream); + + // Second request starts on the sticky fallback index. + const second = await model.doStream({ + prompt: [], + temperature: 1, + } as unknown as LanguageModelV3CallOptions); + await drain(second.stream); + + expect(fallbackSeen).toHaveLength(2); + expect(fallbackSeen[1].temperature).toBeUndefined(); + // The primary saw only the first request's attempt. + expect(primarySeen.every((o) => o.temperature === 1)).toBe(true); + }); +}); diff --git a/src/ipc/utils/fallback_ai_model.ts b/src/ipc/utils/fallback_ai_model.ts index 46815fe7c4..c845c05a39 100644 --- a/src/ipc/utils/fallback_ai_model.ts +++ b/src/ipc/utils/fallback_ai_model.ts @@ -10,8 +10,29 @@ import { DyadError, DyadErrorKind } from "@/errors/dyad_error"; const logger = log.scope("fallback_model"); // Types + +/** + * The model-derived subset of call options — what a model should receive + * because of what it is, as opposed to what the request is. Everything else + * (prompt, tools, headers, request metadata) passes through unchanged. + */ +export interface FallbackModelCallOptions { + temperature?: number; + maxOutputTokens?: number; + providerOptions?: Record; +} + interface FallbackSettings { models: Array; + /** + * Per-model call-option overrides, parallel to `models`. The caller's + * options are computed for the PRIMARY selection and encode that model's + * constraints; an entry here expresses what the model at the same index + * would have received had it been selected as primary. Models without an + * entry get the conservative default on failover: `temperature` stripped + * (valid on every provider), everything else forwarded. + */ + modelCallOptions?: Array; } interface RetryState { @@ -177,6 +198,59 @@ class FallbackModel implements LanguageModelV3 { return model; } + /** + * Call options are resolved for the PRIMARY model before the request is made + * (e.g. `getTemperature(settings.selectedModel)`), so they encode that + * model's constraints, not the fallback's. Forwarding them verbatim across a + * provider switch produces hard 400s — observed: a gpt-5.6 stream error + * failed over to an Anthropic thinking model, which rejected the forwarded + * `temperature` ("`temperature` may only be set to 1 when thinking is + * enabled"), converting a recoverable blip into a fatal stream error. + * + * When calling any model other than the primary, apply that model's + * `modelCallOptions` entry — the options it would have received as the + * primary selection, computed at chain-build time where catalog access + * exists. Without an entry, fall back to stripping `temperature` (an absent + * temperature is valid on every provider). Applies to sticky-index first + * attempts too — after a failover, `currentModelIndex` stays non-zero for + * `modelResetInterval`, so a fresh request's first call can already target a + * fallback model. + */ + private optionsForCurrentModel( + options: LanguageModelV3CallOptions, + ): LanguageModelV3CallOptions { + if (this.currentModelIndex === 0) return options; + const overrides = this.settings.modelCallOptions?.[this.currentModelIndex]; + if (!overrides) { + // No per-model knowledge: strip temperature (an absent temperature is + // valid on every provider) and forward the rest. + if (options.temperature === undefined) return options; + const { temperature: _dropped, ...rest } = options; + return rest; + } + // Rebuild the model-derived subset as if this model had been primary. + // temperature is REPLACED (undefined in the overrides means "unset", not + // "keep the primary's"); maxOutputTokens falls back to the caller's when + // the catalog had none; providerOptions merge at the provider-family key — + // request-scoped keys (e.g. dyad-engine) pass through, the fallback's + // family entry is added, and a stale foreign family key is inert because + // providers only read their own key. + const { temperature: _replaced, ...rest } = options; + return { + ...rest, + ...(overrides.temperature !== undefined + ? { temperature: overrides.temperature } + : {}), + ...(overrides.maxOutputTokens !== undefined + ? { maxOutputTokens: overrides.maxOutputTokens } + : {}), + providerOptions: { + ...options.providerOptions, + ...(overrides.providerOptions ?? {}), + } as LanguageModelV3CallOptions["providerOptions"], + }; + } + private checkAndResetModel(): void { // Only reset if we're not currently in a retry cycle if (this.isRetrying) return; @@ -269,7 +343,9 @@ class FallbackModel implements LanguageModelV3 { this.checkAndResetModel(); return this.retry(async (retryState) => { - const result = await this.getUnderlyingModel().doStream(options); + const result = await this.getUnderlyingModel().doStream( + this.optionsForCurrentModel(options), + ); // Create a wrapped stream that handles errors gracefully const wrappedStream = this.createWrappedStream( @@ -380,7 +456,7 @@ class FallbackModel implements LanguageModelV3 { try { const nextResult = await fallbackModel .getUnderlyingModel() - .doStream(options); + .doStream(fallbackModel.optionsForCurrentModel(options)); await processStream(nextResult.stream); } catch (nextError) { controller.error(nextError); diff --git a/src/ipc/utils/get_model_client.test.ts b/src/ipc/utils/get_model_client.test.ts index 739b476a49..14327890f5 100644 --- a/src/ipc/utils/get_model_client.test.ts +++ b/src/ipc/utils/get_model_client.test.ts @@ -32,6 +32,11 @@ vi.mock("./model_effort", () => ({ })); vi.mock("../shared/language_model_helpers", () => ({ + // The auto chain now computes each fallback model's own call options + // (temperature/maxOutputTokens) via findLanguageModel -> getLanguageModels. + // An empty catalog means "no per-model data", which exercises the + // conservative path without inventing model entries these tests don't need. + getLanguageModels: vi.fn(async () => []), getLanguageModelProviders: vi.fn(async () => [ { id: "auto", diff --git a/src/ipc/utils/get_model_client.ts b/src/ipc/utils/get_model_client.ts index 55feedcaae..87b6fa618c 100644 --- a/src/ipc/utils/get_model_client.ts +++ b/src/ipc/utils/get_model_client.ts @@ -16,6 +16,8 @@ import type { AzureProviderSetting, } from "../../lib/schemas"; import { getEnvVar } from "./read_env"; +import { getModelScopedProviderOptions } from "./provider_options"; +import { getMaxTokens, getTemperature } from "./token_utils"; import log from "electron-log"; import { FREE_OPENROUTER_MODEL_NAMES } from "../shared/language_model_constants"; import { getLanguageModelProviders } from "../shared/language_model_helpers"; @@ -312,7 +314,7 @@ async function getProModelClient({ model.name === "auto" ) { const providers = await getLanguageModelProviders(); - const fallbackModels = await Promise.all( + const fallbackEntries = await Promise.all( AUTO_DYAD_PRO_MODEL_ALIASES.map(async (aliasId) => { const resolvedModel = await resolveBuiltinModelAlias(aliasId); if (!resolvedModel || resolvedModel.apiName.endsWith(":free")) { @@ -326,28 +328,51 @@ async function getProModelClient({ resolvedProvider?.gatewayPrefix || "" }${resolvedModel.apiName}`; - if (resolvedModel.providerId === "openai") { - return provider.responses(resolvedModel.apiName, { - providerId: resolvedModel.providerId, - }); - } - - if (resolvedModel.providerId === "anthropic") { - return provider.anthropic(resolvedModelId, { - providerId: resolvedModel.providerId, - }); - } - - return provider(resolvedModelId, { - providerId: resolvedModel.providerId, - }); + const instance = + resolvedModel.providerId === "openai" + ? provider.responses(resolvedModel.apiName, { + providerId: resolvedModel.providerId, + }) + : resolvedModel.providerId === "anthropic" + ? provider.anthropic(resolvedModelId, { + providerId: resolvedModel.providerId, + }) + : provider(resolvedModelId, { + providerId: resolvedModel.providerId, + }); + + // The stream's call options are computed for the PRIMARY selection, so + // give each chain entry the options it would have received had IT been + // selected: its own temperature and output cap from the catalog, and + // its provider family's thinking/reasoning options at the user's + // chosen effort. Without this, a cross-provider failover ran with the + // primary's options — an Anthropic fallback got no adaptive-thinking + // config plus a temperature that is invalid without it (hard 400). + const chainModelSelection = { + provider: resolvedModel.providerId, + name: resolvedModel.apiName, + }; + const [temperature, maxOutputTokens] = await Promise.all([ + getTemperature(chainModelSelection), + getMaxTokens(chainModelSelection), + ]); + return { + model: instance, + callOptions: { + temperature, + maxOutputTokens, + providerOptions: getModelScopedProviderOptions({ + providerId: resolvedModel.providerId, + modelName: resolvedModel.apiName, + modelSelection: model, + }), + }, + }; }), ); - const validModels = fallbackModels.filter( - (candidate) => candidate !== null, - ); - if (validModels.length === 0) { + const validEntries = fallbackEntries.filter((entry) => entry !== null); + if (validEntries.length === 0) { throw new DyadError( "No auto-mode models could be resolved from the catalog", DyadErrorKind.External, @@ -359,7 +384,8 @@ async function getProModelClient({ // because GPT-5* models need to use responses API to get // full functionality (e.g. thinking summaries). model: createFallback({ - models: validModels, + models: validEntries.map((entry) => entry.model), + modelCallOptions: validEntries.map((entry) => entry.callOptions), }), // Using openAI as the default provider. // TODO: we should remove this and rely on the provider id passed into the provider(). diff --git a/src/ipc/utils/provider_options.test.ts b/src/ipc/utils/provider_options.test.ts index ac5da335ec..8d499dc44c 100644 --- a/src/ipc/utils/provider_options.test.ts +++ b/src/ipc/utils/provider_options.test.ts @@ -1,7 +1,10 @@ import { describe, expect, it } from "vitest"; import type { ModelSelection, UserSettings } from "@/lib/schemas"; -import { getProviderOptions } from "./provider_options"; +import { + getModelScopedProviderOptions, + getProviderOptions, +} from "./provider_options"; const settingsFor = (provider: string, name: string, effortLevel: string) => ({ @@ -86,3 +89,85 @@ describe("getProviderOptions model effort", () => { ).toEqual({ reasoningEffort: "low" }); }); }); + +describe("getModelScopedProviderOptions", () => { + const selection = { + provider: "auto", + name: "auto", + effortLevel: "medium", + } as unknown as ModelSelection; + + it("matches getProviderOptions' family branch for each provider", () => { + // The two functions must stay in lockstep: this asserts the model-scoped + // slice equals what getProviderOptions would emit for the same family, + // so drift in either shows up here. + const anthropicScoped = getModelScopedProviderOptions({ + providerId: "anthropic", + modelName: "claude-opus-4-8", + modelSelection: selection, + }); + const anthropicFull = optionsFor( + settingsFor("anthropic", "claude-opus-4-8", "medium"), + "anthropic", + undefined, + selection, + ); + expect(anthropicScoped.anthropic).toEqual(anthropicFull.anthropic); + + const openaiScoped = getModelScopedProviderOptions({ + providerId: "openai", + modelName: "gpt-5.6-sol", + modelSelection: selection, + }); + const openaiFull = optionsFor( + settingsFor("openai", "gpt-5.6-sol", "medium"), + "openai", + undefined, + selection, + ); + expect(openaiScoped.openai).toEqual(openaiFull.openai); + + const googleScoped = getModelScopedProviderOptions({ + providerId: "google", + modelName: "gemini-3-flash-preview", + modelSelection: selection, + }); + const googleFull = optionsFor( + settingsFor("google", "gemini-3-flash-preview", "medium"), + "google", + undefined, + { ...selection, name: "gemini-3-flash-preview" } as ModelSelection, + ); + expect(googleScoped.google).toEqual(googleFull.google); + }); + + it("gives the anthropic family adaptive thinking (what makes temperature legal)", () => { + const scoped = getModelScopedProviderOptions({ + providerId: "anthropic", + modelName: "claude-opus-4-8", + modelSelection: selection, + }); + expect(scoped.anthropic.thinking).toEqual({ + type: "adaptive", + display: "summarized", + }); + expect(scoped.anthropic.effort).toBe("medium"); + }); + + it("returns nothing for unknown families and non-thinking gemini variants", () => { + expect( + getModelScopedProviderOptions({ + providerId: "xai", + modelName: "grok-4.6", + modelSelection: selection, + }), + ).toEqual({}); + expect( + getModelScopedProviderOptions({ + providerId: "google", + modelName: "gemini-2.5-flash-lite", + modelSelection: selection, + }), + ).toEqual({}); + }); +}); diff --git a/src/ipc/utils/provider_options.ts b/src/ipc/utils/provider_options.ts index b33cb75cac..7d9e0f24dd 100644 --- a/src/ipc/utils/provider_options.ts +++ b/src/ipc/utils/provider_options.ts @@ -27,6 +27,74 @@ export interface GetProviderOptionsParams { modelSelection: ModelSelection; } +/** + * The provider-FAMILY slice of {@link getProviderOptions}: the options a model + * needs because of what it is — thinking/reasoning configuration — independent + * of the request (dyad ids, files, codebase context). + * + * Exists for the auto-mode fallback chain: call options are computed once for + * the primary selection, so a mid-stream failover to another provider's model + * previously ran with the PRIMARY's family options — an Anthropic fallback + * received no `providerOptions.anthropic` at all (no adaptive thinking), and + * the primary's temperature on top of that is a hard 400. Each chain entry gets + * the family options it would have received as the primary selection. + * + * Kept in lockstep with the family branches of getProviderOptions below — + * update both when a provider's thinking config changes. + */ +export function getModelScopedProviderOptions({ + providerId, + modelName, + modelSelection, +}: { + providerId: string; + modelName: string; + modelSelection: ModelSelection; +}): Record { + if (providerId === "openai") { + return { + openai: { + reasoningSummary: "auto", + reasoningEffort: getModelEffort( + modelSelection, + ) as OpenAIResponsesProviderOptions["reasoningEffort"], + } satisfies OpenAIResponsesProviderOptions, + }; + } + if (providerId === "anthropic") { + return { anthropic: getAnthropicProviderOptions(modelSelection) }; + } + if (providerId === "google" || providerId === "vertex") { + const isGeminiModel = modelName.startsWith("gemini"); + const isFlashLite = modelName.includes("flash-lite"); + const isPartnerModel = modelName.includes("/"); + if (!isGeminiModel || isFlashLite || isPartnerModel) return {}; + const effortLevel = getModelEffort(modelSelection); + const isKnownGeminiEffort = ["minimal", "low", "medium", "high"].includes( + effortLevel, + ); + return { + google: { + thinkingConfig: { + includeThoughts: true, + ...(modelName.startsWith("gemini-3") && isKnownGeminiEffort + ? { + thinkingLevel: effortLevel as + | "minimal" + | "low" + | "medium" + | "high", + } + : isKnownGeminiEffort + ? { thinkingBudget: getGeminiThinkingBudgetTokens(effortLevel) } + : {}), + }, + } satisfies GoogleGenerativeAIProviderOptions, + }; + } + return {}; +} + /** * Builds provider options for the AI SDK streamText call. * Handles provider-specific configuration including thinking configs for Google/Vertex/Anthropic.