Tag models with vision capabilities and describe images for text-only models

src/components/VisionFallbackSwitch.tsx

@@ -0,0 +1,24 @@
+import { useSettings } from "@/hooks/useSettings";
+import { Label } from "@/components/ui/label";
+import { Switch } from "@/components/ui/switch";
+
+export function VisionFallbackSwitch() {
+  const { settings, updateSettings } = useSettings();
+  const enabled = settings?.enableVisionFallback ?? false;
+
+  return (
+    <div className="flex items-center space-x-2">
+      <Switch
+        id="vision-fallback"
+        aria-label="Describe images for text-only models"
+        checked={enabled}
+        onCheckedChange={() => {
+          updateSettings({ enableVisionFallback: !enabled });
+        }}
+      />
+      <Label htmlFor="vision-fallback">
+        Describe images for text-only models
+      </Label>
+    </div>
+  );
+}

src/ipc/handlers/__tests__/vision_fallback.integration.test.tsx

@@ -0,0 +1,208 @@
+// Covers the vision fallback end to end: a model the catalog marks
+// `supportsVision: false` must never receive raw image parts, and must instead
+// receive a text description in its place.
+//
+// The catalog fixture in testing/fake-llm-server/index.ts supplies the flag
+// here (the harness points the catalog fetch at the fake server), not
+// MODEL_OPTIONS.
+import { cleanup, screen } from "@testing-library/react";
+import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
+
+import {
+  setupHybridChatHarness,
+  type HybridChatHarness,
+} from "@/testing/hybrid_chat_harness";
+import { h } from "@/testing/hybrid.setup";
+
+// 1x1 transparent PNG.
+const PNG_BASE64 =
+  "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==";
+
+function pngBytes(): Uint8Array<ArrayBuffer> {
+  const decoded = Buffer.from(PNG_BASE64, "base64");
+  // Copy into a plain ArrayBuffer: Buffer's backing store is typed as
+  // ArrayBufferLike, which is not assignable to BlobPart.
+  const bytes = new Uint8Array(new ArrayBuffer(decoded.byteLength));
+  bytes.set(decoded);
+  return bytes;
+}
+
+type DumpMessage = { content?: unknown };
+type DumpBody = { input?: DumpMessage[]; messages?: DumpMessage[] };
+
+/** Counts structured image parts across every message in the dumped request. */
+function imagePartCount(parsedBody: DumpBody): number {
+  const messages = parsedBody?.input ?? parsedBody?.messages ?? [];
+  return messages
+    .flatMap((message) =>
+      Array.isArray(message?.content)
+        ? (message.content as { type?: string }[])
+        : [],
+    )
+    .filter((part) => part?.type === "image" || part?.type === "image_url")
+    .length;
+}
+
+const PRO_SETTINGS = {
+  isTestMode: true,
+  enableDyadPro: true,
+  providerSettings: { auto: { apiKey: { value: "testdyadkey" } } },
+};
+
+describe("vision fallback for a non-vision model (integration)", () => {
+  let harness: HybridChatHarness;
+
+  beforeAll(async () => {
+    harness = await setupHybridChatHarness({
+      electronMock: h,
+      engine: true,
+      // Marked `supportsVision: false` in the fake catalog fixture.
+      selectedModel: { provider: "openai", name: "gpt-5.2-no-vision" },
+
+      settings: { ...PRO_SETTINGS, enableVisionFallback: true },
+    });
+  }, 60_000);
+
+  afterEach(() => {
+    cleanup();
+  });
+
+  afterAll(async () => {
+    await harness?.dispose();
+  });
+
+  it("describes the image as text and sends no image part", async () => {
+    const chatId = await harness.createChat();
+    harness.mount({ chatId });
+
+    harness.setChatAttachments([
+      {
+        name: "mockup.png",
+        content: pngBytes(),
+        mimeType: "image/png",
+        type: "chat-context",
+      },
+    ]);
+    await screen.findByText("mockup.png");
+
+    const streamEnd = harness.waitForNextStreamEnd(chatId);
+    const { send } = await harness.typeInChat("[dump]", { chatId });
+    send();
+    await streamEnd;
+
+    // The turn must complete rather than surfacing a raw provider error.
+    expect(
+      harness.bridge.sentEvents.filter(
+        (event) => event.channel === "chat:response:error",
+      ),
+    ).toHaveLength(0);
+
+    const req = harness.getServerDump({ type: "request" });
+
+    // No raw image part reached the non-vision model...
+    expect(imagePartCount(req.parsed.body)).toBe(0);
+    // ...and a description took its place.
+    expect(req.text).toContain("dyad-image-description");
+    // The "analyze the image" system block must not be emitted when no image
+    // is actually sent.
+    expect(req.text).not.toContain("# Image Analysis Instructions");
+  }, 60_000);
+});
+
+describe("vision fallback off (the default) (integration)", () => {
+  let harness: HybridChatHarness;
+
+  beforeAll(async () => {
+    harness = await setupHybridChatHarness({
+      electronMock: h,
+      engine: true,
+      selectedModel: { provider: "openai", name: "gpt-5.2-no-vision" },
+      // Explicit, though this is also what an untouched install does.
+      settings: { ...PRO_SETTINGS, enableVisionFallback: false },
+    });
+  }, 60_000);
+
+  afterEach(() => {
+    cleanup();
+  });
+
+  afterAll(async () => {
+    await harness?.dispose();
+  });
+
+  it("omits the image without telling the user to switch models", async () => {
+    const chatId = await harness.createChat();
+    harness.mount({ chatId });
+
+    harness.setChatAttachments([
+      {
+        name: "mockup.png",
+        content: pngBytes(),
+        mimeType: "image/png",
+        type: "chat-context",
+      },
+    ]);
+    await screen.findByText("mockup.png");
+
+    const streamEnd = harness.waitForNextStreamEnd(chatId);
+    const { send } = await harness.typeInChat("[dump]", { chatId });
+    send();
+    await streamEnd;
+
+    const req = harness.getServerDump({ type: "request" });
+
+    // Opting out must not send the image anywhere.
+    expect(imagePartCount(req.parsed.body)).toBe(0);
+    // The model is told the images were dropped by configuration...
+    expect(req.text).toContain("off in Settings");
+    // ...and must not be told to nag the user about a missing capability.
+    expect(req.text).not.toContain("no vision-capable model is available");
+  }, 60_000);
+});
+
+describe("vision-capable model still receives images (integration)", () => {
+  let harness: HybridChatHarness;
+
+  beforeAll(async () => {
+    harness = await setupHybridChatHarness({
+      electronMock: h,
+      engine: true,
+      // Not marked in the fake catalog, so it is treated as vision-capable.
+      selectedModel: { provider: "openai", name: "gpt-5.2" },
+      settings: PRO_SETTINGS,
+    });
+  }, 60_000);
+
+  afterEach(() => {
+    cleanup();
+  });
+
+  afterAll(async () => {
+    await harness?.dispose();
+  });
+
+  it("sends the image part and injects no description", async () => {
+    const chatId = await harness.createChat();
+    harness.mount({ chatId });
+
+    harness.setChatAttachments([
+      {
+        name: "mockup.png",
+        content: pngBytes(),
+        mimeType: "image/png",
+        type: "chat-context",
+      },
+    ]);
+    await screen.findByText("mockup.png");
+
+    const streamEnd = harness.waitForNextStreamEnd(chatId);
+    const { send } = await harness.typeInChat("[dump]", { chatId });
+    send();
+    await streamEnd;
+
+    const req = harness.getServerDump({ type: "request" });
+
+    expect(imagePartCount(req.parsed.body)).toBeGreaterThan(0);
+    expect(req.text).not.toContain("dyad-image-description");
+  }, 60_000);
+});

src/ipc/handlers/chat_stream_handlers.test.ts

@@ -10,6 +10,8 @@ import {
 import { processFullResponseActions } from "@/ipc/processors/response_processor";
 import {
   addTrackedValue,
+  isImageFormatError,
+  isImageInputUnsupportedError,
   removeDyadTags,
   removeTrackedValue,
   setPartialResponseForStream,
@@ -1501,3 +1503,86 @@ Some text after the unclosed tag`;
     expect(result).toBe(false);
   });
 });
+
+describe("isImageInputUnsupportedError", () => {
+  const capabilityErrors = [
+    "Invalid content type. image_url is only supported by certain models.",
+    "This model does not support images",
+    "This model cannot accept images.",
+    "This model does not support image input.",
+    "This model doesn't support image input",
+    "Images are not supported by this model",
+    "multimodal input is disabled for this model",
+  ];
+
+  const formatErrors = [
+    "Unsupported image media type: image/tiff",
+    "Unsupported image format",
+    "Invalid content type: image/tiff",
+    "Unsupported content type: image/heic",
+    "Invalid image_url: could not decode the image data",
+    "Error while downloading image_url: failed to fetch the image",
+    "invalid base64 payload for image_url",
+    "corrupted image attachment",
+  ];
+
+  // `image_url` names the request field, so it shows up in errors that have
+  // nothing to do with vision support. Advising a model switch here hides the
+  // real cause, which the user could actually act on.
+  const imageUrlNonCapabilityErrors = [
+    "Invalid image_url: URL must use http or https",
+    "unsupported URL scheme for image_url",
+    "image_url is not accessible",
+    "timed out retrieving image_url",
+  ];
+
+  const unrelatedErrors = [
+    "rate limit exceeded",
+    "context length exceeded",
+    "invalid api key",
+  ];
+
+  it.each(capabilityErrors)("matches the capability error %j", (message) => {
+    expect(isImageInputUnsupportedError(message)).toBe(true);
+    expect(isImageFormatError(message)).toBe(false);
+  });
+
+  it.each(formatErrors)("does not match the format error %j", (message) => {
+    expect(isImageInputUnsupportedError(message)).toBe(false);
+    expect(isImageFormatError(message)).toBe(true);
+  });
+
+  it.each(imageUrlNonCapabilityErrors)(
+    "does not claim missing vision for the image_url error %j",
+    (message) => {
+      expect(isImageInputUnsupportedError(message)).toBe(false);
+    },
+  );
+
+  it("still matches a capability claim made about image_url itself", () => {
+    const message =
+      "Invalid content type. image_url is only supported by certain models.";
+    expect(isImageInputUnsupportedError(message)).toBe(true);
+    expect(isImageFormatError(message)).toBe(false);
+  });
+
+  it.each(unrelatedErrors)(
+    "does not match the unrelated error %j",
+    (message) => {
+      expect(isImageInputUnsupportedError(message)).toBe(false);
+      expect(isImageFormatError(message)).toBe(false);
+    },
+  );
+
+  it("reports an unknown wording as neither, so onError logs it as a gap", () => {
+    const message = "the selected model rejected the attached picture";
+    expect(isImageInputUnsupportedError(message)).toBe(false);
+    expect(isImageFormatError(message)).toBe(false);
+  });
+
+  it("prefers the format guard when a message looks like both", () => {
+    const message = "Unsupported image media type image/tiff for image_url";
+    expect(isImageFormatError(message)).toBe(true);
+    expect(isImageInputUnsupportedError(message)).toBe(false);
+  });
+});