From 90aabbac9c3dd55f1a03209fd162bdbe54aa94a0 Mon Sep 17 00:00:00 2001 From: roboomp Date: Mon, 6 Jul 2026 14:25:59 +0000 Subject: [PATCH] fix(coding-agent): detected llama.cpp model vision metadata - Honored per-model architecture.input_modalities from llama.cpp /v1/models during discovery and selected model refresh. - Added regression coverage for full refresh and cached selected-model metadata refresh. - Updated the coding-agent changelog. Fixes #4719 --- packages/coding-agent/CHANGELOG.md | 4 + .../src/config/model-discovery.ts | 20 +++- .../coding-agent/test/model-discovery.test.ts | 99 +++++++++++++++++++ 3 files changed, 121 insertions(+), 2 deletions(-) diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index ec2a2142b53..849d46f4f04 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Fixed + +- Fixed llama.cpp model discovery to honor per-model `architecture.input_modalities` from `/v1/models`, so router presets that advertise image input are no longer treated as text-only ([#4719](https://github.com/can1357/oh-my-pi/issues/4719)). + ## [16.3.10] - 2026-07-06 ### Changed diff --git a/packages/coding-agent/src/config/model-discovery.ts b/packages/coding-agent/src/config/model-discovery.ts index 61554be0225..34a9715df69 100644 --- a/packages/coding-agent/src/config/model-discovery.ts +++ b/packages/coding-agent/src/config/model-discovery.ts @@ -158,6 +158,7 @@ type LlamaCppDiscoveredModelRuntimeMetadata = { type LlamaCppModelListEntry = { id: string; + input?: ("text" | "image")[]; runtimeContextWindow?: number; /** * `--ctx-size` extracted from the entry's `status.args` (rendered CLI arg @@ -276,6 +277,20 @@ function extractLlamaCppModelContextWindows( }; } +function extractLlamaCppModelInputCapabilities(item: Record): ("text" | "image")[] | undefined { + const architecture = item.architecture; + if (!isRecord(architecture) || !Array.isArray(architecture.input_modalities)) { + return undefined; + } + const modalities = new Set(); + for (const modality of architecture.input_modalities) { + if (typeof modality === "string") { + modalities.add(modality.toLowerCase()); + } + } + return modalities.has("image") ? ["text", "image"] : ["text"]; +} + function parseLlamaCppModelList(payload: unknown): LlamaCppModelListEntry[] { if (!isRecord(payload) || !Array.isArray(payload.data)) { return []; @@ -287,6 +302,7 @@ function parseLlamaCppModelList(payload: unknown): LlamaCppModelListEntry[] { return [ { id: item.id, + input: extractLlamaCppModelInputCapabilities(item), ...extractLlamaCppModelContextWindows(item), configuredContextWindow: extractLlamaCppConfiguredContextWindow(item), }, @@ -545,7 +561,7 @@ export async function discoverLlamaCppModels( provider: providerConfig.provider, baseUrl, reasoning: false, - input: serverMetadata?.input ?? ["text"], + input: item.input ?? serverMetadata?.input ?? ["text"], imageInputDecoder: "stb", cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, contextWindow, @@ -595,7 +611,7 @@ export async function discoverLlamaCppModelRuntimeMetadata( entry.configuredContextWindow ?? serverMetadata?.contextWindow ?? entry.trainingContextWindow; - const input = serverMetadata?.input; + const input = entry.input ?? serverMetadata?.input; if (contextWindow === undefined) { return input === undefined ? undefined : { input }; } diff --git a/packages/coding-agent/test/model-discovery.test.ts b/packages/coding-agent/test/model-discovery.test.ts index a982a485066..dd7d89913bb 100644 --- a/packages/coding-agent/test/model-discovery.test.ts +++ b/packages/coding-agent/test/model-discovery.test.ts @@ -766,6 +766,41 @@ describe("ModelRegistry runtime discovery", () => { expect(llama?.input).toEqual(["text", "image"]); }); + test("llama.cpp discovery marks per-model architecture image modalities as vision-capable", async () => { + const fetchMock: FetchImpl = async input => { + const url = String(input); + if (url === "http://127.0.0.1:8080/models") { + return new Response( + JSON.stringify({ + data: [ + { + id: "q51q41_mtp_30tps_120k", + architecture: { + input_modalities: ["text", "image"], + output_modalities: ["text"], + }, + meta: { n_ctx: 123904 }, + }, + ], + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ); + } + if (url === "http://127.0.0.1:8080/props") { + return new Response(JSON.stringify({ default_generation_settings: { n_ctx: 123904 } }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + } + throw new Error(`Unexpected URL: ${url}`); + }; + const registry = new ModelRegistry(authStorage, modelsJsonPath, { fetch: fetchMock }); + await registry.refresh(); + const llama = registry.find("llama.cpp", "q51q41_mtp_30tps_120k"); + expect(llama?.contextWindow).toBe(123904); + expect(llama?.input).toEqual(["text", "image"]); + }); + test("llama.cpp discovery ignores positive props defaults as per-request limits, not hard caps", async () => { const fetchMock: FetchImpl = async input => { const url = String(input); @@ -1108,6 +1143,70 @@ describe("ModelRegistry runtime discovery", () => { expect(registry.find("llama.cpp", "vision-model")?.input).toEqual(["text", "image"]); }); + test("llama.cpp selected model refresh reads image capability from per-model architecture", async () => { + writeModelCache( + "llama.cpp", + Date.now(), + [ + buildModel({ + id: "router-vision-model", + name: "router-vision-model", + provider: "llama.cpp", + api: "openai-responses", + baseUrl: "http://127.0.0.1:8080", + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 128000, + maxTokens: 32768, + }), + ], + true, + "", + cacheDbPath, + ); + const fetchMock: FetchImpl = async input => { + const url = String(input); + if (url === "http://127.0.0.1:8080/models") { + return new Response( + JSON.stringify({ + data: [ + { + id: "router-vision-model", + architecture: { + input_modalities: ["text", "image"], + output_modalities: ["text"], + }, + meta: { n_ctx: 239104 }, + }, + ], + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ); + } + if (url === "http://127.0.0.1:8080/props") { + return new Response( + JSON.stringify({ + default_generation_settings: { + n_ctx: 239104, + params: { max_tokens: -1, n_predict: -1 }, + }, + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ); + } + throw new Error(`Unexpected URL: ${url}`); + }; + const registry = new ModelRegistry(authStorage, modelsJsonPath, { fetch: fetchMock }); + const stale = registry.find("llama.cpp", "router-vision-model"); + if (!stale) throw new Error("cached llama.cpp model missing"); + expect(stale.input).toEqual(["text"]); + const refreshed = await registry.refreshSelectedModelMetadata(stale); + expect(refreshed.contextWindow).toBe(239104); + expect(refreshed.input).toEqual(["text", "image"]); + expect(registry.find("llama.cpp", "router-vision-model")?.input).toEqual(["text", "image"]); + }); + test("llama.cpp selected model refresh leaves the cached model untouched when /models no longer lists it", async () => { writeModelCache( "llama.cpp",