From eac37728a8428992ecd6205ebf0a8b0e63bce4c1 Mon Sep 17 00:00:00 2001 From: Sarath S Menon Date: Thu, 8 Oct 2026 02:21:03 +0000 Subject: [PATCH 1/2] test(provider): reject screenshot uploads at V4 gateway --- .../opencode/test/provider/provider.test.ts | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/packages/opencode/test/provider/provider.test.ts b/packages/opencode/test/provider/provider.test.ts index c6c8fb20b..936b45aa1 100644 --- a/packages/opencode/test/provider/provider.test.ts +++ b/packages/opencode/test/provider/provider.test.ts @@ -2059,10 +2059,10 @@ it.effect("opencode loader keeps paid models when auth exists", () => ) it.instance( - "configuration-only OpenAI gateway wraps the selected Responses model", + "configuration-only OpenAI gateway keeps screenshots inline without uploads", Effect.gen(function* () { yield* remove("OPENAI_API_KEY") - const receipt = { uploads: 0, deletes: 0, bodies: [] as Array> } + const receipt = { uploads: 0, deletes: 0, capabilities: 0, bodies: [] as Array> } const server = yield* Effect.acquireRelease( Effect.sync(() => Bun.serve({ @@ -2073,8 +2073,10 @@ it.instance( receipt.deletes++ return Response.json({ deleted: true }) } - if (url.pathname.endsWith("/capability")) + if (url.pathname.endsWith("/capability")) { + receipt.capabilities++ return Response.json({ supported: true, scope: "owned-default-selector" }) + } if (url.pathname.endsWith("/image-files")) { receipt.uploads++ return Response.json({ file_id: "file-owned", signature: "proof", expires_at: Date.now() / 1000 + 3600 }) @@ -2112,7 +2114,6 @@ it.instance( const provider = yield* Provider.Service const model = yield* provider.getModel(ProviderV2.ID.openai, ModelV2.ID.make("gateway-fixture")) const language = yield* provider.getLanguage(model) - expect(typeof (language as unknown as { closeImageFiles?: unknown }).closeImageFiles).toBe("function") const screenshot = Buffer.from("owned screenshot fixture").toString("base64") for (const turn of [1, 2, 3]) yield* Effect.promise(() => @@ -2133,15 +2134,16 @@ it.instance( ], }), ) - expect(receipt.uploads).toBe(1) + expect(receipt.uploads).toBe(0) + expect(receipt.capabilities).toBe(0) expect(receipt.bodies).toHaveLength(3) for (const body of receipt.bodies) { const serialized = JSON.stringify(body) - expect(serialized).toContain("file-owned") - expect(serialized).not.toContain(screenshot) + expect(serialized).not.toContain("file-owned") + expect(serialized).not.toContain("file_id") + expect(serialized).toContain(`data:image/png;base64,${screenshot}`) expect(serialized).toContain("high") } - yield* Effect.promise(() => (language as unknown as { closeImageFiles(): Promise }).closeImageFiles()) - expect(receipt.deletes).toBe(1) + expect(receipt.deletes).toBe(0) }), ) From 757645b9474818eb672297cf9adf99ad7b60a269 Mon Sep 17 00:00:00 2001 From: Sarath S Menon Date: Thu, 8 Oct 2026 02:22:10 +0000 Subject: [PATCH 2/2] fix(provider): restore inline screenshots while retaining transport recovery --- packages/opencode/src/provider/image-files.ts | 640 ------------------ packages/opencode/src/provider/provider.ts | 52 +- .../provider/anthropic-image-files.test.ts | 241 ------- .../provider/image-files-lifecycle.test.ts | 547 --------------- .../test/provider/image-files.test.ts | 521 -------------- 5 files changed, 1 insertion(+), 2000 deletions(-) delete mode 100644 packages/opencode/src/provider/image-files.ts delete mode 100644 packages/opencode/test/provider/anthropic-image-files.test.ts delete mode 100644 packages/opencode/test/provider/image-files-lifecycle.test.ts delete mode 100644 packages/opencode/test/provider/image-files.test.ts diff --git a/packages/opencode/src/provider/image-files.ts b/packages/opencode/src/provider/image-files.ts deleted file mode 100644 index 21b41d09b..000000000 --- a/packages/opencode/src/provider/image-files.ts +++ /dev/null @@ -1,640 +0,0 @@ -import type { LanguageModelV3, LanguageModelV3CallOptions } from "@ai-sdk/provider" - -type Uploaded = { file_id: string; expires_at: number; signature: string } -type Options = { baseURL: string; apiKey: string; fetch?: typeof fetch; now?: () => number } -type Lease = { keys: Set; files: Set } -type RunCache = { - leases: Set - lock: Promise - expiresAt: number - chain: boolean - chainKeys: Set - owners: Set - cache: Map> - ready: Map - busy: boolean - cleanupQueued: boolean - cleaning?: Promise - refreshed: Set - retired: Uploaded[] - waiting: Array<() => void> - active: number - closed: boolean -} -const runCaches = new Map() - -export function withImageFiles(model: LanguageModelV3, options: Options) { - let state: RunCache = { - leases: new Set(), - lock: Promise.resolve(), - expiresAt: 0, - chain: false, - chainKeys: new Set(), - owners: new Set(), - cache: new Map(), - ready: new Map(), - busy: false, - cleanupQueued: false, - refreshed: new Set(), - retired: [], - waiting: [], - active: 0, - closed: false, - } - const owner = Symbol() - let closed = false - const now = options.now ?? Date.now - const endpoint = options.baseURL.replace(/\/$/, "") + "/image-files" - const send = options.fetch ?? fetch - let cacheID: string | undefined - let scope: string | undefined - let routeHeaders: Record = {} - const anthropic = /\/anthropic\/v1\/?$/.test(new URL(options.baseURL).pathname) - const enabled = - Boolean(options.apiKey) && - /^\/(?:_v4\/bcode-relay\/[^/]+\/)?api\/v4\/llm\/(?:openai|anthropic)\/v1\/?$/.test( - new URL(options.baseURL).pathname, - ) - - async function locked(call: () => Promise): Promise { - const prior = state.lock - const next = Promise.withResolvers() - state.lock = next.promise - await prior - state.busy = true - try { - return await call() - } finally { - state.busy = false - next.resolve() - } - } - - // Retry only explicit transient HTTP failures. A network failure after an upload - // has an unknown outcome, so blindly retrying it could create orphaned files. - async function request(url: string, init: RequestInit) { - for (let attempt = 0; ; attempt++) { - init.signal?.throwIfAborted() - const response = await send(url, init) - if (![429, 502, 503, 504].includes(response.status) || attempt === 2) return response - await response.body?.cancel() - await new Promise((resolve, reject) => { - const abort = () => { - clearTimeout(timer) - reject(init.signal?.reason) - } - const timer = setTimeout( - () => { - init.signal?.removeEventListener("abort", abort) - resolve() - }, - 100 * 2 ** attempt, - ) - init.signal?.addEventListener("abort", abort, { once: true }) - if (init.signal?.aborted) abort() - }) - } - } - - function identity(data: string | Uint8Array | URL, mediaType: string) { - const raw = data instanceof URL ? data.toString().split(",", 2)[1] : data - const bytes = typeof raw === "string" ? Buffer.from(raw, "base64") : raw - if (bytes.byteLength > 5 * 1024 * 1024) throw new Error("Screenshot exceeds upload limit") - return { bytes, key: `${mediaType}:${new Bun.CryptoHasher("sha256").update(bytes).digest("hex")}` } - } - - // One shared drain prevents concurrent wrappers from multiplying DELETE work. - // Its per-file timeout is independent of a caller's bounded foreground wait. - async function cleanup(signal?: AbortSignal, preserveRegistry = false, budget = { remaining: 1000 }) { - const forget = () => { - if ( - state.closed && - !state.cache.size && - !state.retired.length && - !state.leases.size && - cacheID && - runCaches.get(cacheID) === state - ) - runCaches.delete(cacheID) - } - const joined = Boolean(state.cleaning) - if (!state.cleaning) { - state.cleaning = (async () => { - for (;;) { - if (state.closed) { - for (const [key, file] of state.ready) { - if ([...state.leases].some((lease) => lease.keys.has(key))) continue - if (state.retired.length >= 300) break - state.cache.delete(key) - state.ready.delete(key) - state.refreshed.delete(key) - state.retired.push(file) - } - } - const pinned = new Set([...state.leases].flatMap((lease) => [...lease.files])) - const batch = state.retired - .filter((file) => !pinned.has(file.file_id) && (!state.chain || state.closed)) - .slice(0, 16) - if (!batch.length) return true - const removed = await Promise.all( - batch.map(async (file) => { - if (file.expires_at <= now() / 1000) return file - const response = await request( - `${endpoint}/${encodeURIComponent(file.file_id)}?model=${encodeURIComponent(model.modelId)}`, - { - method: "DELETE", - headers: { ...routeHeaders, "x-bu-image-scope": scope! }, - signal: AbortSignal.timeout(1000), - }, - ).catch(() => undefined) - await response?.body?.cancel().catch(() => {}) - return response?.ok || response?.status === 404 ? file : undefined - }), - ) - // Filter the current queue: other requests can retire files while a - // batch is in flight. Failed tombstones survive for an explicit retry. - const done = new Set(removed.filter((file) => file !== undefined)) - state.retired = state.retired.filter((file) => !done.has(file)) - if (done.size !== batch.length) return false - } - })().finally(() => { - state.cleaning = undefined - // A caller reopening this cache holds the lock and keeps its registry - // entry stable until it has either reopened it or reported backpressure. - if (!state.busy) forget() - }) - } - const started = performance.now() - const result = await new Promise((resolve) => { - const finish = (result?: boolean) => { - clearTimeout(timer) - signal?.removeEventListener("abort", abort) - resolve(result) - } - const abort = () => finish() - const timer = setTimeout(abort, Math.max(0, budget.remaining)) - signal?.addEventListener("abort", abort, { once: true }) - void state.cleaning!.then(finish, abort) - if (signal?.aborted) abort() - }) - budget.remaining = Math.max(0, budget.remaining - (performance.now() - started)) - // An explicit retry may have joined the tail of a failed drain. Use only - // its remaining budget to start one fresh attempt after that drain ends. - if (joined && result === false && budget.remaining > 0 && !signal?.aborted) - await cleanup(signal, preserveRegistry, budget) - if (!preserveRegistry) forget() - } - - async function prepare(params: LanguageModelV3CallOptions, lease: Lease): Promise { - if (!enabled) return params - const cleanupBudget = { remaining: 1000 } - const stateful = - !anthropic && - (typeof params.providerOptions?.openai?.previousResponseId === "string" || - params.providerOptions?.openai?.store === true) - if (stateful) { - state.chain = true - for (const key of state.cache.keys()) state.chainKeys.add(key) - } - const hasImages = params.prompt.some( - (message) => - (message.role === "user" && - message.content.some( - (part) => - part.type === "file" && - part.mediaType.startsWith("image/") && - (!(part.data instanceof URL) || part.data.protocol === "data:"), - )) || - (message.role === "tool" && - message.content.some( - (part) => - part.type === "tool-result" && - part.output.type === "content" && - part.output.value.some((item) => item.type === "image-data"), - )), - ) - if (!hasImages) return params - if (closed) throw new Error("Screenshot file cache is closed") - const previous = params.providerOptions?.openai?.previousResponseId - const legacy = params.prompt.some( - (message) => - message.role === "assistant" && - message.content.some( - (part) => - part.type === "reasoning" && - String(part.providerOptions?.openai?.reasoningEncryptedContent ?? "").startsWith("gAAAAA"), - ), - ) - const currentHeaders = { - Authorization: `Bearer ${options.apiKey}`, - ...(typeof previous === "string" ? { "x-bu-previous-response-id": previous } : {}), - ...(legacy ? { "x-bu-openai-reasoning": "1" } : {}), - } - const signal = AbortSignal.any([...(params.abortSignal ? [params.abortSignal] : []), AbortSignal.timeout(30_000)]) - const capability = await request(`${endpoint}/capability?model=${encodeURIComponent(model.modelId)}`, { - headers: currentHeaders, - signal, - }) - // Older gateways have no upload contract. Never fall back after selecting reference mode. - if (capability.status === 404 && !scope) return params - if (!capability.ok) throw new Error(`Screenshot capability failed (${capability.status})`) - const route = (await capability.json()) as { - supported: boolean - scope?: string - max_references?: number - proof_transport?: string - } - if (!route.supported && !scope) return params - if (!route.supported || !route.scope || (scope && scope !== route.scope)) - throw new Error("Screenshot route or account changed; start a new run") - if (closed) throw new Error("Screenshot file cache is closed") - scope = route.scope - cacheID = `${endpoint}:${model.modelId}:${scope}` - for (const [id, cached] of runCaches) { - if (cached.closed && !cached.leases.size && cached.expiresAt <= now() / 1000) runCaches.delete(id) - } - if (!runCaches.has(cacheID) && runCaches.size >= 128) throw new Error("Too many active screenshot caches") - state = runCaches.get(cacheID) ?? state - routeHeaders = currentHeaders - if (state.closed) - await locked(async () => { - if (!state.closed) return - await cleanup(signal, true, cleanupBudget) - if (state.cache.size || state.retired.length || state.leases.size) - throw new Error("Screenshot file cache is closing; retry cleanup first") - state.closed = false - state.chain = false - state.chainKeys.clear() - }) - if (closed) throw new Error("Screenshot file cache is closed") - if (stateful) { - state.chain = true - for (const key of state.cache.keys()) state.chainKeys.add(key) - } - state.owners.add(owner) - runCaches.set(cacheID, state) - routeHeaders = currentHeaders - const limit = route.proof_transport === "body" && route.max_references === 300 ? 300 : 64 - const keys = new Set() - for (const message of params.prompt) { - if (message.role === "user") - for (const part of message.content) { - if ( - part.type === "file" && - part.mediaType.startsWith("image/") && - (!(part.data instanceof URL) || part.data.protocol === "data:") - ) - keys.add(identity(part.data, part.mediaType).key) - } - if (message.role === "tool") - for (const part of message.content) { - if (part.type === "tool-result" && part.output.type === "content") - for (const item of part.output.value) { - if (item.type === "image-data") keys.add(identity(item.data, item.mediaType).key) - } - } - } - const cached = [...keys].every((key) => (state.ready.get(key)?.expires_at ?? 0) > now() / 1000 + 30) - const work = async () => { - signal.throwIfAborted() - if (closed || state.closed) throw new Error("Screenshot file cache is closed") - if (keys.size > limit) - throw new Error( - `Screenshot request needs ${keys.size} distinct images; gateway limit is ${limit}. Compact history before retrying`, - ) - const pinned = new Set([...state.chainKeys, ...[...state.leases].flatMap((item) => [...item.keys])]) - if (new Set([...pinned, ...keys]).size > 300) - throw new Error( - "Screenshot cache has 300 images in active requests or a stateful response chain; finish active requests or start a new chain", - ) - lease.keys = keys - state.leases.add(lease) - if (!cached) await cleanup(signal, false, cleanupBudget) - const missing = [...keys].filter((key) => !state.cache.has(key)).length - for (const key of state.cache.keys()) { - if (state.cache.size + missing <= 300) break - if (keys.has(key) || state.chainKeys.has(key) || [...state.leases].some((item) => item.keys.has(key))) continue - if (state.retired.length >= 300) - throw new Error("Screenshot cleanup backlog is full; retry cleanup before uploading") - state.retired.push(state.ready.get(key)!) - state.cache.delete(key) - state.ready.delete(key) - state.refreshed.delete(key) - } - if (state.cache.size + missing > 300) - throw new Error("Screenshot cache has 300 pinned images; retry after active requests finish") - // Provider-side previousResponseId history may still depend on omitted - // files. Keep explicit stateful chains until close; never assume a copy. - if (state.chain) for (const key of keys) state.chainKeys.add(key) - // Reclaim files newly retired by eviction, after the pre-eviction backlog pass. - if (!cached) await cleanup(signal, false, cleanupBudget) - const headers = { ...currentHeaders, "x-bu-image-scope": scope! } - const proofs: Record = {} - async function upload(data: string | Uint8Array | URL, mediaType: string) { - const { bytes, key } = identity(data, mediaType) - const ready = state.ready.get(key) - if (ready && ready.expires_at > now() / 1000 + 30) { - lease.files.add(ready.file_id) - proofs[ready.file_id] = { expires_at: ready.expires_at, signature: ready.signature } - return ready.file_id - } - if (state.active >= 2) await new Promise((resolve) => state.waiting.push(resolve)) - else state.active++ - try { - signal.throwIfAborted() - if (closed || state.closed) throw new Error("Screenshot file cache is closed") - let existing = state.cache.get(key) - if (existing) { - const file = await existing - if (state.cache.get(key) !== existing) existing = state.cache.get(key) - else if (file.expires_at <= now() / 1000 + 30) { - if (state.refreshed.has(key)) throw new Error("Screenshot reference expired again; start a new run") - if (state.retired.length >= 300) throw new Error("Screenshot cleanup backlog is full") - state.refreshed.add(key) - state.retired.push(file) - state.ready.delete(key) - existing = undefined - } - } - const pending = - existing ?? - (async () => { - const response = await request(`${endpoint}?model=${encodeURIComponent(model.modelId)}`, { - method: "POST", - headers: { ...headers, "content-type": mediaType }, - body: new Uint8Array(bytes), - signal, - }) - if (!response.ok) throw new Error(`Screenshot upload failed (${response.status})`) - const uploaded = (await response.json()) as Uploaded - if (!uploaded.file_id?.startsWith(anthropic ? "file_" : "file-")) - throw new Error("Invalid screenshot file ID") - if (!(uploaded.expires_at > now() / 1000 + 30)) throw new Error("Screenshot reference expired") - state.expiresAt = Math.max(state.expiresAt, uploaded.expires_at) - return uploaded - })() - state.cache.set(key, pending) - const uploaded = await pending.catch((error) => { - if (state.cache.get(key) === pending) { - state.cache.delete(key) - state.ready.delete(key) - if (!state.refreshed.has(key)) state.chainKeys.delete(key) - state.refreshed.delete(key) - } - throw error - }) - state.ready.set(key, uploaded) - if (uploaded.expires_at <= now() / 1000 + 30) throw new Error("Screenshot reference expired; start a new run") - signal.throwIfAborted() - if (closed || state.closed) throw new Error("Screenshot file cache is closed") - lease.files.add(uploaded.file_id) - proofs[uploaded.file_id] = { expires_at: uploaded.expires_at, signature: uploaded.signature } - return uploaded.file_id - } finally { - const next = state.waiting.shift() - if (next) next() - else state.active-- - } - } - const prompt = await settled( - params.prompt.map(async (message) => { - if (message.role === "tool") - return { - ...message, - content: await settled( - message.content.map(async (part) => { - if (part.type !== "tool-result" || part.output.type !== "content") return part - return { - ...part, - output: { - ...part.output, - value: await settled( - part.output.value.map(async (item) => { - if (item.type !== "image-data") return item - return { - type: "image-url" as const, - url: `bu-${anthropic ? "anthropic" : "openai"}-file:${await upload(item.data, item.mediaType)}`, - providerOptions: item.providerOptions, - } - }), - ), - }, - } - }), - ), - } - if (message.role !== "user") return message - return { - ...message, - content: await settled( - message.content.map(async (part) => { - if ( - part.type !== "file" || - !part.mediaType.startsWith("image/") || - (part.data instanceof URL && part.data.protocol !== "data:") - ) - return part - const id = await upload(part.data, part.mediaType) - return { ...part, data: anthropic ? new URL(`bu-anthropic-file:${id}`) : id } - }), - ), - } - }), - ) - const encoded = JSON.stringify(proofs) - if (route.proof_transport !== "body" && Buffer.byteLength(encoded) > 12 * 1024) - throw new Error("Screenshot proofs exceed legacy header limit; upgrade the gateway") - return { - ...params, - prompt, - headers: { - ...params.headers, - "x-bu-image-scope": scope!, - "x-bu-image-proofs": encoded, - ...(route.proof_transport === "body" ? { "x-bu-image-proof-transport": "body" } : {}), - }, - } - } - return cached ? work() : locked(work) - } - - async function release(lease: Lease) { - state.leases.delete(lease) - if (state.cleanupQueued) return - // Cache hits complete promptly, but the final lease may end during close. - // Coalesce a follow-up pass rather than relying on another future release. - if (state.busy) { - state.cleanupQueued = true - void locked(async () => { - state.cleanupQueued = false - await cleanup() - }).catch(() => {}) - return - } - await locked(() => cleanup()) - } - - async function invoke( - params: LanguageModelV3CallOptions, - lease: Lease, - call: (prepared: LanguageModelV3CallOptions) => PromiseLike, - ): Promise { - try { - return await call(await prepare(params, lease)) - } catch (error) { - const message = error instanceof Error ? error.message : "" - if (!/file.*(not found|does not exist|expired|deleted)/i.test(message)) throw error - await locked(async () => { - const entries = await settled([...state.cache].map(async ([key, pending]) => ({ key, file: await pending }))) - const missing = entries.filter(({ file }) => lease.files.has(file.file_id) && message.includes(file.file_id)) - if (!missing.length || missing.some(({ key }) => state.refreshed.has(key))) throw error - if (state.retired.length + missing.length > 300) throw error - for (const { key, file } of missing) { - state.refreshed.add(key) - state.retired.push(file) - state.cache.delete(key) - state.ready.delete(key) - } - }) - return await call(await prepare(params, lease)) - } - } - - return { - specificationVersion: model.specificationVersion, - provider: model.provider, - modelId: model.modelId, - supportedUrls: model.supportedUrls, - async doGenerate(params: LanguageModelV3CallOptions) { - const lease: Lease = { keys: new Set(), files: new Set() } - try { - return await invoke(params, lease, (prepared) => model.doGenerate(prepared)) - } finally { - await release(lease) - } - }, - async doStream(params: LanguageModelV3CallOptions) { - const lease: Lease = { keys: new Set(), files: new Set() } - try { - const result = await invoke(params, lease, (prepared) => model.doStream(prepared)) - const reader = result.stream.getReader() - let finished = false - const finish = async () => { - if (finished) return - finished = true - params.abortSignal?.removeEventListener("abort", abort) - await release(lease) - } - const abort = () => { - void reader - .cancel(params.abortSignal?.reason) - .finally(finish) - .catch(() => {}) - } - params.abortSignal?.addEventListener("abort", abort, { once: true }) - if (params.abortSignal?.aborted) abort() - return { - ...result, - stream: new ReadableStream({ - async pull(controller) { - try { - const item = await reader.read() - if (item.done) { - await finish() - controller.close() - return - } - controller.enqueue(item.value) - } catch (error) { - await finish() - controller.error(error) - } - }, - async cancel(reason) { - try { - await reader.cancel(reason) - } finally { - await finish() - } - }, - }), - } - } catch (error) { - await release(lease) - throw error - } - }, - async closeImageFiles() { - closed = true - await locked(async () => { - state.owners.delete(owner) - if (state.owners.size) return - state.closed = true - await cleanup() - }) - }, - } -} - -async function settled(items: Promise[]): Promise { - const results = await Promise.allSettled(items) - const failed = results.find((item) => item.status === "rejected") - if (failed?.status === "rejected") throw failed.reason - return results.map((item) => (item as PromiseFulfilledResult).value) -} - -// The pinned SDK lacks tool-result file IDs; only small placeholders enter its JSON serializer. -export function imageFileRequest(init?: RequestInit): RequestInit | undefined { - if (!init || !new Headers(init.headers).has("x-bu-image-scope") || typeof init.body !== "string") return init - const body = JSON.parse(init.body) - const headers = new Headers(init.headers) - if (headers.get("x-bu-image-proof-transport") === "body") { - const proofs = headers.get("x-bu-image-proofs") ?? "{}" - if (Buffer.byteLength(proofs) > 128 * 1024 || Object.keys(JSON.parse(proofs)).length > 300) - throw new Error("Screenshot proof body limit exceeded") - body._image_file_proofs = JSON.parse(proofs) - body._image_file_scope = headers.get("x-bu-image-scope") - headers.delete("x-bu-image-proofs") - } - headers.delete("x-bu-image-proof-transport") - if (Array.isArray(body.input)) - body.input = body.input.map((item: Record) => { - if (!item || typeof item !== "object") return item - const field = - item.type === "function_call_output" || item.type === "custom_tool_call_output" ? "output" : "content" - if (!Array.isArray(item[field])) return item - return { - ...item, - [field]: item[field].map((part: Record) => { - if ( - part?.type !== "input_image" || - typeof part.image_url !== "string" || - !part.image_url.startsWith("bu-openai-file:file-") - ) - return part - const { image_url, ...rest } = part - return { ...rest, file_id: image_url.slice("bu-openai-file:".length) } - }), - } - }) - if (Array.isArray(body.messages)) { - const rewrite = (content: Array>): Array> => - content.map((part) => { - if (part?.type === "tool_result" && Array.isArray(part.content)) - return { ...part, content: rewrite(part.content) } - const source = part?.source as { type?: string; url?: string } | undefined - if (part?.type !== "image" || source?.type !== "url" || !source.url?.startsWith("bu-anthropic-file:file_")) - return part - return { ...part, source: { type: "file", file_id: source.url.slice("bu-anthropic-file:".length) } } - }) - body.messages = body.messages.map((message: Record) => - Array.isArray(message.content) ? { ...message, content: rewrite(message.content) } : message, - ) - headers.set( - "anthropic-beta", - [...new Set([...(headers.get("anthropic-beta")?.split(",") ?? []), "files-api-2025-04-14"])].join(","), - ) - return { ...init, headers, body: JSON.stringify(body) } - } - return { ...init, headers, body: JSON.stringify(body) } -} diff --git a/packages/opencode/src/provider/provider.ts b/packages/opencode/src/provider/provider.ts index a7c719da0..73a8080d1 100644 --- a/packages/opencode/src/provider/provider.ts +++ b/packages/opencode/src/provider/provider.ts @@ -1,4 +1,3 @@ -import { imageFileRequest, withImageFiles } from "./image-files" import { LayerNode } from "@opencode-ai/core/effect/layer-node" import os from "os" import { ConfigV1 } from "@opencode-ai/core/v1/config/config" @@ -1159,7 +1158,6 @@ export interface Interface { } interface State { - imageFiles: Set<() => Promise> models: Map providers: Record catalog: Record @@ -1346,12 +1344,6 @@ const layer = Layer.effect( const providers: Record = {} as Record const languages = new Map() - const imageFiles = new Set<() => Promise>() - yield* Effect.addFinalizer(() => - Effect.promise(async () => { - await Promise.allSettled([...imageFiles].map((close) => close())) - }), - ) const modelLoaders: { [providerID: string]: CustomModelLoader } = {} @@ -1661,7 +1653,6 @@ const layer = Layer.effect( return { models: languages, - imageFiles, providers, catalog, sdk, @@ -1761,7 +1752,7 @@ const layer = Layer.effect( if (combined) opts.signal = combined const res = await fetchFn(input, { - ...imageFileRequest(opts), + ...opts, // @ts-ignore see here: https://github.com/oven-sh/bun/issues/16682 timeout: false, }).finally(() => headerTimeoutCtl?.clear()) @@ -1777,47 +1768,6 @@ const layer = Layer.effect( name: model.providerID, ...options, }) - if ( - model.api.npm === "@ai-sdk/openai" && - loaded.responses && - typeof options.apiKey === "string" && - typeof baseURL === "string" - ) { - const responses = loaded.responses.bind(loaded) - loaded.responses = (id: string) => { - const wrapped = withImageFiles(responses(id), { baseURL, apiKey: options.apiKey, fetch: options.fetch }) - s.imageFiles.add(wrapped.closeImageFiles) - return wrapped - } - // Config-only providers can select the SDK default without a custom loader. - // OpenAI defaults to Responses; wrap that selector as well. - const languageModel = loaded.languageModel.bind(loaded) - loaded.languageModel = (id: string) => { - const wrapped = withImageFiles(languageModel(id), { - baseURL, - apiKey: options.apiKey, - fetch: options.fetch, - }) - s.imageFiles.add(wrapped.closeImageFiles) - return wrapped - } - } - if ( - model.api.npm === "@ai-sdk/anthropic" && - typeof options.apiKey === "string" && - typeof baseURL === "string" - ) { - const languageModel = loaded.languageModel.bind(loaded) - loaded.languageModel = (id: string) => { - const wrapped = withImageFiles(languageModel(id), { - baseURL, - apiKey: options.apiKey, - fetch: options.fetch, - }) - s.imageFiles.add(wrapped.closeImageFiles) - return wrapped - } - } s.sdk.set(key, loaded) return loaded as SDK } diff --git a/packages/opencode/test/provider/anthropic-image-files.test.ts b/packages/opencode/test/provider/anthropic-image-files.test.ts deleted file mode 100644 index 4d98fdcff..000000000 --- a/packages/opencode/test/provider/anthropic-image-files.test.ts +++ /dev/null @@ -1,241 +0,0 @@ -import { expect, test } from "bun:test" -import { createAnthropic } from "@ai-sdk/anthropic" -import { generateText } from "ai" -import { SessionV1 } from "@opencode-ai/core/v1/session" -import type { Provider } from "../../src/provider/provider" -import { MessageV2 } from "../../src/session/message-v2" -import { imageFileRequest, withImageFiles } from "../../src/provider/image-files" - -const screenshot = Buffer.alloc(1024 * 1024, 42) -const history = [ - { - info: { - id: "msg-a", - parentID: "msg-u", - sessionID: "session", - role: "assistant", - modelID: "claude-test", - providerID: "anthropic", - time: { created: 0 }, - }, - parts: [ - { - id: "part", - sessionID: "session", - messageID: "msg-a", - type: "tool", - tool: "browser_execute", - callID: "call-1", - state: { - status: "completed", - input: { code: "screenshot()" }, - output: "Captured", - title: "Screenshot", - metadata: {}, - time: { start: 0, end: 1 }, - attachments: [ - { - id: "file", - sessionID: "session", - messageID: "msg-a", - type: "file", - mime: "image/png", - url: `data:image/png;base64,${screenshot.toString("base64")}`, - }, - ], - }, - }, - ], - }, -] as unknown as SessionV1.WithParts[] -const spec = { - id: "claude-test", - providerID: "anthropic", - api: { id: "claude-test", npm: "@ai-sdk/anthropic" }, - capabilities: { input: { image: true } }, -} as Provider.Model -function fixture(options: { relay?: boolean; supported?: boolean; missing?: number; uploadStatus?: number } = {}) { - const state = { - uploads: 0, - deletes: 0, - calls: 0, - scope: crypto.randomUUID() as string, - bodies: [] as Array<{ messages: Array<{ content: Array> }> }>, - headers: [] as Headers[], - } - const server = Bun.serve({ - port: 0, - async fetch(req) { - if (req.url.includes("/capability")) - return Response.json({ supported: options.supported ?? true, scope: state.scope }) - if (req.method === "DELETE") { - state.deletes++ - return Response.json({ deleted: true }) - } - if (req.url.includes("/image-files")) { - expect(req.headers.get("content-type")).toBe("image/png") - expect(Buffer.from(await req.arrayBuffer()).equals(screenshot)).toBe(true) - state.uploads++ - if (options.uploadStatus) return new Response("failed", { status: options.uploadStatus }) - return Response.json({ - file_id: `file_${state.uploads}`, - signature: "proof", - expires_at: Date.now() / 1000 + 3600, - }) - } - state.calls++ - state.bodies.push(await req.json()) - state.headers.push(req.headers) - if (state.calls <= (options.missing ?? 0)) - return Response.json( - { type: "error", error: { type: "not_found_error", message: `File file_${state.uploads} not found` } }, - { status: 404 }, - ) - return Response.json({ - id: "msg_test", - type: "message", - role: "assistant", - model: "claude-test", - content: [{ type: "text", text: "ok" }], - stop_reason: "end_turn", - stop_sequence: null, - usage: { input_tokens: 1, output_tokens: 1 }, - }) - }, - }) - const baseURL = `${server.url}${options.relay ? "_v4/bcode-relay/worker-1/" : ""}api/v4/llm/anthropic/v1` - const native = createAnthropic({ - baseURL, - apiKey: "v4rt_test", - headers: { "anthropic-beta": "interleaved-thinking-2025-05-14,fine-grained-tool-streaming-2025-05-14" }, - fetch: Object.assign((url: RequestInfo | URL, init?: RequestInit) => fetch(url, imageFileRequest(init)), { - preconnect: fetch.preconnect, - }), - }).languageModel("claude-test") - const adapted = withImageFiles(native, { baseURL, apiKey: "v4rt_test" }) - const model = process.env.ANTHROPIC_FILES_BASELINE ? native : adapted - return { - state, - model, - async close() { - await adapted.closeImageFiles() - server.stop(true) - }, - } -} -const prompt = [ - { - role: "user" as const, - content: [ - { type: "text" as const, text: "Read screenshot" }, - { - type: "file" as const, - mediaType: "image/png", - data: screenshot, - providerOptions: { anthropic: { cacheControl: { type: "ephemeral" } } }, - }, - ], - }, -] -for (const relay of [false, true]) - test(`actual BrowserCode Anthropic screenshot history three turns (relay=${relay})`, async () => { - const f = fixture({ relay }) - const original = JSON.stringify(history) - try { - for (let turn = 0; turn < 3; turn++) - await generateText({ - model: f.model, - messages: [...(await MessageV2.toModelMessages(history, spec)), { role: "user", content: `turn ${turn}` }], - maxRetries: 0, - }) - if (process.env.ANTHROPIC_FILES_EVIDENCE) - await Bun.write( - process.env.ANTHROPIC_FILES_EVIDENCE + (relay ? "-relay" : ""), - JSON.stringify({ - uploads: f.state.uploads, - bodies: f.state.bodies, - historyIntact: JSON.stringify(history) === original, - }), - ) - expect(f.state.uploads).toBe(1) - expect(f.state.bodies).toHaveLength(3) - for (const body of f.state.bodies) { - expect(body.messages[0].content).toEqual([ - { type: "tool_use", id: "call-1", name: "browser_execute", input: { code: "screenshot()" } }, - ]) - expect(body.messages[1].content[0]).toEqual({ - type: "tool_result", - tool_use_id: "call-1", - content: [ - { type: "text", text: "Captured" }, - { type: "image", source: { type: "file", file_id: "file_1" } }, - ], - }) - expect(JSON.stringify(body)).not.toContain(screenshot.toString("base64")) - expect(JSON.stringify(body)).not.toContain("bu-anthropic-file:") - } - expect(JSON.stringify(history)).toBe(original) - expect(f.state.headers[0].get("anthropic-beta")).toContain("files-api-2025-04-14") - expect(f.state.headers[0].get("anthropic-beta")).toContain("interleaved-thinking-2025-05-14") - expect(f.state.headers[0].get("anthropic-beta")).toContain("fine-grained-tool-streaming-2025-05-14") - } finally { - await f.close() - } - }) -test("Anthropic user images preserve cache control and source history", async () => { - const f = fixture() - try { - await f.model.doGenerate({ prompt }) - expect(f.state.uploads).toBe(1) - expect(f.state.bodies[0].messages[0].content[1]).toEqual({ - type: "image", - source: { type: "file", file_id: "file_1" }, - cache_control: { type: "ephemeral" }, - }) - expect(prompt[0].content[1]).toMatchObject({ data: screenshot }) - } finally { - await f.close() - } -}) -test("unsupported Anthropic routes retain legacy behavior before selection", async () => { - const f = fixture({ supported: false }) - try { - await f.model.doGenerate({ prompt }) - expect(f.state.uploads).toBe(0) - expect(f.state.bodies[0].messages[0].content[1]).toMatchObject({ source: { type: "base64" } }) - } finally { - await f.close() - } -}) -test("Anthropic upload failure never silently falls back", async () => { - const f = fixture({ uploadStatus: 503 }) - try { - await expect(f.model.doGenerate({ prompt })).rejects.toThrow("upload failed") - expect(f.state.calls).toBe(0) - } finally { - await f.close() - } -}) -test("Anthropic scope rotation fails before another inference", async () => { - const f = fixture() - try { - await f.model.doGenerate({ prompt }) - f.state.scope = "rotated" - await expect(f.model.doGenerate({ prompt })).rejects.toThrow("account changed") - expect(f.state.calls).toBe(1) - } finally { - await f.close() - } -}) -for (const missing of [1, 2]) - test(`Anthropic missing file gets one bounded refresh (${missing})`, async () => { - const f = fixture({ missing }) - try { - if (missing === 1) await f.model.doGenerate({ prompt }) - else await expect(f.model.doGenerate({ prompt })).rejects.toThrow() - expect(f.state.uploads).toBe(2) - expect(f.state.calls).toBe(2) - } finally { - await f.close() - } - }) diff --git a/packages/opencode/test/provider/image-files-lifecycle.test.ts b/packages/opencode/test/provider/image-files-lifecycle.test.ts deleted file mode 100644 index b6062b80d..000000000 --- a/packages/opencode/test/provider/image-files-lifecycle.test.ts +++ /dev/null @@ -1,547 +0,0 @@ -import { expect, test } from "bun:test" -import type { LanguageModelV3, LanguageModelV3CallOptions, LanguageModelV3StreamPart } from "@ai-sdk/provider" -import { imageFileRequest, withImageFiles } from "../../src/provider/image-files" - -function images(start: number, count: number): LanguageModelV3CallOptions { - return { - prompt: [ - { - role: "user", - content: Array.from({ length: count }, (_, i) => ({ - type: "file", - mediaType: "image/png", - data: Buffer.from(`screenshot-${start + i}`), - })), - }, - ], - } -} -function fixture(options: { legacy?: boolean; anthropic?: boolean } = {}) { - const state = { - now: Date.now(), - scope: crypto.randomUUID(), - deleteStatus: 0, - deleteGate: undefined as Promise | undefined, - deleteStarted: 0, - deleteDelay: 0, - deleteActive: 0, - deletePeak: 0, - uploadDelay: 0, - hangDeletes: false, - uploadGate: undefined as Promise | undefined, - uploads: 0, - capabilities: 0, - uploadFailures: 0, - capabilityFailures: 0, - deleteFailures: 0, - calls: [] as LanguageModelV3CallOptions[], - deleted: [] as string[], - live: new Set(), - gate: undefined as Promise | undefined, - stream: undefined as ReadableStreamDefaultController | undefined, - } - const server = Bun.serve({ - port: 0, - async fetch(req) { - if (req.url.includes("/capability")) { - state.capabilities++ - if (state.capabilityFailures-- > 0) return new Response("unavailable", { status: 503 }) - return Response.json({ - supported: true, - scope: state.scope, - ...(options.legacy ? {} : { max_references: 300, proof_transport: "body" }), - }) - } - if (req.method === "DELETE") { - state.deleteStarted++ - state.deleteActive++ - state.deletePeak = Math.max(state.deletePeak, state.deleteActive) - try { - await Bun.sleep(state.deleteDelay) - await state.deleteGate - if (state.hangDeletes) return new Promise(() => {}) - if (state.deleteStatus) return new Response("failed", { status: state.deleteStatus }) - if (state.deleteFailures-- > 0) return new Response("unavailable", { status: 503 }) - const id = new URL(req.url).pathname.split("/").at(-1)! - state.deleted.push(id) - state.live.delete(id) - return Response.json({ deleted: true }) - } finally { - state.deleteActive-- - } - } - state.uploads++ - const upload = state.uploads - await Bun.sleep(state.uploadDelay) - await state.uploadGate - if (state.uploadFailures-- > 0) return new Response("unavailable", { status: 503 }) - const id = `${options.anthropic ? "file_" : "file-"}${upload}` - state.live.add(id) - return Response.json({ file_id: id, expires_at: Math.floor(state.now / 1000) + 3600, signature: "a".repeat(64) }) - }, - }) - const native: LanguageModelV3 = { - specificationVersion: "v3", - provider: options.anthropic ? "anthropic" : "openai", - modelId: "test", - supportedUrls: {}, - async doGenerate(params) { - state.calls.push(params) - await state.gate - return { - content: [], - finishReason: { unified: "stop", raw: "stop" }, - usage: { - inputTokens: { total: 0, noCache: 0, cacheRead: 0, cacheWrite: 0 }, - outputTokens: { total: 0, text: 0, reasoning: 0 }, - }, - warnings: [], - } - }, - async doStream(params) { - state.calls.push(params) - return { - stream: new ReadableStream({ - start(controller) { - state.stream = controller - }, - }), - warnings: [], - } - }, - } - const baseURL = `${server.url}api/v4/llm/${native.provider}/v1` - const wrap = (modelId = native.modelId) => - withImageFiles({ ...native, modelId }, { baseURL, apiKey: "test", now: () => state.now }) - const model = wrap() - return { - state, - model, - wrap, - async close() { - await model.closeImageFiles() - server.stop(true) - }, - } -} - -for (const anthropic of [false, true]) { - test(`200 distinct screenshots and retained history upload once (anthropic=${anthropic})`, async () => { - const f = fixture({ anthropic }) - try { - await f.model.doGenerate(images(0, 200)) - await f.model.doGenerate(images(0, 201)) - expect(f.state.uploads).toBe(201) - expect(f.state.calls[1].prompt[0].content).toHaveLength(201) - expect(JSON.stringify(f.state.calls)).not.toContain(Buffer.from("screenshot-0").toString("base64")) - } finally { - await f.close() - } - }) -} -test("more than 300 lifetime images with bounded history evict only unused files", async () => { - const f = fixture() - try { - await f.model.doGenerate(images(0, 300)) - const message = f.state.calls[0].prompt[0] - if (message.role !== "user") throw new Error("Expected user images") - const ids = message.content.map((part) => (part.type === "file" ? String(part.data) : "")) - await f.model.doGenerate(images(1, 300)) - expect(f.state.uploads).toBe(301) - expect(f.state.deleted).toEqual([ids[0]]) - expect(f.state.live.size).toBe(300) - await f.model.doGenerate(images(2, 300)) - expect(f.state.uploads).toBe(302) - expect(f.state.deleted).toEqual([ids[0], ids[1]]) - } finally { - await f.close() - } -}) -test("301 simultaneous images fail before any upload without discarding content", async () => { - const f = fixture({ anthropic: true }) - try { - await expect(f.model.doGenerate(images(0, 301))).rejects.toThrow("300") - expect(f.state.uploads).toBe(0) - expect(f.state.calls).toHaveLength(0) - } finally { - await f.close() - } -}) -test("in-flight generation pins its files against eviction and close", async () => { - const f = fixture() - try { - await f.model.doGenerate(images(0, 300)) - const gate = Promise.withResolvers() - f.state.gate = gate.promise - const pending = f.model.doGenerate(images(0, 1)) - while (f.state.calls.length < 2) await Bun.sleep(1) - const part = f.state.calls[1].prompt[0].content[0] - const pinned = typeof part !== "string" && part.type === "file" ? String(part.data) : "missing" - f.state.gate = undefined - await f.model.doGenerate(images(300, 1)) - expect(f.state.deleted).not.toContain(pinned) - await f.model.closeImageFiles() - expect(f.state.deleted).not.toContain(pinned) - gate.resolve() - await pending - await f.model.closeImageFiles() - expect(f.state.live.size).toBe(0) - } finally { - await f.close() - } -}) -test("stream retains references until cancellation then permits cleanup", async () => { - const f = fixture() - try { - const result = await f.model.doStream(images(0, 1)) - await f.model.closeImageFiles() - expect(f.state.deleted).toEqual([]) - await result.stream.cancel() - await f.model.closeImageFiles() - expect(f.state.live.size).toBe(0) - } finally { - await f.close() - } -}) -test("transient capability and upload failures retry within one call", async () => { - const f = fixture() - try { - f.state.capabilityFailures = 1 - f.state.uploadFailures = 1 - await f.model.doGenerate(images(0, 1)) - expect(f.state.capabilities).toBe(2) - expect(f.state.uploads).toBe(2) - expect(f.state.calls).toHaveLength(1) - } finally { - await f.close() - } -}) -test("failed uploads and cleanup can recover on explicit retry", async () => { - const f = fixture() - try { - f.state.uploadFailures = 10 - await expect(f.model.doGenerate(images(0, 1))).rejects.toThrow("upload failed") - expect(f.state.uploads).toBe(3) - f.state.uploadFailures = 0 - await f.model.doGenerate(images(0, 1)) - f.state.deleteFailures = 10 - await f.model.closeImageFiles() - expect(f.state.live.size).toBe(1) - f.state.deleteFailures = 0 - await f.model.closeImageFiles() - expect(f.state.live.size).toBe(0) - } finally { - await f.close() - } -}) -test("negotiated 300 proofs move into bounded body metadata before HTTP", async () => { - const f = fixture() - try { - await f.model.doGenerate(images(0, 300)) - const result = imageFileRequest({ - headers: f.state.calls[0].headers as Record, - body: JSON.stringify({ input: [] }), - })! - const headers = new Headers(result.headers) - const body = JSON.parse(result.body as string) - expect(headers.has("x-bu-image-proofs")).toBe(false) - expect(Object.keys(body._image_file_proofs)).toHaveLength(300) - expect(JSON.stringify([...headers]).length).toBeLessThan(1024) - } finally { - await f.close() - } -}) - -test("legacy gateway keeps 64-reference contract and header proofs", async () => { - const f = fixture({ legacy: true }) - try { - await f.model.doGenerate(images(0, 64)) - const result = imageFileRequest({ - headers: f.state.calls[0].headers as Record, - body: JSON.stringify({ input: [] }), - })! - expect(new Headers(result.headers).has("x-bu-image-proofs")).toBe(true) - expect(JSON.parse(result.body as string)._image_file_proofs).toBeUndefined() - await expect(f.model.doGenerate(images(0, 65))).rejects.toThrow("64") - expect(f.state.uploads).toBe(64) - } finally { - await f.close() - } -}) -for (const ending of ["complete", "error", "abort"] as const) { - test(`stream ${ending} releases pins after close`, async () => { - const f = fixture() - try { - const abort = new AbortController() - const result = await f.model.doStream({ ...images(0, 1), abortSignal: abort.signal }) - const reader = result.stream.getReader() - await f.model.closeImageFiles() - expect(f.state.deleted).toEqual([]) - if (ending === "complete") f.state.stream!.close() - if (ending === "error") f.state.stream!.error(new Error("stream failed")) - if (ending === "abort") abort.abort() - if (ending === "error") await expect(reader.read()).rejects.toThrow("stream failed") - else expect((await reader.read()).done).toBe(true) - await f.model.closeImageFiles() - expect(f.state.live.size).toBe(0) - } finally { - await f.close() - } - }) -} -test("overlapping active requests cannot evict each other's full working set", async () => { - const f = fixture() - const gate = Promise.withResolvers() - try { - f.state.gate = gate.promise - const pending = f.model.doGenerate(images(0, 300)) - while (!f.state.calls.length) await Bun.sleep(1) - await expect(f.model.doGenerate(images(300, 1))).rejects.toThrow("active requests") - expect(f.state.deleted).toEqual([]) - f.state.gate = undefined - gate.resolve() - await pending - await f.model.doGenerate(images(300, 1)) - expect(f.state.uploads).toBe(301) - } finally { - gate.resolve() - await f.close() - } -}) -test("abort during transient retry stops uploading and leaves later call usable", async () => { - const f = fixture() - try { - f.state.uploadFailures = 10 - const abort = new AbortController() - const pending = f.model.doGenerate({ ...images(0, 1), abortSignal: abort.signal }) - while (!f.state.uploads) await Bun.sleep(1) - abort.abort() - await expect(pending).rejects.toThrow() - expect(f.state.uploads).toBe(1) - expect(f.state.calls).toHaveLength(0) - f.state.uploadFailures = 0 - await f.model.doGenerate(images(0, 1)) - expect(f.state.uploads).toBe(2) - } finally { - await f.close() - } -}) - -test("hung deletes have a total cleanup deadline and remain retryable", async () => { - const f = fixture() - try { - await f.model.doGenerate(images(0, 20)) - f.state.hangDeletes = true - const started = performance.now() - await f.model.closeImageFiles() - expect(performance.now() - started).toBeLessThan(2500) - expect(f.state.live.size).toBe(20) - f.state.hangDeletes = false - await f.model.closeImageFiles() - expect(f.state.live.size).toBe(0) - } finally { - await f.close() - } -}) -test("expired closed caches cannot exhaust the global run slots after failed deletes", async () => { - const f = fixture() - try { - f.state.deleteStatus = 400 - for (let i = 0; i < 130; i++) { - f.state.scope = crypto.randomUUID() - const model = f.wrap() - await model.doGenerate(images(i, 1)) - await model.closeImageFiles() - f.state.now += 3601_000 - } - expect(f.state.calls).toHaveLength(130) - } finally { - await f.close() - } -}) -test("delete outage bounds retired files and permits recovery without losing pins", async () => { - const f = fixture() - try { - f.state.deleteStatus = 400 - await f.model.doGenerate(images(0, 300)) - await f.model.doGenerate(images(300, 300)) - expect(f.state.live.size).toBe(600) - await expect(f.model.doGenerate(images(600, 1))).rejects.toThrow("cleanup backlog") - expect(f.state.uploads).toBe(600) - f.state.deleteStatus = 0 - await f.model.doGenerate(images(600, 1)) - expect(f.state.uploads).toBe(601) - await f.model.closeImageFiles() - await f.model.closeImageFiles() - expect(f.state.live.size).toBe(0) - } finally { - await f.close() - } -}) - -test("different models never share uploaded files even with an identical gateway scope", async () => { - const f = fixture() - const sibling = f.wrap("another-model") - try { - await f.model.doGenerate(images(0, 1)) - await sibling.doGenerate(images(0, 1)) - expect(f.state.uploads).toBe(2) - await f.model.closeImageFiles() - expect(f.state.live.size).toBe(1) - await sibling.closeImageFiles() - expect(f.state.live.size).toBe(0) - } finally { - await sibling.closeImageFiles() - await f.close() - } -}) - -for (const store of [false, true]) { - test(`stateful Responses keep omitted files until close (explicit store=${store})`, async () => { - const f = fixture() - try { - await f.model.doGenerate({ - ...images(0, 300), - ...(store ? { providerOptions: { openai: { store: true } } } : {}), - }) - await f.model.doGenerate({ prompt: [], providerOptions: { openai: { previousResponseId: "resp-prior" } } }) - await expect(f.model.doGenerate(images(300, 1))).rejects.toThrow("stateful response chain") - expect(f.state.uploads).toBe(300) - expect(f.state.deleted).toEqual([]) - await f.model.closeImageFiles() - expect(f.state.live.size).toBe(0) - } finally { - await f.close() - } - }) -} - -test("cached image requests do not wait behind unrelated slow uploads", async () => { - const f = fixture() - const gate = Promise.withResolvers() - try { - await f.model.doGenerate(images(0, 1)) - f.state.uploadGate = gate.promise - const uploading = f.model.doGenerate(images(1, 1)) - while (f.state.uploads < 2) await Bun.sleep(1) - await Promise.race([ - f.model.doGenerate(images(0, 1)), - Bun.sleep(500).then(() => { - throw new Error("cached request blocked by upload") - }), - ]) - gate.resolve() - await uploading - expect(f.state.uploads).toBe(2) - } finally { - gate.resolve() - await f.close() - } -}) - -test("fresh wrapper retries cleanup left by a closed owner", async () => { - const f = fixture() - const sibling = f.wrap() - try { - await f.model.doGenerate(images(0, 1)) - f.state.deleteStatus = 400 - await f.model.closeImageFiles() - await expect(sibling.doGenerate(images(0, 1))).rejects.toThrow("closing") - f.state.deleteStatus = 0 - await sibling.doGenerate(images(0, 1)) - expect(f.state.uploads).toBe(2) - expect(f.state.deleted).toHaveLength(1) - } finally { - await sibling.closeImageFiles() - await f.close() - } -}) - -test("last stream ending during close cleanup schedules a final cleanup pass", async () => { - const f = fixture() - const gate = Promise.withResolvers() - try { - const result = await f.model.doStream(images(0, 1)) - await f.model.doGenerate(images(1, 1)) - f.state.deleteGate = gate.promise - const closing = f.model.closeImageFiles() - while (!f.state.deleteStarted) await Bun.sleep(1) - await result.stream.cancel() - gate.resolve() - await closing - for (let i = 0; i < 100 && f.state.live.size; i++) await Bun.sleep(5) - expect(f.state.live.size).toBe(0) - } finally { - gate.resolve() - await f.close() - } -}) - -for (const latency of [false, true]) { - test(`healthy cleanup keeps up with 1000 lifetime images and 100 active (latency=${latency})`, async () => { - const f = fixture() - f.state.deleteDelay = latency ? 200 : 0 - f.state.uploadDelay = latency ? 20 : 0 - try { - for (let turn = 0; turn < 10; turn++) { - await f.model.doGenerate(images(turn * 100, 100)) - expect(f.state.live.size).toBeLessThanOrEqual(600) - if (latency) await Bun.sleep(50) - } - expect(f.state.uploads).toBe(1000) - expect(f.state.calls).toHaveLength(10) - await f.model.closeImageFiles() - const deadline = Date.now() + 6000 - while (f.state.live.size && Date.now() < deadline) await Bun.sleep(10) - expect(f.state.live.size).toBe(0) - expect(f.state.deleted).toHaveLength(1000) - expect(new Set(f.state.deleted).size).toBe(1000) - expect(f.state.deletePeak).toBeLessThanOrEqual(16) - } finally { - await f.close() - } - }, 30000) -} - -test("preparation shares one second of cleanup wait across eviction passes", async () => { - const f = fixture() - try { - await f.model.doGenerate(images(0, 300)) - f.state.deleteStatus = 400 - await f.model.doGenerate(images(300, 100)) - f.state.deleteStatus = 0 - f.state.deleteDelay = 200 - const started = performance.now() - const result = await f.model.doStream(images(400, 100)) - expect(performance.now() - started).toBeLessThan(1900) - expect(f.state.uploads).toBe(500) - expect(f.state.deletePeak).toBeLessThanOrEqual(16) - await result.stream.cancel() - await f.model.closeImageFiles() - const deadline = Date.now() + 7000 - while (f.state.live.size && Date.now() < deadline) await Bun.sleep(10) - expect(f.state.live.size).toBe(0) - } finally { - await f.close() - } -}, 15000) - -test("background cleanup stops on failure and keeps tombstones for explicit recovery", async () => { - const f = fixture() - try { - await f.model.doGenerate(images(0, 100)) - f.state.deleteStatus = 400 - await f.model.closeImageFiles() - const attempts = f.state.deleteStarted - expect(attempts).toBeGreaterThan(0) - expect(attempts).toBeLessThanOrEqual(16) - await Bun.sleep(100) - expect(f.state.deleteStarted).toBe(attempts) - expect(f.state.live.size).toBe(100) - f.state.deleteStatus = 0 - await f.model.closeImageFiles() - expect(f.state.live.size).toBe(0) - } finally { - await f.close() - } -}) diff --git a/packages/opencode/test/provider/image-files.test.ts b/packages/opencode/test/provider/image-files.test.ts deleted file mode 100644 index d6971c3d6..000000000 --- a/packages/opencode/test/provider/image-files.test.ts +++ /dev/null @@ -1,521 +0,0 @@ -import { expect, test } from "bun:test" -import { generateText } from "ai" -import { SessionV1 } from "@opencode-ai/core/v1/session" -import type { Provider } from "../../src/provider/provider" -import { MessageV2 } from "../../src/session/message-v2" -import { createOpenAI } from "@ai-sdk/openai" -import { imageFileRequest, withImageFiles } from "../../src/provider/image-files" - -const screenshot = Buffer.alloc(1024 * 1024, 42) -const prompt = [ - { - role: "user" as const, - content: [ - { type: "text" as const, text: "Read this screenshot" }, - { - type: "file" as const, - mediaType: "image/png", - data: screenshot, - providerOptions: { openai: { imageDetail: "high" } }, - }, - ], - }, - { - role: "assistant" as const, - content: [{ type: "tool-call" as const, toolCallId: "call_1", toolName: "browse", input: {} }], - }, - { - role: "tool" as const, - content: [ - { - type: "tool-result" as const, - toolCallId: "call_1", - toolName: "browse", - output: { type: "text" as const, value: "done" }, - }, - ], - }, -] - -test("one screenshot reused in three turns is uploaded once before gateway JSON", async () => { - const requests: Array<{ input: Array> }> = [] - let uploads = 0 - const testScope = crypto.randomUUID() - const server = Bun.serve({ - port: 0, - async fetch(req) { - const path = new URL(req.url).pathname - if (path.endsWith("/image-files/capability")) return Response.json({ supported: true, scope: testScope }) - if (path.endsWith("/image-files")) { - expect(req.headers.get("content-type")).toBe("image/png") - expect(Buffer.from(await req.arrayBuffer()).equals(screenshot)).toBe(true) - uploads++ - return Response.json({ file_id: "file-screenshot", signature: "proof", expires_at: Date.now() / 1000 + 3600 }) - } - requests.push(await req.json()) - return Response.json({ - id: "resp_test", - object: "response", - created_at: 1, - model: "gpt-test", - output: [ - { - type: "message", - id: "msg_1", - role: "assistant", - status: "completed", - content: [{ type: "output_text", text: "ok", annotations: [] }], - }, - ], - usage: { input_tokens: 1, output_tokens: 1, total_tokens: 2 }, - status: "completed", - }) - }, - }) - try { - const baseURL = `${server.url}api/v4/llm/openai/v1` - const native = createOpenAI({ - baseURL, - apiKey: "v4rt_test", - fetch: Object.assign((url: RequestInfo | URL, init?: RequestInit) => fetch(url, imageFileRequest(init)), { - preconnect: fetch.preconnect, - }), - }).responses("gpt-test") - const model = process.env.IMAGE_FILES_BASELINE ? native : withImageFiles(native, { baseURL, apiKey: "v4rt_test" }) - for (let i = 0; i < 3; i++) - await model.doGenerate({ prompt: [...prompt, { role: "user", content: [{ type: "text", text: `turn ${i}` }] }] }) - if (process.env.IMAGE_FILES_EVIDENCE) - await Bun.write( - process.env.IMAGE_FILES_EVIDENCE, - JSON.stringify({ - uploads, - bodies: requests, - originalHistoryIntact: JSON.stringify(prompt).includes(JSON.stringify(screenshot)), - }), - ) - expect(uploads).toBe(1) - expect(requests).toHaveLength(3) - for (const body of requests) { - expect(JSON.stringify(body)).not.toContain(screenshot.toString("base64")) - expect(body.input[0].content).toEqual([ - { type: "input_text", text: "Read this screenshot" }, - { type: "input_image", file_id: "file-screenshot", detail: "high" }, - ]) - expect(body.input[1]).toMatchObject({ type: "function_call", call_id: "call_1", name: "browse", arguments: "{}" }) - expect(body.input[2]).toMatchObject({ type: "function_call_output", call_id: "call_1", output: "done" }) - } - expect(prompt[0].content[1]).toMatchObject({ data: screenshot }) - } finally { - server.stop(true) - } -}) - -test("actual BrowserCode screenshot history is externalized without changing persisted history", async () => { - const history = [ - { - info: { - id: "msg-a", - parentID: "msg-u", - sessionID: "session", - role: "assistant", - modelID: "gpt-test", - providerID: "openai", - time: { created: 0 }, - }, - parts: [ - { - id: "part", - sessionID: "session", - messageID: "msg-a", - type: "tool", - tool: "browser_execute", - callID: "call-1", - state: { - status: "completed", - input: { code: "screenshot()" }, - output: "Captured", - title: "Screenshot", - metadata: {}, - time: { start: 0, end: 1 }, - attachments: [ - { - id: "file", - sessionID: "session", - messageID: "msg-a", - type: "file", - mime: "image/png", - url: `data:image/png;base64,${screenshot.toString("base64")}`, - }, - ], - }, - }, - ], - }, - ] as unknown as SessionV1.WithParts[] - const original = JSON.stringify(history) - const spec = { - id: "gpt-test", - providerID: "openai", - api: { id: "gpt-test", npm: "@ai-sdk/openai" }, - capabilities: { input: { image: true } }, - } as Provider.Model - let uploads = 0 - const bodies: Record[] = [] - const testScope = crypto.randomUUID() - const server = Bun.serve({ - port: 0, - async fetch(req) { - if (req.url.includes("/capability")) return Response.json({ supported: true, scope: testScope }) - if (req.url.includes("/image-files")) { - uploads++ - return Response.json({ file_id: "file-image", signature: "proof", expires_at: Date.now() / 1000 + 3600 }) - } - bodies.push(await req.json()) - return Response.json({ - id: "resp_a", - created_at: 1, - model: "gpt-test", - output: [ - { - type: "message", - id: "msg-a", - role: "assistant", - status: "completed", - content: [{ type: "output_text", text: "ok", annotations: [] }], - }, - ], - usage: { input_tokens: 1, output_tokens: 1, total_tokens: 2 }, - status: "completed", - }) - }, - }) - try { - const baseURL = `${server.url}api/v4/llm/openai/v1` - const native = createOpenAI({ - baseURL, - apiKey: "v4rt_test", - fetch: Object.assign((url: RequestInfo | URL, init?: RequestInit) => fetch(url, imageFileRequest(init)), { - preconnect: fetch.preconnect, - }), - }).responses("gpt-test") - const model = process.env.IMAGE_FILES_BASELINE ? native : withImageFiles(native, { baseURL, apiKey: "v4rt_test" }) - for (let turn = 0; turn < 3; turn++) - await generateText({ - model, - messages: [...(await MessageV2.toModelMessages(history, spec)), { role: "user", content: `turn ${turn}` }], - maxRetries: 0, - }) - expect(uploads).toBe(1) - for (const body of bodies) { - const input = body.input as Array> - expect(input[0]).toMatchObject({ type: "function_call", call_id: "call-1", name: "browser_execute" }) - expect(input[1]).toMatchObject({ - type: "function_call_output", - call_id: "call-1", - output: [ - { type: "input_text", text: "Captured" }, - { type: "input_image", file_id: "file-image" }, - ], - }) - expect(JSON.stringify(body)).toContain('"file_id":"file-image"') - expect(JSON.stringify(body)).not.toContain(screenshot.toString("base64")) - } - expect(JSON.stringify(history)).toBe(original) - } finally { - server.stop(true) - } -}) - -function fixture( - options: { supported?: boolean; missing?: number; uploadDelay?: number; uploadStatus?: number; relay?: boolean } = {}, -) { - const state = { - uploads: 0, - responses: 0, - deletes: 0, - scope: String(crypto.randomUUID()), - now: Date.now(), - active: 0, - peak: 0, - bodies: [] as string[], - } - const testScope = crypto.randomUUID() - const server = Bun.serve({ - port: 0, - async fetch(req) { - if (req.url.includes("/capability")) - return Response.json({ supported: options.supported ?? true, scope: state.scope }) - if (req.method === "DELETE") { - state.deletes++ - return Response.json({ deleted: true }) - } - if (req.url.includes("/image-files")) { - state.uploads++ - state.active++ - state.peak = Math.max(state.peak, state.active) - if (options.uploadDelay) await Bun.sleep(options.uploadDelay) - state.active-- - if (options.uploadStatus) return new Response("failed", { status: options.uploadStatus }) - return Response.json({ - file_id: `file-${state.uploads}`, - expires_at: Math.floor(state.now / 1000) + 3600, - signature: "proof", - }) - } - state.responses++ - const body = await req.text() - state.bodies.push(body) - if (state.responses <= (options.missing ?? 0)) - return Response.json( - { - error: { - message: `File file-${state.uploads} not found`, - type: "invalid_request_error", - code: "invalid_image", - }, - }, - { status: 400 }, - ) - return Response.json({ - id: "resp_a", - created_at: 1, - model: "gpt-test", - output: [ - { - type: "message", - id: "msg-a", - role: "assistant", - status: "completed", - content: [{ type: "output_text", text: "ok", annotations: [] }], - }, - ], - usage: { input_tokens: 1, output_tokens: 1, total_tokens: 2 }, - status: "completed", - }) - }, - }) - const baseURL = `${server.url}${options.relay ? "_v4/bcode-relay/relay-123/" : ""}api/v4/llm/openai/v1` - const native = createOpenAI({ - baseURL, - apiKey: "v4rt_test", - fetch: Object.assign((url: RequestInfo | URL, init?: RequestInit) => fetch(url, imageFileRequest(init)), { - preconnect: fetch.preconnect, - }), - }).responses("gpt-test") - const model = withImageFiles(native, { baseURL, apiKey: "v4rt_test", now: () => state.now }) - return { - state, - model, - wrap: () => withImageFiles(native, { baseURL, apiKey: "v4rt_test", now: () => state.now }), - stop: () => server.stop(true), - } -} - -test("unsupported actual route keeps inline behavior", async () => { - const f = fixture({ supported: false }) - try { - await f.model.doGenerate({ prompt }) - expect(f.state.uploads).toBe(0) - expect(f.state.bodies[0]).toContain(screenshot.toString("base64")) - } finally { - f.stop() - } -}) - -test("concurrent repeated screenshot has one upload and cleanup deletes it", async () => { - const f = fixture({ uploadDelay: 30 }) - try { - await Promise.all([f.model.doGenerate({ prompt }), f.model.doGenerate({ prompt }), f.model.doGenerate({ prompt })]) - expect(f.state.uploads).toBe(1) - await f.model.closeImageFiles() - expect(f.state.deletes).toBe(1) - } finally { - f.stop() - } -}) - -test("deleted reference recovers once with original previous_response_id and no inline retry", async () => { - const f = fixture({ missing: 1 }) - try { - await f.model.doGenerate({ prompt, providerOptions: { openai: { previousResponseId: "resp_prior", store: true } } }) - expect(f.state.uploads).toBe(2) - expect(f.state.responses).toBe(2) - for (const body of f.state.bodies) { - expect(body).not.toContain(screenshot.toString("base64")) - expect(JSON.parse(body).previous_response_id).toBe("resp_prior") - } - } finally { - f.stop() - } -}) - -test("repeated invalid references and failed upload stay bounded", async () => { - const f = fixture({ missing: 10 }) - try { - await expect(f.model.doGenerate({ prompt })).rejects.toThrow("not found") - expect(f.state.uploads).toBe(2) - expect(f.state.responses).toBe(2) - await expect(f.model.doGenerate({ prompt })).rejects.toThrow("not found") - expect(f.state.uploads).toBe(2) - } finally { - f.stop() - } - const failed = fixture({ uploadStatus: 503 }) - try { - await expect(failed.model.doGenerate({ prompt })).rejects.toThrow("upload failed") - expect(failed.state.responses).toBe(0) - expect(failed.state.uploads).toBe(3) - } finally { - failed.stop() - } -}) - -test("account rotation fails before forwarding any inline screenshot", async () => { - const f = fixture() - try { - await f.model.doGenerate({ prompt }) - f.state.scope = "another-account" - await expect(f.model.doGenerate({ prompt })).rejects.toThrow("account changed") - expect(f.state.uploads).toBe(1) - expect(f.state.responses).toBe(1) - } finally { - f.stop() - } -}) - -test("producer bounds concurrent image uploads", async () => { - const f = fixture({ uploadDelay: 10 }) - try { - await f.model.doGenerate({ - prompt: [ - { - role: "user", - content: Array.from({ length: 6 }, (_, i) => ({ - type: "file", - mediaType: "image/png", - data: Buffer.alloc(1024, i), - })), - }, - ], - }) - expect(f.state.uploads).toBe(6) - expect(f.state.peak).toBeLessThanOrEqual(2) - } finally { - f.stop() - } -}) - -test("aborted upload never falls back to a large request", async () => { - const f = fixture({ uploadDelay: 100 }) - try { - const abort = new AbortController() - const pending = f.model.doGenerate({ prompt, abortSignal: abort.signal }) - await Bun.sleep(20) - abort.abort() - await expect(pending).rejects.toThrow() - expect(f.state.responses).toBe(0) - } finally { - f.stop() - } -}) - -test("placeholder conversion only changes image parts in native input arrays", () => { - const unrelated = { type: "input_image", image_url: "bu-openai-file:file-meta" } - const result = imageFileRequest({ - headers: { "x-bu-image-scope": "run" }, - body: JSON.stringify({ - metadata: unrelated, - input: [ - { - type: "function_call_output", - output: [ - { type: "input_image", image_url: "bu-openai-file:file-a", detail: "high" }, - { type: "input_text", text: "bu-openai-file:file-a" }, - ], - }, - ], - }), - }) - expect(JSON.parse(result!.body as string)).toEqual({ - metadata: unrelated, - input: [ - { - type: "function_call_output", - output: [ - { type: "input_image", file_id: "file-a", detail: "high" }, - { type: "input_text", text: "bu-openai-file:file-a" }, - ], - }, - ], - }) -}) - -test("reusable worker relay URL uses upload references", async () => { - const f = fixture({ relay: true }) - try { - await f.model.doGenerate({ prompt }) - expect(f.state.uploads).toBe(1) - expect(f.state.bodies[0]).not.toContain(screenshot.toString("base64")) - } finally { - f.stop() - } -}) - -test("expired duplicate references coalesce even with a full 64-image run cache", async () => { - const f = fixture() - try { - const images = Array.from({ length: 64 }, (_, i) => ({ - type: "file" as const, - mediaType: "image/png", - data: Buffer.alloc(1024, i), - })) - await f.model.doGenerate({ prompt: [{ role: "user", content: images }] }) - expect(f.state.uploads).toBe(64) - f.state.now += 3600_000 - await f.model.doGenerate({ prompt: [{ role: "user", content: [images[0], images[0]] }] }) - expect(f.state.uploads).toBe(65) - f.state.now += 3600_000 - await expect(f.model.doGenerate({ prompt: [{ role: "user", content: [images[0]] }] })).rejects.toThrow( - "expired again", - ) - expect(f.state.uploads).toBe(65) - } finally { - f.stop() - } -}) - -test("transient upload rejection permits a later explicit retry", async () => { - const options: { uploadStatus?: number } = { uploadStatus: 503 } - const f = fixture(options) - try { - await expect(f.model.doGenerate({ prompt })).rejects.toThrow("upload failed") - expect(f.state.responses).toBe(0) - options.uploadStatus = undefined - await f.model.doGenerate({ prompt }) - expect(f.state.uploads).toBe(4) - expect(f.state.responses).toBe(1) - } finally { - await f.model.closeImageFiles() - f.stop() - } -}) - -test("closing one wrapper preserves references owned by a sibling", async () => { - const f = fixture() - const sibling = f.wrap() - try { - await f.model.doGenerate({ prompt }) - await sibling.doGenerate({ prompt }) - await f.model.closeImageFiles() - expect(f.state.deletes).toBe(0) - await expect(f.model.doGenerate({ prompt })).rejects.toThrow("closed") - await sibling.doGenerate({ prompt }) - expect(f.state.uploads).toBe(1) - await sibling.closeImageFiles() - expect(f.state.deletes).toBe(1) - } finally { - await f.model.closeImageFiles() - await sibling.closeImageFiles() - f.stop() - } -})