diff --git a/src/server/responses/fetch-helpers.ts b/src/server/responses/fetch-helpers.ts index 1cbb466b44..b4affdc0ab 100644 --- a/src/server/responses/fetch-helpers.ts +++ b/src/server/responses/fetch-helpers.ts @@ -1,4 +1,5 @@ import type { Server } from "bun"; +import { codexWsUpstreamFetch, shouldUseCodexWsUpstream } from "./ws-upstream"; import { bridgeToResponsesSSE, buildResponseJSON, formatErrorResponse, type ResponsesTerminalStatus } from "../../bridge"; import { getConfigPath, @@ -131,7 +132,17 @@ export function safeOriginLabel(url: string): string { export function providerFetch(provider: OcxProviderConfig): typeof globalThis.fetch { - return (provider as OcxProviderConfig & { fetch?: typeof globalThis.fetch }).fetch ?? globalThis.fetch; + const base = (provider as OcxProviderConfig & { fetch?: typeof globalThis.fetch }).fetch ?? globalThis.fetch; + // ChatGPT Codex backend: streaming turns ride the responses_websockets + // transport (measured ~3s faster TTFT than the SSE POST queue); everything + // else keeps the provider's HTTP fetch. See ws-upstream.ts for the details. + const wrapped = (input: Parameters[0], init?: RequestInit) => { + if (typeof input === "string" && init && shouldUseCodexWsUpstream(input, init)) { + return codexWsUpstreamFetch(input, init, base); + } + return base(input, init); + }; + return wrapped as typeof globalThis.fetch; } diff --git a/src/server/responses/ws-upstream.ts b/src/server/responses/ws-upstream.ts new file mode 100644 index 0000000000..5f0092ac48 --- /dev/null +++ b/src/server/responses/ws-upstream.ts @@ -0,0 +1,199 @@ +// Upstream WebSocket transport for the ChatGPT Codex backend. +// +// Why this exists: the Codex backend serves the responses_websockets path from +// a measurably faster queue than the plain SSE POST path. Measured 2026-08-12 +// KST (same account, same payload, strictly sequential): gpt-5.6-luna TTFT p50 +// ~1.0s over WS vs ~3.9s over SSE. Codex CLI itself defaults to the WS +// transport; opencodex previously always POSTed SSE, which is where its extra +// 2-3s of TTFT came from. +// +// The wrapper only swaps the transport. It dials wss:// with the same headers, +// sends the JSON body as a single `response.create` frame, and re-encodes the +// returned event frames as an SSE byte stream, so every downstream consumer +// (passthrough relay, adapter parsers, usage sniffing) is unchanged. + +const CODEX_RESPONSES_HTTP_URL = "https://chatgpt.com/backend-api/codex/responses"; +const CODEX_RESPONSES_WS_URL = "wss://chatgpt.com/backend-api/codex/responses"; +const WS_BETA = "responses_websockets=2026-02-06"; +// If the 101 never arrives (network black hole), give SSE a chance well before +// the caller's connect timeout (default 200s) would fire. +const UPGRADE_DEADLINE_MS = 10_000; + +export function shouldUseCodexWsUpstream(url: string, init?: RequestInit): boolean { + if (url !== CODEX_RESPONSES_HTTP_URL) return false; + if ((init?.method ?? "GET").toUpperCase() !== "POST") return false; + const body = init?.body; + if (typeof body !== "string") return false; + // Only root-level stream:true selects WS: JSON-mode calls keep the HTTP path + // because the WS path only speaks the event protocol, and a nested + // {"metadata":{"stream":true}} must not flip the transport. Parsing (not + // substring matching) also keeps whitespace-formatted bodies routable. + try { + const parsed = JSON.parse(body) as unknown; + return typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) + && (parsed as Record).stream === true; + } catch { + return false; + } +} + +export function codexWsUpstreamFetch( + url: string, + init: RequestInit, + sseFallback: typeof globalThis.fetch, +): Promise { + const signal = init.signal ?? undefined; + if (signal?.aborted) { + return Promise.reject(signal.reason ?? new DOMException("The operation was aborted.", "AbortError")); + } + + let frameText: string; + try { + const body = JSON.parse(init.body as string) as Record; + // The WS create frame is implicitly streaming; the backend rejects the + // HTTP-only `stream` flag inside a frame. + delete body.stream; + frameText = JSON.stringify({ ...body, type: "response.create" }); + } catch { + return sseFallback(url, init); + } + + const headers: Record = {}; + new Headers(init.headers ?? {}).forEach((value, key) => { + // HTTP-body framing headers do not apply to a WS handshake. + if (key === "content-type" || key === "content-length" || key === "accept" || key === "accept-encoding") return; + headers[key] = value; + }); + headers["openai-beta"] = headers["openai-beta"] + ? headers["openai-beta"].includes("responses_websockets") + ? headers["openai-beta"] + : `${headers["openai-beta"]}, ${WS_BETA}` + : WS_BETA; + // A genuine caller `originator` is already in these headers via the forward + // set. Never fabricate one here: pool/forward traffic must not impersonate + // Codex CLI, per the metadata-integrity contract. (The backend's fast lane + // keys on WS + originator, so callers without the tag simply keep their own + // provenance and scheduling.) + + return new Promise((resolve, reject) => { + let ws: WebSocket; + try { + // Bun accepts per-handshake headers; the DOM lib types only list protocol arrays. + ws = new WebSocket(CODEX_RESPONSES_WS_URL, { headers } as unknown as string[]); + } catch { + resolve(sseFallback(url, init)); + return; + } + + let opened = false; + let settledPreOpen = false; + let terminal = false; + let controller: ReadableStreamDefaultController | null = null; + const encoder = new TextEncoder(); + + const upgradeTimer = setTimeout(() => { + if (opened || settledPreOpen) return; + settledPreOpen = true; + try { ws.close(); } catch { /* already closing */ } + resolve(sseFallback(url, init)); + }, UPGRADE_DEADLINE_MS); + + const onAbort = () => { + if (!opened) { + if (settledPreOpen) return; + // Settle BEFORE close(): the close handler treats a pre-open close as + // an upgrade rejection and would dial the SSE fallback for a request + // the caller just cancelled. + settledPreOpen = true; + clearTimeout(upgradeTimer); + try { ws.close(); } catch { /* already closing */ } + reject(signal?.reason ?? new DOMException("The operation was aborted.", "AbortError")); + return; + } + try { ws.close(); } catch { /* already closing */ } + if (controller && !terminal) { + terminal = true; + // Mirror an aborted fetch: the body read rejects with the abort reason. + try { controller.error(signal?.reason ?? new DOMException("The operation was aborted.", "AbortError")); } catch { /* stream already done */ } + } + }; + signal?.addEventListener("abort", onAbort, { once: true }); + + ws.addEventListener("open", () => { + if (settledPreOpen) return; + clearTimeout(upgradeTimer); + try { + ws.send(frameText); + } catch { + // send() throwing means the frame never left, so no upstream turn + // started and the SSE resend cannot double-generate. Falling back + // (instead of erroring a synthetic 200 body) keeps the pre-stream + // HTTP error/refresh/failover machinery in charge. + settledPreOpen = true; + try { ws.close(); } catch { /* already closing */ } + resolve(sseFallback(url, init)); + return; + } + opened = true; + const stream = new ReadableStream({ + start(c) { controller = c; }, + cancel() { try { ws.close(); } catch { /* already closing */ } }, + }); + resolve(new Response(stream, { + status: 200, + // The 101 response headers (x-codex-*-reset-at quota hints) are not + // exposed by Bun's WebSocket; the periodic quota poller covers those. + headers: { "content-type": "text/event-stream; charset=utf-8" }, + })); + }); + + ws.addEventListener("message", (event) => { + if (!controller || terminal) return; + const text = typeof event.data === "string" ? event.data : ""; + if (!text) return; + let type: unknown; + try { type = (JSON.parse(text) as { type?: unknown }).type; } catch { return; } + if (typeof type !== "string") return; + // Relay only the event surface the SSE path produces today. WS-only + // frames (codex.rate_limits, responsesapi.websocket_timing) are dropped + // so downstream clients see exactly the stream shape they always got. + if (!type.startsWith("response.") && type !== "error") return; + try { + controller.enqueue(encoder.encode(`event: ${type}\ndata: ${text}\n\n`)); + } catch { + return; + } + if (type === "response.completed" || type === "response.failed" || type === "response.incomplete" || type === "error") { + terminal = true; + try { controller.close(); } catch { /* already closed */ } + try { ws.close(); } catch { /* already closing */ } + } + }); + + ws.addEventListener("close", () => { + signal?.removeEventListener("abort", onAbort); + if (!opened) { + if (settledPreOpen) return; + settledPreOpen = true; + clearTimeout(upgradeTimer); + // Upgrade rejected (401/403/429/5xx). Retry over plain SSE so the real + // HTTP status reaches the existing refresh/rotation handlers. No turn + // started upstream, so the resend cannot double-generate. + resolve(sseFallback(url, init)); + return; + } + if (controller && !terminal) { + terminal = true; + // Connection dropped before a Responses terminal event. A clean EOF + // here would reach clients with no response.completed/failed at all — + // relaySseWithFailedTail() only synthesizes a failed terminal when the + // body read THROWS. Error the stream like a reset TCP socket. + try { controller.error(new Error("codex websocket closed before a Responses terminal event")); } catch { /* stream already done */ } + } + }); + + ws.addEventListener("error", () => { + /* Bun always follows error with close; the close handler settles. */ + }); + }); +} diff --git a/tests/ws-upstream.test.ts b/tests/ws-upstream.test.ts new file mode 100644 index 0000000000..b58472eabb --- /dev/null +++ b/tests/ws-upstream.test.ts @@ -0,0 +1,314 @@ +import { afterEach, describe, expect, jest, test } from "bun:test"; +import { providerFetch } from "../src/server/responses/fetch-helpers"; +import { codexWsUpstreamFetch, shouldUseCodexWsUpstream } from "../src/server/responses/ws-upstream"; +import type { OcxProviderConfig } from "../src/types"; + +const CODEX_URL = "https://chatgpt.com/backend-api/codex/responses"; + +function streamingInit(body: Record = {}): RequestInit { + return { + method: "POST", + headers: { "content-type": "application/json", authorization: "Bearer test" }, + body: JSON.stringify({ model: "gpt-5.6-luna", stream: true, ...body }), + }; +} + +describe("shouldUseCodexWsUpstream", () => { + test("matches only streaming POSTs to the Codex backend", () => { + expect(shouldUseCodexWsUpstream(CODEX_URL, streamingInit())).toBe(true); + // Non-streaming turns keep HTTP: the WS path only speaks the event protocol. + expect(shouldUseCodexWsUpstream(CODEX_URL, { + method: "POST", + body: JSON.stringify({ model: "gpt-5.6-luna" }), + })).toBe(false); + expect(shouldUseCodexWsUpstream(CODEX_URL, { method: "GET" })).toBe(false); + expect(shouldUseCodexWsUpstream("https://api.openai.com/v1/responses", streamingInit())).toBe(false); + // Body must be the adapter's serialized string, not a stream. + expect(shouldUseCodexWsUpstream(CODEX_URL, { method: "POST", body: new Blob(["x"]) as unknown as string })).toBe(false); + }); + + test("requires a ROOT-level stream flag, not a serialized substring", () => { + // Nested stream:true must not flip the transport. + expect(shouldUseCodexWsUpstream(CODEX_URL, { + method: "POST", + body: JSON.stringify({ model: "gpt-5.6-luna", metadata: { stream: true } }), + })).toBe(false); + // Whitespace-formatted JSON still routes. + expect(shouldUseCodexWsUpstream(CODEX_URL, { + method: "POST", + body: "{\n \"model\": \"gpt-5.6-luna\",\n \"stream\" : true\n}", + })).toBe(true); + // Non-boolean stream values stay on HTTP. + expect(shouldUseCodexWsUpstream(CODEX_URL, { + method: "POST", + body: JSON.stringify({ stream: "true" }), + })).toBe(false); + // Malformed JSON stays on HTTP. + expect(shouldUseCodexWsUpstream(CODEX_URL, { method: "POST", body: "{\"stream\":true" })).toBe(false); + }); +}); + +type Listener = (event: unknown) => void; + +/** Minimal scriptable stand-in for Bun's WebSocket. */ +class FakeWebSocket { + static instances: FakeWebSocket[] = []; + static script: (ws: FakeWebSocket) => void = () => {}; + url: string; + sent: string[] = []; + closed = false; + listeners = new Map(); + + constructor(url: string) { + this.url = url; + FakeWebSocket.instances.push(this); + queueMicrotask(() => FakeWebSocket.script(this)); + } + + addEventListener(type: string, listener: Listener) { + const list = this.listeners.get(type) ?? []; + list.push(listener); + this.listeners.set(type, list); + } + + emit(type: string, event: unknown = {}) { + for (const listener of this.listeners.get(type) ?? []) listener(event); + } + + send(data: string) { + this.sent.push(data); + } + + close() { + if (this.closed) return; + this.closed = true; + this.emit("close", {}); + } +} + +const RealWebSocket = globalThis.WebSocket; + +afterEach(() => { + globalThis.WebSocket = RealWebSocket; + FakeWebSocket.instances = []; + FakeWebSocket.script = () => {}; +}); + +function installFake(script: (ws: FakeWebSocket) => void) { + FakeWebSocket.script = script; + globalThis.WebSocket = FakeWebSocket as unknown as typeof WebSocket; +} + +describe("providerFetch routing", () => { + test("routes eligible Codex streaming turns to WS and everything else to the base fetch", async () => { + installFake(ws => { + ws.emit("open", {}); + ws.emit("message", { data: JSON.stringify({ type: "response.completed", response: {} }) }); + }); + const baseCalls: string[] = []; + const sentinel = new Response("base"); + const provider = { + fetch: (async (input: unknown) => { + baseCalls.push(String(input)); + return sentinel.clone(); + }) as unknown as typeof fetch, + } as unknown as OcxProviderConfig; + const wrapped = providerFetch(provider); + + // Eligible: WS adapter serves it, base fetch untouched. + const wsResponse = await wrapped(CODEX_URL, streamingInit()); + expect(wsResponse.headers.get("content-type")).toContain("text/event-stream"); + expect(baseCalls).toHaveLength(0); + expect(FakeWebSocket.instances).toHaveLength(1); + + // Non-streaming body: base fetch. + await wrapped(CODEX_URL, { method: "POST", body: JSON.stringify({ model: "m" }) }); + // Different host: base fetch. + await wrapped("https://api.openai.com/v1/responses", streamingInit()); + // Request-object input: base fetch (WS path only handles string URLs). + await wrapped(new Request(CODEX_URL, streamingInit() as RequestInit)); + expect(baseCalls).toHaveLength(3); + expect(FakeWebSocket.instances).toHaveLength(1); + }); +}); + +describe("codexWsUpstreamFetch", () => { + test("relays event frames as an SSE response and sends one response.create frame", async () => { + installFake(ws => { + ws.emit("open", {}); + ws.emit("message", { data: JSON.stringify({ type: "codex.rate_limits", limits: {} }) }); + ws.emit("message", { data: JSON.stringify({ type: "response.created", response: { id: "r1" } }) }); + ws.emit("message", { data: JSON.stringify({ type: "response.output_text.delta", delta: "hi" }) }); + ws.emit("message", { data: JSON.stringify({ type: "response.completed", response: { id: "r1" } }) }); + }); + const fallback = () => { throw new Error("fallback must not run"); }; + const response = await codexWsUpstreamFetch(CODEX_URL, streamingInit(), fallback as unknown as typeof fetch); + + expect(response.status).toBe(200); + expect(response.headers.get("content-type")).toContain("text/event-stream"); + const text = await response.text(); + // WS-only frames are dropped so clients see the exact SSE surface they always got. + expect(text).not.toContain("codex.rate_limits"); + expect(text).toContain("event: response.created"); + expect(text).toContain('data: {"type":"response.output_text.delta","delta":"hi"}'); + expect(text).toContain("event: response.completed"); + + const ws = FakeWebSocket.instances[0]; + expect(ws.url).toBe("wss://chatgpt.com/backend-api/codex/responses"); + expect(ws.sent).toHaveLength(1); + const frame = JSON.parse(ws.sent[0]) as Record; + expect(frame.type).toBe("response.create"); + // The HTTP-only stream flag must not reach the WS create frame. + expect("stream" in frame).toBe(false); + expect(ws.closed).toBe(true); + }); + + test("a request body with a top-level type field cannot override the frame discriminator", async () => { + installFake(ws => { + ws.emit("open", {}); + ws.emit("message", { data: JSON.stringify({ type: "response.completed", response: {} }) }); + }); + await codexWsUpstreamFetch(CODEX_URL, streamingInit({ type: "evil.frame" }), (() => { + throw new Error("fallback must not run"); + }) as unknown as typeof fetch); + const frame = JSON.parse(FakeWebSocket.instances[0].sent[0]) as Record; + expect(frame.type).toBe("response.create"); + }); + + test("falls back to the HTTP fetch when the upgrade is rejected before open", async () => { + installFake(ws => ws.close()); + const sentinel = new Response("sse-fallback", { status: 429 }); + let fallbackCalls = 0; + const fallback = (async () => { + fallbackCalls += 1; + return sentinel; + }) as unknown as typeof fetch; + const response = await codexWsUpstreamFetch(CODEX_URL, streamingInit(), fallback); + // The real HTTP status must reach the existing refresh/rotation handlers. + expect(response).toBe(sentinel); + expect(fallbackCalls).toBe(1); + }); + + test("falls back to the HTTP fetch when the upgrade deadline elapses without open or close", async () => { + jest.useFakeTimers(); + try { + installFake(() => { /* handshake never settles */ }); + const sentinel = new Response("sse-timeout-fallback", { status: 200 }); + let fallbackCalls = 0; + const fallback = (async () => { + fallbackCalls += 1; + return sentinel; + }) as unknown as typeof fetch; + + const responsePromise = codexWsUpstreamFetch(CODEX_URL, streamingInit(), fallback); + expect(FakeWebSocket.instances).toHaveLength(1); + jest.advanceTimersByTime(10_000); + const response = await responsePromise; + + expect(response).toBe(sentinel); + expect(fallbackCalls).toBe(1); + expect(FakeWebSocket.instances[0].closed).toBe(true); + } finally { + jest.useRealTimers(); + } + }); + + test("falls back to the HTTP fetch when the frame send throws", async () => { + installFake(ws => { + ws.send = () => { throw new Error("socket write failed"); }; + ws.emit("open", {}); + }); + const sentinel = new Response("sse-after-send-failure", { status: 200 }); + let fallbackCalls = 0; + const fallback = (async () => { + fallbackCalls += 1; + return sentinel; + }) as unknown as typeof fetch; + // The frame never left the client, so no upstream turn started and the SSE + // resend is safe; a synthetic 200 with an errored body would bypass the + // pre-stream HTTP error/refresh/failover machinery. + const response = await codexWsUpstreamFetch(CODEX_URL, streamingInit(), fallback); + expect(response).toBe(sentinel); + expect(fallbackCalls).toBe(1); + expect(FakeWebSocket.instances[0].closed).toBe(true); + }); + + test("errors the stream when the socket drops before a Responses terminal event", async () => { + installFake(ws => { + ws.emit("open", {}); + ws.emit("message", { data: JSON.stringify({ type: "response.created", response: { id: "r1" } }) }); + ws.close(); + }); + const fallback = () => { throw new Error("fallback must not run after open"); }; + const response = await codexWsUpstreamFetch(CODEX_URL, streamingInit(), fallback as unknown as typeof fetch); + // A clean EOF here would let a terminal-less stream reach clients: + // relaySseWithFailedTail() only synthesizes response.failed when the body + // read throws. The read must therefore reject, like a reset TCP socket. + await expect(response.text()).rejects.toThrow("closed before a Responses terminal event"); + }); + + test("a mid-stream drop surfaces as a synthesized failed terminal through the passthrough relay", async () => { + const { relaySseWithFailedTail } = await import("../src/server/relay"); + installFake(ws => { + ws.emit("open", {}); + ws.emit("message", { data: JSON.stringify({ type: "response.created", response: { id: "r1" } }) }); + ws.emit("message", { data: JSON.stringify({ type: "response.output_text.delta", delta: "partial" }) }); + // Drop on a later tick: controller.error() discards chunks still queued, + // so a synchronous close would erase frames a real client had already + // received over the wire. + setTimeout(() => ws.close(), 10); + }); + const response = await codexWsUpstreamFetch(CODEX_URL, streamingInit(), (() => { + throw new Error("fallback must not run after open"); + }) as unknown as typeof fetch); + const relayed = relaySseWithFailedTail(response.body!, new AbortController()); + const text = await new Response(relayed).text(); + expect(text).toContain("event: response.created"); + // The relay converts the erroring read into a failed terminal + [DONE], so + // no client ever sees a terminal-less stream. + expect(text).toContain("event: response.failed"); + expect(text).toContain("data: [DONE]"); + }); + + test("preserves caller headers on the handshake without fabricating an originator", async () => { + const seen: Record[] = []; + FakeWebSocket.script = ws => { + ws.emit("open", {}); + ws.emit("message", { data: JSON.stringify({ type: "response.completed", response: {} }) }); + }; + class HeaderCapturingWebSocket extends FakeWebSocket { + constructor(url: string, options?: { headers?: Record }) { + super(url); + seen.push(options?.headers ?? {}); + } + } + globalThis.WebSocket = HeaderCapturingWebSocket as unknown as typeof WebSocket; + const fallback = (() => { throw new Error("fallback must not run"); }) as unknown as typeof fetch; + + await codexWsUpstreamFetch(CODEX_URL, streamingInit(), fallback); + // Without a caller originator none is invented: pool/forward traffic must + // not impersonate Codex CLI (metadata-integrity contract). + expect(seen[0].originator).toBeUndefined(); + expect(seen[0]["openai-beta"]).toContain("responses_websockets"); + expect(seen[0].authorization).toBe("Bearer test"); + // HTTP body-framing headers do not belong on a WS handshake. + expect(seen[0]["content-type"]).toBeUndefined(); + + // A genuine caller originator is forwarded verbatim. + await codexWsUpstreamFetch(CODEX_URL, { + ...streamingInit(), + headers: { ...streamingInit().headers as Record, originator: "codex_cli_rs" }, + }, fallback); + expect(seen[1].originator).toBe("codex_cli_rs"); + }); + + test("aborting before open rejects like an aborted fetch", async () => { + installFake(() => { /* never opens */ }); + const controller = new AbortController(); + const promise = codexWsUpstreamFetch(CODEX_URL, { ...streamingInit(), signal: controller.signal }, (() => { + throw new Error("fallback must not run"); + }) as unknown as typeof fetch); + controller.abort(); + await expect(promise).rejects.toThrow(); + }); +});