|
41 | 41 | // Level-2 hook in packages/opencode is a thin adapter. |
42 | 42 |
|
43 | 43 | import fs from "fs/promises" |
| 44 | +import path from "path" |
44 | 45 | import { Effect, Schema } from "effect" |
45 | 46 | import { SessionStore } from "./session-store" |
46 | 47 | import { Skills } from "./skills" |
@@ -78,14 +79,44 @@ export interface ExecuteContext { |
78 | 79 | readonly onChunk?: (output: string) => Effect.Effect<void> |
79 | 80 | } |
80 | 81 |
|
| 82 | +// One screenshot collected during an execute() call. Drained into the |
| 83 | +// Level-2 wrapper's `attachments[]` so the agent sees the image inline on the |
| 84 | +// next assistant turn — no decode/write/read dance from inside the snippet. |
| 85 | +export interface CollectedScreenshot { |
| 86 | + readonly mime: "image/png" | "image/jpeg" | "image/webp" |
| 87 | + readonly base64: string |
| 88 | +} |
| 89 | + |
81 | 90 | export interface ExecuteResult { |
82 | 91 | readonly output: string |
83 | 92 | // The snippet's `return` value, JSON-serialized when possible. `undefined` |
84 | 93 | // serializes as `null` (JSON has no undefined). Non-serializable values |
85 | 94 | // fall back to `String(v)`. |
86 | 95 | readonly result: string |
| 96 | + // Every successful `Page.captureScreenshot` made by the snippet, in the |
| 97 | + // order the CDP responses came back. Empty when the snippet didn't take |
| 98 | + // any screenshots. |
| 99 | + readonly screenshots: readonly CollectedScreenshot[] |
| 100 | +} |
| 101 | + |
| 102 | +const SCREENSHOT_FORMAT_TO_MIME: Record<string, CollectedScreenshot["mime"]> = { |
| 103 | + png: "image/png", |
| 104 | + jpeg: "image/jpeg", |
| 105 | + webp: "image/webp", |
| 106 | +} |
| 107 | + |
| 108 | +const SCREENSHOT_FORMAT_TO_EXT: Record<string, string> = { |
| 109 | + png: "png", |
| 110 | + jpeg: "jpg", |
| 111 | + webp: "webp", |
87 | 112 | } |
88 | 113 |
|
| 114 | +const screenshotMime = (format: unknown): CollectedScreenshot["mime"] => |
| 115 | + SCREENSHOT_FORMAT_TO_MIME[typeof format === "string" ? format : "png"] ?? "image/png" |
| 116 | + |
| 117 | +const screenshotExt = (format: unknown): string => |
| 118 | + SCREENSHOT_FORMAT_TO_EXT[typeof format === "string" ? format : "png"] ?? "png" |
| 119 | + |
89 | 120 | // AsyncFunction is not a global — pull it off an async arrow's constructor. |
90 | 121 | const AsyncFunction = (async () => {}).constructor as new ( |
91 | 122 | ...args: string[] |
@@ -145,12 +176,46 @@ export const make = Effect.fn("BrowserExecute.make")(function* (dataDir: string) |
145 | 176 | debug: tee, |
146 | 177 | }) |
147 | 178 |
|
| 179 | + // Screenshot tap. Subscribes to the Session's call-result stream for |
| 180 | + // the duration of this execute() call; every successful |
| 181 | + // `Page.captureScreenshot` is collected (drained into `attachments[]` |
| 182 | + // by the Level-2 wrapper so the agent sees the image inline) and, |
| 183 | + // when `BCODE_SCREENSHOT_DIR` is set, also written to disk for |
| 184 | + // eval-judge consumption. Two consumers of one tap. |
| 185 | + // |
| 186 | + // Concurrency note: parallel execute() calls against the same Session |
| 187 | + // (rare but possible — different sessionIDs share no Session, but a |
| 188 | + // single sessionID with two in-flight tool calls would) each subscribe |
| 189 | + // independently and would each see all screenshots produced during |
| 190 | + // their lifetime. Acceptable for v1; opencode tool calls within one |
| 191 | + // assistant message are serialized anyway. |
| 192 | + const screenshots: CollectedScreenshot[] = [] |
| 193 | + const dumpDir = process.env.BCODE_SCREENSHOT_DIR |
| 194 | + const startedAt = Date.now() |
| 195 | + let seq = 0 |
| 196 | + const unsubscribe = session.onCallResult((method, params, result) => { |
| 197 | + if (method !== "Page.captureScreenshot") return |
| 198 | + const r = result as { data?: unknown } |
| 199 | + if (typeof r?.data !== "string") return |
| 200 | + const p = (params ?? {}) as { format?: unknown } |
| 201 | + const mime = screenshotMime(p.format) |
| 202 | + const ext = screenshotExt(p.format) |
| 203 | + const idx = seq++ |
| 204 | + screenshots.push({ mime, base64: r.data }) |
| 205 | + if (dumpDir) { |
| 206 | + const filename = `${ctx.sessionID}-${startedAt}-${String(idx).padStart(3, "0")}.${ext}` |
| 207 | + fs.mkdir(dumpDir, { recursive: true }) |
| 208 | + .then(() => fs.writeFile(path.join(dumpDir, filename), Buffer.from(r.data as string, "base64"))) |
| 209 | + .catch(() => { /* eval-side dump is best-effort */ }) |
| 210 | + } |
| 211 | + }) |
| 212 | + |
148 | 213 | const ran = yield* Effect.tryPromise({ |
149 | 214 | try: () => wrapped(session, snippetConsole), |
150 | 215 | catch: (err) => new Error(`browser_execute snippet threw: ${err instanceof Error ? err.stack ?? err.message : String(err)}`), |
151 | | - }) |
| 216 | + }).pipe(Effect.ensuring(Effect.sync(() => unsubscribe()))) |
152 | 217 |
|
153 | | - return { output, result: serialize(ran) } satisfies ExecuteResult |
| 218 | + return { output, result: serialize(ran), screenshots } satisfies ExecuteResult |
154 | 219 | }).pipe( |
155 | 220 | Effect.scoped, |
156 | 221 | Effect.timeoutOrElse({ |
|
0 commit comments