diff --git a/backend/src/http.ts b/backend/src/http.ts index 26d65a18c..a65d9e921 100644 --- a/backend/src/http.ts +++ b/backend/src/http.ts @@ -1,3 +1,5 @@ +import { createHash } from "node:crypto"; + import type { Server } from "bun"; import { type AuthSession, type AuthUser, getAuthSessionFromSessionId } from "./auth.ts"; @@ -56,6 +58,38 @@ export function json(data: unknown, init: BunResponseInit = {}): Response { return Response.json(data, { ...init, headers }); } +function hasMatchingEtag(request: Request, etag: string): boolean { + const candidates = request.headers.get("if-none-match"); + if (!candidates) return false; + const normalizedEtag = etag.replace(/^W\//u, ""); + return candidates.split(",").some((candidate) => { + const normalizedCandidate = candidate.trim().replace(/^W\//u, ""); + return normalizedCandidate === "*" || normalizedCandidate === normalizedEtag; + }); +} + +/** + * Serves private JSON with a strong validator so repeated polling can reuse the + * browser's response body even when it still revalidates with the backend. + */ +export function jsonWithEtag( + request: Request, + data: unknown, + init: BunResponseInit = {} +): Response { + const body = JSON.stringify(data); + const etag = `"${createHash("sha256").update(body).digest("base64url")}"`; + const headers = new Headers(init.headers); + headers.set("Cache-Control", "private, no-cache"); + headers.set("Content-Type", "application/json"); + headers.set("ETag", etag); + headers.set("Vary", "Cookie, Authorization"); + if (hasMatchingEtag(request, etag)) { + return new Response(undefined, { ...init, headers, status: 304 }); + } + return new Response(body, { ...init, headers }); +} + export function text( body: string, { diff --git a/backend/src/lib/coalescedSnapshot.ts b/backend/src/lib/coalescedSnapshot.ts new file mode 100644 index 000000000..a26a9c546 --- /dev/null +++ b/backend/src/lib/coalescedSnapshot.ts @@ -0,0 +1,245 @@ +export interface CoalescedSnapshotMetrics { + activeLoads: number; + averageLoadMs: number; + coalescedHits: number; + failures: number; + freshHits: number; + lastLoadMs: number; + loads: number; + name: string; + requests: number; + staleHits: number; +} + +interface CoalescedSnapshotOptions { + freshForMs: number; + load: () => Promise; + name: string; + now?: () => number; + retryAfterMs?: number; + staleForMs: number; +} + +interface SnapshotEntry { + inFlight?: Promise; + lastFailure?: { error: unknown }; + loadedAt?: number; + nextRetryAt?: number; + value?: T; +} + +interface MutableSnapshotMetrics { + activeLoads: number; + coalescedHits: number; + failures: number; + freshHits: number; + lastLoadMs: number; + loads: number; + requests: number; + staleHits: number; + totalLoadMs: number; +} + +const snapshotRegistry = new Map< + string, + { + metrics: MutableSnapshotMetrics; + name: string; + reset: () => void; + } +>(); + +function emptyMetrics(): MutableSnapshotMetrics { + return { + activeLoads: 0, + coalescedHits: 0, + failures: 0, + freshHits: 0, + lastLoadMs: 0, + loads: 0, + requests: 0, + staleHits: 0, + totalLoadMs: 0, + }; +} + +function positiveDuration(value: number, name: string): number { + if (!Number.isFinite(value) || value <= 0) { + throw new TypeError(`${name} must be a positive finite number`); + } + return value; +} + +/** + * Shares one read-only producer across callers and keeps a bounded stale value + * available while the next snapshot is refreshed. + */ +export class CoalescedSnapshot { + #entry: SnapshotEntry = {}; + #metricsGeneration = 0; + readonly #freshForMs: number; + readonly #load: () => Promise; + readonly #metrics = emptyMetrics(); + readonly #name: string; + readonly #now: () => number; + readonly #retryAfterMs: number; + readonly #staleForMs: number; + + constructor(options: CoalescedSnapshotOptions) { + const name = options.name.trim(); + if (!name) throw new TypeError("Snapshot name is required"); + this.#name = name; + this.#freshForMs = positiveDuration(options.freshForMs, "freshForMs"); + this.#staleForMs = positiveDuration(options.staleForMs, "staleForMs"); + if (this.#staleForMs < this.#freshForMs) { + throw new TypeError("staleForMs must be greater than or equal to freshForMs"); + } + this.#retryAfterMs = positiveDuration( + options.retryAfterMs ?? options.freshForMs, + "retryAfterMs" + ); + this.#load = options.load; + this.#now = options.now ?? Date.now; + snapshotRegistry.set(name, { + metrics: this.#metrics, + name, + reset: () => this.reset(), + }); + } + + #startLoad(entry: SnapshotEntry): Promise { + const startedAt = this.#now(); + const metricsGeneration = this.#metricsGeneration; + this.#metrics.activeLoads += 1; + this.#metrics.loads += 1; + + const load = async () => { + try { + const value = await this.#load(); + if (this.#entry === entry) { + entry.lastFailure = undefined; + entry.loadedAt = this.#now(); + entry.nextRetryAt = undefined; + entry.value = value; + } + return value; + } catch (error) { + if (this.#metricsGeneration === metricsGeneration) { + this.#metrics.failures += 1; + } + if (this.#entry === entry) { + entry.lastFailure = { error }; + entry.nextRetryAt = this.#now() + this.#retryAfterMs; + } + throw error; + } + }; + const inFlight = load(); + entry.inFlight = inFlight; + const recordSettlement = async () => { + try { + await inFlight; + } catch { + // The reader or background refresh handler observes the load error. + } finally { + const elapsedMs = Math.max(0, this.#now() - startedAt); + if (this.#metricsGeneration === metricsGeneration) { + this.#metrics.activeLoads = Math.max( + 0, + this.#metrics.activeLoads - 1 + ); + this.#metrics.lastLoadMs = elapsedMs; + this.#metrics.totalLoadMs += elapsedMs; + } + if (this.#entry === entry && entry.inFlight === inFlight) { + entry.inFlight = undefined; + } + } + }; + void recordSettlement(); + return inFlight; + } + + /** Returns the shared snapshot for this fixed read path. */ + async read(): Promise { + this.#metrics.requests += 1; + const entry = this.#entry; + const now = this.#now(); + const age = + entry.value === undefined || entry.loadedAt === undefined + ? Infinity + : Math.max(0, now - entry.loadedAt); + + if (age <= this.#freshForMs && entry.value !== undefined) { + this.#metrics.freshHits += 1; + return entry.value; + } + + if (age <= this.#staleForMs && entry.value !== undefined) { + this.#metrics.staleHits += 1; + if (entry.inFlight) { + this.#metrics.coalescedHits += 1; + } else if ((entry.nextRetryAt ?? 0) <= now) { + const refresh = this.#startLoad(entry); + void refresh.catch((error: unknown) => { + console.warn( + `[PollingSnapshot:${this.#name}] Background refresh failed`, + error + ); + }); + } + return entry.value; + } + + if (entry.inFlight) { + this.#metrics.coalescedHits += 1; + return await entry.inFlight; + } + + if ((entry.nextRetryAt ?? 0) > now && entry.lastFailure) { + throw entry.lastFailure.error; + } + + return await this.#startLoad(entry); + } + + /** Invalidates the visible value and detaches any older in-flight producer. */ + invalidate(): void { + this.#entry = {}; + } + + /** Clears cached values and counters. Intended for deterministic tests. */ + reset(): void { + this.invalidate(); + this.#metricsGeneration += 1; + Object.assign(this.#metrics, emptyMetrics()); + } +} + +/** Returns process-local coalescing telemetry without cached payloads or keys. */ +export function getCoalescedSnapshotMetrics(): CoalescedSnapshotMetrics[] { + return snapshotRegistry + .values() + .map(({ metrics, name }) => ({ + activeLoads: metrics.activeLoads, + averageLoadMs: + metrics.loads === 0 + ? 0 + : Math.round((metrics.totalLoadMs / metrics.loads) * 100) / 100, + coalescedHits: metrics.coalescedHits, + failures: metrics.failures, + freshHits: metrics.freshHits, + lastLoadMs: metrics.lastLoadMs, + loads: metrics.loads, + name, + requests: metrics.requests, + staleHits: metrics.staleHits, + })) + .toArray() + .toSorted((left, right) => left.name.localeCompare(right.name)); +} + +/** Resets every registered snapshot between tests. */ +export function resetCoalescedSnapshotsForTests(): void { + for (const snapshot of snapshotRegistry.values()) snapshot.reset(); +} diff --git a/backend/src/lib/processes.ts b/backend/src/lib/processes.ts index 8b5549657..983f39bf6 100644 --- a/backend/src/lib/processes.ts +++ b/backend/src/lib/processes.ts @@ -16,10 +16,23 @@ export interface RunProcessResult { stdout: string; } +export interface ChildProcessMetrics { + active: number; + failed: number; + started: number; + succeeded: number; +} + export type BunProcess = ReturnType; const DEFAULT_MAX_BUFFER = 10 * 1024 * 1024; const DEFAULT_FORCE_KILL_GRACE_MS = 3000; +const childProcessMetrics: ChildProcessMetrics = { + active: 0, + failed: 0, + started: 0, + succeeded: 0, +}; /** Returns the absolute Bun executable already running the Dashboard process. */ export function resolveBunExecutable(): string { @@ -64,7 +77,7 @@ export function spawnProcess( throw new DOMException("Process aborted before start", "AbortError"); } const command = scopedJobProcessCommand(executable, arguments_); - return Bun.spawn({ + const process = Bun.spawn({ cmd: [command.executable, ...command.arguments], cwd: options.cwd, detached: options.detached ?? true, @@ -73,6 +86,28 @@ export function spawnProcess( stdin: "ignore", stdout: "pipe", }); + childProcessMetrics.active += 1; + childProcessMetrics.started += 1; + void process.exited + .then((code) => { + if (code === 0) { + childProcessMetrics.succeeded += 1; + } else { + childProcessMetrics.failed += 1; + } + }) + .catch(() => { + childProcessMetrics.failed += 1; + }) + .finally(() => { + childProcessMetrics.active = Math.max(0, childProcessMetrics.active - 1); + }); + return process; +} + +/** Returns aggregate process telemetry without command arguments or environment data. */ +export function getChildProcessMetrics(): ChildProcessMetrics { + return { ...childProcessMetrics }; } export function killProcessGroup(process_: BunProcess, signal: NodeJS.Signals): void { diff --git a/backend/src/routes/agentRoutes.ts b/backend/src/routes/agentRoutes.ts index e785c6c74..c085f841e 100644 --- a/backend/src/routes/agentRoutes.ts +++ b/backend/src/routes/agentRoutes.ts @@ -1,4 +1,5 @@ -import { json, readJson } from "../http.ts"; +import { HttpError, json, readJson } from "../http.ts"; +import { CoalescedSnapshot } from "../lib/coalescedSnapshot.ts"; import { errorMessage, httpStatusCode } from "../lib/errors.ts"; import { buildAgentStatuses, @@ -23,6 +24,24 @@ function missingConfig(): Response { return json({ error: "Agent configuration not found" }, { status: 404 }); } +const agentStatusesSnapshot = new CoalescedSnapshot<{ + agents: Awaited>; + timestamp: number; +}>({ + freshForMs: 1500, + load: async () => { + closeStaleActiveTasks(); + const config = parseAgentsConfig(); + if (!config) throw new HttpError("Agent configuration not found", 404); + return { + agents: await buildAgentStatuses(config), + timestamp: Date.now(), + }; + }, + name: "openclaw.agent-statuses", + staleForMs: 5000, +}); + export const agentRoutes = { "/api/agents/:id/metadata": { PUT: async (request: ParametersRequest<"id">) => { @@ -37,7 +56,11 @@ export const agentRoutes = { if (!body || typeof body !== "object" || Array.isArray(body)) { return json({ error: "Missing or invalid body" }, { status: 400 }); } - return json(await updateAgentCurrentTask(agentId, body?.currentTask)); + try { + return json(await updateAgentCurrentTask(agentId, body?.currentTask)); + } finally { + agentStatusesSnapshot.invalidate(); + } } catch (error) { return agentError(error, "Agent metadata update failed"); } @@ -79,13 +102,7 @@ export const agentRoutes = { "/api/agents/status": { GET: async () => { try { - closeStaleActiveTasks(); - const config = parseAgentsConfig(); - if (!config) return missingConfig(); - return json({ - agents: await buildAgentStatuses(config), - timestamp: Date.now(), - }); + return json(await agentStatusesSnapshot.read()); } catch (error) { return agentError(error, "Agent status failed"); } diff --git a/backend/src/routes/cronRoutes.ts b/backend/src/routes/cronRoutes.ts index 69cc21d26..0977bcf13 100644 --- a/backend/src/routes/cronRoutes.ts +++ b/backend/src/routes/cronRoutes.ts @@ -10,6 +10,11 @@ import { getOpenClawCronDisableIntent, setOpenClawCronDisableIntent, } from "../services/openClawCronMetadata.ts"; +import { + getOpenClawCronListSnapshot, + invalidateOpenClawCronListSnapshot, + normalizeOpenClawCronJobs, +} from "../services/openClawCronSnapshot.ts"; import { withCronTaskLinks } from "../services/taskAutomation.ts"; type ParametersRequest = Request & { params: Record }; @@ -25,19 +30,6 @@ interface CronJob { [key: string]: unknown; } -interface CronListResponse { - items?: CronJob[]; - jobs?: CronJob[]; -} - -function normalizeJobs(payload: unknown): CronJob[] { - if (!payload || typeof payload !== "object") return []; - const value = payload as CronListResponse; - if (Array.isArray(value.jobs)) return value.jobs; - if (Array.isArray(value.items)) return value.items; - return []; -} - function cronError(error: unknown, fallback: string): Response { return json( { error: errorMessage(error, fallback) }, @@ -45,6 +37,14 @@ function cronError(error: unknown, fallback: string): Response { ); } +async function runCronMutation(operation: () => Promise): Promise { + try { + return await operation(); + } finally { + invalidateOpenClawCronListSnapshot(); + } +} + async function updateCronWithDisableIntent( jobId: string, patch: Record, @@ -71,10 +71,10 @@ export const cronRoutes = { "/api/cron/jobs": { GET: async () => { try { - const payload = await gateway.request("cron.list", { - includeDisabled: true, + const payload = await getOpenClawCronListSnapshot(); + return json({ + jobs: withCronTaskLinks(normalizeOpenClawCronJobs(payload)), }); - return json({ jobs: withCronTaskLinks(normalizeJobs(payload)) }); } catch (error) { return cronError(error, "Failed to list cron jobs"); } @@ -87,9 +87,11 @@ export const cronRoutes = { try { previousIntent = getOpenClawCronDisableIntent(request.params.id); setOpenClawCronDisableIntent(request.params.id, undefined); - const payload = await gateway.request("cron.remove", { - jobId: request.params.id, - }); + const payload = await runCronMutation(() => + gateway.request("cron.remove", { + jobId: request.params.id, + }) + ); return json({ isOk: true, payload }); } catch (error) { try { @@ -108,9 +110,11 @@ export const cronRoutes = { "/api/cron/jobs/:id/run": { POST: async (request: ParametersRequest<"id">) => { try { - const payload = await gateway.request("cron.run", { - jobId: request.params.id, - }); + const payload = await runCronMutation(() => + gateway.request("cron.run", { + jobId: request.params.id, + }) + ); return json({ isOk: true, payload }); } catch (error) { return cronError(error, "Failed to run cron job"); @@ -144,10 +148,12 @@ export const cronRoutes = { ? undefined : normalizeJobDisableIntent(body.disableIntent); if (disableIntent) assertJobDisableIntentIsCurrent(disableIntent); - await updateCronWithDisableIntent( - request.params.id, - { enabled: body.enabled }, - disableIntent + await runCronMutation(() => + updateCronWithDisableIntent( + request.params.id, + { enabled: body.enabled }, + disableIntent + ) ); return json({ isOk: true }); } catch (error) { @@ -172,16 +178,20 @@ export const cronRoutes = { } const cronPatch = patch as Record; if (cronPatch.enabled === true) { - await updateCronWithDisableIntent( - request.params.id, - cronPatch, - undefined + await runCronMutation(() => + updateCronWithDisableIntent( + request.params.id, + cronPatch, + undefined + ) ); } else { - await gateway.request("cron.update", { - jobId: request.params.id, - patch: cronPatch, - }); + await runCronMutation(() => + gateway.request("cron.update", { + jobId: request.params.id, + patch: cronPatch, + }) + ); } return json({ isOk: true }); } catch (error) { diff --git a/backend/src/routes/dockerRoutes.ts b/backend/src/routes/dockerRoutes.ts index aac75788c..33c34fd69 100644 --- a/backend/src/routes/dockerRoutes.ts +++ b/backend/src/routes/dockerRoutes.ts @@ -1,5 +1,6 @@ import { database } from "../database.ts"; -import { json, readJson } from "../http.ts"; +import { json, jsonWithEtag, readJson } from "../http.ts"; +import { CoalescedSnapshot } from "../lib/coalescedSnapshot.ts"; import { errorMessage, httpStatusCode } from "../lib/errors.ts"; import { runProcess } from "../lib/processes.ts"; import { @@ -306,12 +307,12 @@ async function getContainerInspectMap(containerIds: string[]) { return map; } -export async function getContainers() { +export async function getContainers(statsRows?: DockerStatsRow[]) { const psRows = parseJsonLines( await runDocker(["ps", "-a", "--format", "{{json .}}"]) ); - const statsRows = await getContainerStatsRows(); - const statsById = new Map(statsRows.map((row) => [row.ID, row])); + const resolvedStatsRows = statsRows ?? (await getContainerStatsRows()); + const statsById = new Map(resolvedStatsRows.map((row) => [row.ID, row])); const inspectMap = await getContainerInspectMap(psRows.map((row) => row.ID)); return psRows.map((row) => { @@ -372,6 +373,25 @@ export async function getContainerStatsRows() { ); } +async function getContainerLogs(containerId: string, tail: number): Promise { + const { code, stderr, stdout } = await runProcess( + dockerBin, + ["logs", "--tail", String(tail), containerId], + { + cwd: getDockerRoot(), + env: process.env, + maxBuffer: 10 * 1024 * 1024, + timeoutMs: DOCKER_REQUEST_TIMEOUT_MS, + } + ); + if (code !== 0) { + throw new Error( + `docker logs failed with exit code ${code}: ${stderr.trim() || stdout.trim()}` + ); + } + return [String(stdout), String(stderr)].filter(Boolean).join("\n").trim(); +} + async function getContainerDetails(containerId: string) { const containers = await getContainers(); const summary = findContainerSummary(containers, containerId); @@ -661,12 +681,21 @@ async function runQueuedDockerAction(options: { timeoutMs: options.timeoutMs, }); return successfulJobExecutionOutput( - await waitForJobExecution(execution.id, { - timeoutMs: options.timeoutMs + 30 * 60 * 1000, - }) + await waitForDockerMutationExecution( + execution.id, + options.timeoutMs + 30 * 60 * 1000 + ) ); } +async function waitForDockerMutationExecution(executionId: string, timeoutMs: number) { + try { + return await waitForJobExecution(executionId, { timeoutMs }); + } finally { + invalidateDockerReadSnapshots(); + } +} + function outputString(output: Record, key: string): string { return typeof output[key] === "string" ? output[key] : ""; } @@ -709,15 +738,50 @@ async function runStackAction(request: Request): Promise { }); } +const dockerStatsSnapshot = new CoalescedSnapshot< + Awaited> +>({ + freshForMs: 2000, + load: getContainerStatsRows, + name: "docker.stats", + staleForMs: 15_000, +}); + +const dockerStateSnapshot = new CoalescedSnapshot< + Awaited> +>({ + freshForMs: 2000, + load: async () => getContainers(await dockerStatsSnapshot.read()), + name: "docker.state", + staleForMs: 15_000, +}); + +/** Returns the shared read-only container sampler for polling routes. */ +export async function getDockerContainersSnapshot() { + return await dockerStateSnapshot.read(); +} + +function invalidateDockerReadSnapshots(): void { + dockerStateSnapshot.invalidate(); + dockerStatsSnapshot.invalidate(); +} + +function dockerSnapshotJson(request: Request | undefined, data: unknown): Response { + return request ? jsonWithEtag(request, data) : json(data); +} + export const dockerRoutes = { "/api/docker/containers": { - GET: async () => json({ containers: await getContainers() }), + GET: async (request?: Request) => + dockerSnapshotJson(request, { + containers: await getDockerContainersSnapshot(), + }), }, "/api/docker/containers/stats": { - GET: async () => { - const rows = await getContainerStatsRows(); - return json({ - stats: rows.map((row) => ({ + GET: async (request?: Request) => { + const statsRows = await dockerStatsSnapshot.read(); + return dockerSnapshotJson(request, { + stats: statsRows.map((row) => ({ blockIO: row.BlockIO, cpu: row.CPUPerc, id: row.ID, @@ -771,29 +835,7 @@ export const dockerRoutes = { if (!containerId) return invalidDockerIdentifier("containerId"); const requestedTail = Math.trunc(queryNumber(request, "tail", 200)) || 200; const tail = Math.min(MAX_LOG_TAIL, Math.max(MIN_LOG_TAIL, requestedTail)); - const { code, stderr, stdout } = await runProcess( - dockerBin, - ["logs", "--tail", String(tail), containerId], - { - cwd: getDockerRoot(), - env: process.env, - maxBuffer: 10 * 1024 * 1024, - timeoutMs: DOCKER_REQUEST_TIMEOUT_MS, - } - ); - if (code !== 0) { - throw new Error( - `docker logs failed with exit code ${code}: ${ - stderr.trim() || stdout.trim() - }` - ); - } - return json({ - content: [String(stdout), String(stderr)] - .filter(Boolean) - .join("\n") - .trim(), - }); + return json({ content: await getContainerLogs(containerId, tail) }); }, }, "/api/docker/exec/:jobId": { @@ -962,9 +1004,9 @@ export const dockerRoutes = { POST: async () => { try { const scheduledRun = enqueueScheduledJob("docker.updater", "manual"); - const execution = await waitForJobExecution( + const execution = await waitForDockerMutationExecution( scheduledRun.executionId as string, - { timeoutMs: 60 * 60 * 1000 } + 60 * 60 * 1000 ); const steps = dockerUpdaterSteps(execution); return json({ @@ -1011,9 +1053,7 @@ export const dockerRoutes = { timeoutMs: 30 * 60 * 1000, }); steps = dockerUpdaterSteps( - await waitForJobExecution(execution.id, { - timeoutMs: 60 * 60 * 1000, - }) + await waitForDockerMutationExecution(execution.id, 60 * 60 * 1000) ); } catch (error) { return json( diff --git a/backend/src/routes/metricsRoutes.ts b/backend/src/routes/metricsRoutes.ts index 5d5ede111..4b07698f5 100644 --- a/backend/src/routes/metricsRoutes.ts +++ b/backend/src/routes/metricsRoutes.ts @@ -4,7 +4,16 @@ import path from "node:path"; import gateway from "../gateway.ts"; import { json } from "../http.ts"; -import { runProcess } from "../lib/processes.ts"; +import { + CoalescedSnapshot, + type CoalescedSnapshotMetrics, + getCoalescedSnapshotMetrics, +} from "../lib/coalescedSnapshot.ts"; +import { + type ChildProcessMetrics, + getChildProcessMetrics, + runProcess, +} from "../lib/processes.ts"; import { stringFallback } from "../lib/values.ts"; interface CpuMetrics { @@ -64,10 +73,16 @@ interface TokenMetrics { } interface MetricsResponse extends SystemMetricsResponse { + polling: { + snapshots: CoalescedSnapshotMetrics[]; + }; + processes: ChildProcessMetrics; tokens: TokenMetrics; } const PREFERRED_LINUX_NETWORK_INTERFACE = "enp0s6"; +const METRICS_FRESH_MS = 2000; +const METRICS_STALE_MS = 10_000; const metricsRouteState: { networkSampleLock: Promise; @@ -328,14 +343,25 @@ function getTokenMetrics(): TokenMetrics { }; } +const metricsSnapshot = new CoalescedSnapshot({ + freshForMs: METRICS_FRESH_MS, + load: async () => ({ + ...(await getSystemMetrics()), + polling: { + snapshots: getCoalescedSnapshotMetrics(), + }, + processes: getChildProcessMetrics(), + tokens: getTokenMetrics(), + }), + name: "system.metrics", + staleForMs: METRICS_STALE_MS, +}); + export const metricsRoutes = { "/api/metrics": { GET: async () => { try { - return json({ - ...(await getSystemMetrics()), - tokens: getTokenMetrics(), - } satisfies MetricsResponse); + return json(await metricsSnapshot.read()); } catch (error) { console.error("[Metrics] Failed to fetch metrics:", error); return json({ error: "Failed to fetch metrics" }, { status: 500 }); diff --git a/backend/src/routes/pullRequestRoutes.ts b/backend/src/routes/pullRequestRoutes.ts index 76959d6ad..d95373e84 100644 --- a/backend/src/routes/pullRequestRoutes.ts +++ b/backend/src/routes/pullRequestRoutes.ts @@ -1,4 +1,5 @@ -import { json, readJson } from "../http.ts"; +import { json, jsonWithEtag, readJson } from "../http.ts"; +import { CoalescedSnapshot } from "../lib/coalescedSnapshot.ts"; import { errorMessage, httpStatusCode } from "../lib/errors.ts"; import { getPullRequestPreviewStatus, @@ -40,13 +41,58 @@ function parsePullRequestNumber(value: unknown): number | Response { } } +const pullRequestPreviewSnapshot = new CoalescedSnapshot< + Awaited> +>({ + freshForMs: 2000, + load: getPullRequestPreviewStatus, + name: "git.pull-request-preview", + staleForMs: 10_000, +}); + +const pullRequestListSnapshot = new CoalescedSnapshot< + Awaited> +>({ + freshForMs: 15_000, + load: async () => { + const pullRequests = await listDashboardPullRequests(); + await reconcileClosedPullRequestPreview(pullRequests); + pullRequestPreviewSnapshot.invalidate(); + return pullRequests; + }, + name: "github.pull-requests", + staleForMs: 120_000, +}); + +const productionCheckoutSnapshot = new CoalescedSnapshot< + Awaited> +>({ + freshForMs: 3000, + load: () => getProductionCheckoutStatus(), + name: "git.production-checkout", + staleForMs: 30_000, +}); + +function pullRequestSnapshotJson(request: Request | undefined, data: unknown): Response { + return request ? jsonWithEtag(request, data) : json(data); +} + +async function runPullRequestMutation(operation: () => Promise): Promise { + try { + return await operation(); + } finally { + pullRequestListSnapshot.invalidate(); + productionCheckoutSnapshot.invalidate(); + } +} + export const pullRequestRoutes = { "/api/pull-requests": { - GET: async () => { + GET: async (request?: Request) => { try { - const pullRequests = await listDashboardPullRequests(); - await reconcileClosedPullRequestPreview(pullRequests); - return json({ pullRequests }); + return pullRequestSnapshotJson(request, { + pullRequests: await pullRequestListSnapshot.read(), + }); } catch (error) { return routeError(error); } @@ -60,7 +106,11 @@ export const pullRequestRoutes = { const body = request.body ? await readJson<{ deploy?: unknown } | undefined>(request) : undefined; - return json(await runPullRequestApproval(number, body?.deploy === true)); + return json( + await runPullRequestMutation(() => + runPullRequestApproval(number, body?.deploy === true) + ) + ); } catch (error) { return routeError(error); } @@ -78,7 +128,11 @@ export const pullRequestRoutes = { typeof body?.comment === "string" && body.comment.trim() ? body.comment.trim() : "Closed from Mira Dashboard after Rajohan rejected it."; - return json(await runPullRequestRejection(number, comment)); + return json( + await runPullRequestMutation(() => + runPullRequestRejection(number, comment) + ) + ); } catch (error) { return routeError(error); } @@ -89,7 +143,11 @@ export const pullRequestRoutes = { const number = parsePullRequestNumber(request.params.number); if (number instanceof Response) return number; try { - return json(await runPullRequestReviewApproval(number)); + return json( + await runPullRequestMutation(() => + runPullRequestReviewApproval(number) + ) + ); } catch (error) { return routeError(error); } @@ -100,7 +158,9 @@ export const pullRequestRoutes = { const number = parsePullRequestNumber(request.params.number); if (number instanceof Response) return number; try { - return json(await runPullRequestBranchUpdate(number)); + return json( + await runPullRequestMutation(() => runPullRequestBranchUpdate(number)) + ); } catch (error) { return routeError(error); } @@ -111,13 +171,17 @@ export const pullRequestRoutes = { const number = parsePullRequestNumber(request.params.number); if (number instanceof Response) return number; try { - return json( - { - isOk: true, - preview: await prepareAndStartPullRequestPreview(number), - }, - { status: 202 } - ); + try { + return json( + { + isOk: true, + preview: await prepareAndStartPullRequestPreview(number), + }, + { status: 202 } + ); + } finally { + pullRequestPreviewSnapshot.invalidate(); + } } catch (error) { return routeError(error, "PR preview startup failed"); } @@ -128,10 +192,14 @@ export const pullRequestRoutes = { const number = parsePullRequestNumber(request.params.number); if (number instanceof Response) return number; try { - return json({ - isOk: true, - preview: await prepareAndStopPullRequestPreview(number), - }); + try { + return json({ + isOk: true, + preview: await prepareAndStopPullRequestPreview(number), + }); + } finally { + pullRequestPreviewSnapshot.invalidate(); + } } catch (error) { return routeError(error, "PR preview stop failed"); } @@ -140,10 +208,12 @@ export const pullRequestRoutes = { "/api/pull-requests/deploy": { POST: async () => { try { - return json({ - deployment: await prepareAndStartDeployLatest(), - isOk: true, - }); + return json( + await runPullRequestMutation(async () => ({ + deployment: await prepareAndStartDeployLatest(), + isOk: true, + })) + ); } catch (error) { return routeError(error); } @@ -177,10 +247,13 @@ export const pullRequestRoutes = { { status: 400 } ); } - return json({ - deployment: await prepareAndStartRollback(body.targetCommit), - isOk: true, - }); + const targetCommit = body.targetCommit; + return json( + await runPullRequestMutation(async () => ({ + deployment: await prepareAndStartRollback(targetCommit), + isOk: true, + })) + ); } catch (error) { return routeError(error); } @@ -189,7 +262,7 @@ export const pullRequestRoutes = { "/api/pull-requests/production-checkout": { GET: async () => { try { - return json({ checkout: await getProductionCheckoutStatus() }); + return json({ checkout: await productionCheckoutSnapshot.read() }); } catch (error) { return routeError(error); } @@ -198,7 +271,7 @@ export const pullRequestRoutes = { "/api/pull-requests/preview": { GET: async () => { try { - return json({ preview: await getPullRequestPreviewStatus() }); + return json({ preview: await pullRequestPreviewSnapshot.read() }); } catch (error) { return routeError(error, "PR preview status failed"); } diff --git a/backend/src/routes/taskRoutes.ts b/backend/src/routes/taskRoutes.ts index 01a939c81..221a2b80f 100644 --- a/backend/src/routes/taskRoutes.ts +++ b/backend/src/routes/taskRoutes.ts @@ -9,6 +9,10 @@ import { HttpError, json, readJson } from "../http.ts"; import { errorMessage, httpStatusCode } from "../lib/errors.ts"; import { objectFallback } from "../lib/values.ts"; import { isDevelopmentExternalNotificationSuppressed } from "../requestPolicy.ts"; +import { + getOpenClawCronListSnapshot, + normalizeOpenClawCronJobs, +} from "../services/openClawCronSnapshot.ts"; type Status = "todo" | "in-progress" | "blocked" | "done"; type Assignee = TaskAssigneeId; @@ -200,14 +204,8 @@ function cronJobId(job: CronJob): string { async function fetchCronJobsById(): Promise> { try { - const payload = await gateway.request("cron.list", { includeDisabled: true }); - if (!payload || typeof payload !== "object") return new Map(); - const value = payload as { jobs?: CronJob[]; items?: CronJob[] }; - const jobs = Array.isArray(value.jobs) - ? value.jobs - : Array.isArray(value.items) - ? value.items - : []; + const payload = await getOpenClawCronListSnapshot(); + const jobs = normalizeOpenClawCronJobs(payload); return new Map( jobs .map((job) => [cronJobId(job), job] as const) diff --git a/backend/src/services/agents.ts b/backend/src/services/agents.ts index 17e909ee9..a8fc799bb 100644 --- a/backend/src/services/agents.ts +++ b/backend/src/services/agents.ts @@ -4,6 +4,7 @@ import Path from "node:path"; import { database } from "../database.ts"; import gateway from "../gateway.ts"; +import { CoalescedSnapshot } from "../lib/coalescedSnapshot.ts"; import { guardedPath, mkdirGuarded, @@ -405,7 +406,7 @@ function resolveConfiguredModelName( } /** Returns Gateway sessions for agent keys, preferring live Gateway data and falling back to cached files on failure. */ -async function getGatewaySessionsForAgents(): Promise { +async function loadGatewaySessionsForAgents(): Promise { const cached: GatewaySessionSummary[] = (() => { try { return gateway @@ -479,6 +480,17 @@ async function getGatewaySessionsForAgents(): Promise { return cached; } +const gatewayAgentSessionsSnapshot = new CoalescedSnapshot({ + freshForMs: 1500, + load: loadGatewaySessionsForAgents, + name: "openclaw.agent-sessions", + staleForMs: 10_000, +}); + +function getGatewaySessionsForAgents(): Promise { + return gatewayAgentSessionsSnapshot.read(); +} + /** Returns a millisecond timestamp for Gateway values that may already be numeric or ISO strings. */ function toTimestamp(value: unknown): number | undefined { if (typeof value === "number" && Number.isFinite(value)) { diff --git a/backend/src/services/openClawCronSnapshot.ts b/backend/src/services/openClawCronSnapshot.ts new file mode 100644 index 000000000..86b908261 --- /dev/null +++ b/backend/src/services/openClawCronSnapshot.ts @@ -0,0 +1,50 @@ +import gateway from "../gateway.ts"; +import { CoalescedSnapshot } from "../lib/coalescedSnapshot.ts"; + +const cronListSnapshot = new CoalescedSnapshot({ + freshForMs: 3000, + load: async () => { + const payload = await gateway.request("cron.list", { + includeDisabled: true, + }); + normalizeOpenClawCronJobs>(payload); + return payload; + }, + name: "openclaw.cron-list", + staleForMs: 30_000, +}); + +interface CronListResponse { + items?: T[]; + jobs?: T[]; +} + +/** Reads the shared raw Gateway cron list for route-specific normalization. */ +export function getOpenClawCronListSnapshot(): Promise { + return cronListSnapshot.read(); +} + +/** Extracts cron jobs while letting each consumer retain its own narrow job type. */ +export function normalizeOpenClawCronJobs(payload: unknown): T[] { + if (!payload || typeof payload !== "object" || Array.isArray(payload)) { + throw new TypeError("Invalid OpenClaw cron list response"); + } + const value = payload as CronListResponse; + const jobs = Array.isArray(value.jobs) + ? value.jobs + : Array.isArray(value.items) + ? value.items + : undefined; + if ( + !jobs || + jobs.some((job) => !job || typeof job !== "object" || Array.isArray(job)) + ) { + throw new TypeError("Invalid OpenClaw cron list response"); + } + return jobs; +} + +/** Invalidates cron state after any Gateway mutation. */ +export function invalidateOpenClawCronListSnapshot(): void { + cronListSnapshot.invalidate(); +} diff --git a/backend/src/services/taskAutomation.ts b/backend/src/services/taskAutomation.ts index 773e0ef03..e9a678ee2 100644 --- a/backend/src/services/taskAutomation.ts +++ b/backend/src/services/taskAutomation.ts @@ -1,9 +1,12 @@ import { TASK_ASSIGNEES, type TaskAssigneeId } from "../constants/taskActors.ts"; import { database } from "../database.ts"; -import gateway from "../gateway.ts"; import { errorMessage } from "../lib/errors.ts"; import type { JobDisableIntent } from "./jobDisableIntent.ts"; import { openClawCronDisableIntentsByJobId } from "./openClawCronMetadata.ts"; +import { + getOpenClawCronListSnapshot, + normalizeOpenClawCronJobs, +} from "./openClawCronSnapshot.ts"; export interface CronTaskLink { number: number; @@ -28,11 +31,6 @@ interface CronJob { [key: string]: unknown; } -interface CronListResponse { - items?: CronJob[]; - jobs?: CronJob[]; -} - interface HeartbeatTaskAutomation { cronJobId: string; missing?: boolean; @@ -106,13 +104,6 @@ function cronJobId(job: CronJob): string { return String(job.jobId || job.id || ""); } -function normalizedCronJobs(payload: unknown): CronJob[] { - if (!payload || typeof payload !== "object") return []; - const value = payload as CronListResponse; - if (Array.isArray(value.jobs)) return value.jobs; - return Array.isArray(value.items) ? value.items : []; -} - function openTaskAutomationRows(): TaskAutomationRow[] { return database .prepare( @@ -179,8 +170,8 @@ export async function getHeartbeatAutomationSnapshot(): Promise( + await getOpenClawCronListSnapshot() ); } catch (error) { isCronDataAvailable = false; diff --git a/backend/test/coalescedSnapshot.test.ts b/backend/test/coalescedSnapshot.test.ts new file mode 100644 index 000000000..b90c82492 --- /dev/null +++ b/backend/test/coalescedSnapshot.test.ts @@ -0,0 +1,138 @@ +import { describe, expect, it, jest } from "bun:test"; + +import { + CoalescedSnapshot, + getCoalescedSnapshotMetrics, +} from "../src/lib/coalescedSnapshot.ts"; + +describe("coalesced snapshots", () => { + it("shares a cold in-flight load across concurrent readers", async () => { + const load = Promise.withResolvers(); + const producer = jest.fn(() => load.promise); + const snapshot = new CoalescedSnapshot({ + freshForMs: 1000, + load: producer, + name: "test.cold-single-flight", + staleForMs: 5000, + }); + + const first = snapshot.read(); + const second = snapshot.read(); + await Promise.resolve(); + + expect(producer).toHaveBeenCalledTimes(1); + load.resolve("shared"); + await expect(Promise.all([first, second])).resolves.toEqual(["shared", "shared"]); + expect( + getCoalescedSnapshotMetrics().find( + (entry) => entry.name === "test.cold-single-flight" + ) + ).toMatchObject({ + coalescedHits: 1, + loads: 1, + requests: 2, + }); + }); + + it("serves bounded stale data while one background refresh runs", async () => { + let now = 1000; + const refresh = Promise.withResolvers(); + const producer = jest + .fn<() => Promise>() + .mockResolvedValueOnce("first") + .mockImplementationOnce(() => refresh.promise); + const snapshot = new CoalescedSnapshot({ + freshForMs: 500, + load: producer, + name: "test.stale-while-revalidate", + now: () => now, + staleForMs: 5000, + }); + + await expect(snapshot.read()).resolves.toBe("first"); + now = 2000; + await expect(snapshot.read()).resolves.toBe("first"); + await expect(snapshot.read()).resolves.toBe("first"); + expect(producer).toHaveBeenCalledTimes(2); + + now = 2100; + refresh.resolve("second"); + await refresh.promise; + await Promise.resolve(); + await expect(snapshot.read()).resolves.toBe("second"); + }); + + it("does not let an invalidated load repopulate a newer generation", async () => { + const oldLoad = Promise.withResolvers(); + const newLoad = Promise.withResolvers(); + const producer = jest + .fn<() => Promise>() + .mockImplementationOnce(() => oldLoad.promise) + .mockImplementationOnce(() => newLoad.promise); + const snapshot = new CoalescedSnapshot({ + freshForMs: 1000, + load: producer, + name: "test.invalidation-generation", + staleForMs: 5000, + }); + + const oldRead = snapshot.read(); + await Promise.resolve(); + snapshot.invalidate(); + const newRead = snapshot.read(); + await Promise.resolve(); + expect(producer).toHaveBeenCalledTimes(2); + + oldLoad.resolve("old"); + await expect(oldRead).resolves.toBe("old"); + newLoad.resolve("new"); + await expect(newRead).resolves.toBe("new"); + await expect(snapshot.read()).resolves.toBe("new"); + }); + + it("propagates refresh failures after the hard stale boundary", async () => { + let now = 1000; + const producer = jest + .fn<() => Promise>() + .mockResolvedValueOnce("first") + .mockRejectedValueOnce(new Error("producer unavailable")); + const snapshot = new CoalescedSnapshot({ + freshForMs: 500, + load: producer, + name: "test.hard-stale", + now: () => now, + staleForMs: 1000, + }); + + await expect(snapshot.read()).resolves.toBe("first"); + now = 2501; + await expect(snapshot.read()).rejects.toThrow("producer unavailable"); + }); + + it("reuses a cold failure until its retry delay expires", async () => { + let now = 1000; + const failure = new Error("producer unavailable"); + const producer = jest + .fn<() => Promise>() + .mockRejectedValueOnce(failure) + .mockResolvedValueOnce("recovered"); + const snapshot = new CoalescedSnapshot({ + freshForMs: 500, + load: producer, + name: "test.cold-failure-backoff", + now: () => now, + retryAfterMs: 5000, + staleForMs: 1000, + }); + + await expect(snapshot.read()).rejects.toBe(failure); + await Promise.resolve(); + now = 2000; + await expect(snapshot.read()).rejects.toBe(failure); + expect(producer).toHaveBeenCalledTimes(1); + + now = 6001; + await expect(snapshot.read()).resolves.toBe("recovered"); + expect(producer).toHaveBeenCalledTimes(2); + }); +}); diff --git a/backend/test/httpCaching.test.ts b/backend/test/httpCaching.test.ts new file mode 100644 index 000000000..9a4aa83a9 --- /dev/null +++ b/backend/test/httpCaching.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it } from "bun:test"; + +import { jsonWithEtag } from "../src/http.ts"; + +describe("private JSON validators", () => { + it("returns 304 for a matching ETag without making the response public", async () => { + const first = jsonWithEtag(new Request("https://dashboard.test/api/poll"), { + items: ["one"], + }); + const etag = first.headers.get("etag"); + + expect(first.status).toBe(200); + expect(etag).toMatch(/^"[A-Za-z0-9_-]+"$/u); + expect(first.headers.get("cache-control")).toBe("private, no-cache"); + expect(first.headers.get("vary")).toBe("Cookie, Authorization"); + await expect(first.json()).resolves.toEqual({ items: ["one"] }); + + const revalidated = jsonWithEtag( + new Request("https://dashboard.test/api/poll", { + headers: { "If-None-Match": `W/${etag}` }, + }), + { items: ["one"] } + ); + expect(revalidated.status).toBe(304); + await expect(revalidated.text()).resolves.toBe(""); + }); + + it("returns a new body when the validator no longer matches", async () => { + const response = jsonWithEtag( + new Request("https://dashboard.test/api/poll", { + headers: { "If-None-Match": '"old"' }, + }), + { items: ["new"] } + ); + + expect(response.status).toBe(200); + await expect(response.json()).resolves.toEqual({ items: ["new"] }); + }); +}); diff --git a/backend/test/openClawCronSnapshot.test.ts b/backend/test/openClawCronSnapshot.test.ts new file mode 100644 index 000000000..9ecd0a519 --- /dev/null +++ b/backend/test/openClawCronSnapshot.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it, jest } from "bun:test"; + +import gateway from "../src/gateway.ts"; +import { + getOpenClawCronListSnapshot, + normalizeOpenClawCronJobs, +} from "../src/services/openClawCronSnapshot.ts"; + +describe("OpenClaw cron snapshot", () => { + it("normalizes jobs before items and accepts a healthy empty list", () => { + expect( + normalizeOpenClawCronJobs<{ id: string }>({ + items: [{ id: "item" }], + jobs: [{ id: "job" }], + }) + ).toEqual([{ id: "job" }]); + expect( + normalizeOpenClawCronJobs<{ id: string }>({ + items: [{ id: "item" }], + jobs: "unavailable", + }) + ).toEqual([{ id: "item" }]); + expect(normalizeOpenClawCronJobs({ jobs: [] })).toEqual([]); + }); + + it("rejects malformed payloads and cron entries", () => { + for (const payload of [ + undefined, + [], + {}, + { jobs: [undefined] }, + { items: [[]] }, + { jobs: ["invalid"] }, + ]) { + expect(() => normalizeOpenClawCronJobs(payload)).toThrow( + "Invalid OpenClaw cron list response" + ); + } + }); + + it("records malformed Gateway responses as snapshot load failures", async () => { + jest.spyOn(gateway, "request").mockResolvedValue({}); + + await expect(getOpenClawCronListSnapshot()).rejects.toThrow( + "Invalid OpenClaw cron list response" + ); + }); +}); diff --git a/backend/test/routeAndServiceBehavior.test.ts b/backend/test/routeAndServiceBehavior.test.ts index 680b284fe..3348004ce 100644 --- a/backend/test/routeAndServiceBehavior.test.ts +++ b/backend/test/routeAndServiceBehavior.test.ts @@ -103,6 +103,10 @@ case "$args" in printf '%s\n' '{"ID":"abc123def456","CPUPerc":"1.00%","MemPerc":"2.00%","MemUsage":"10MiB / 1GiB","NetIO":"1kB / 2kB","BlockIO":"3kB / 4kB","PIDs":"5"}' ;; 'inspect abc123def456'|'inspect abc123def456 abc123def456') + if [[ "$MIRA_TEST_DOCKER_INSPECT_FAILURE" == "1" ]]; then + echo 'container disappeared before inspect' >&2 + exit 1 + fi cat <<'JSON' [{"Id":"abc123def4567890","Created":"2026-06-25T00:00:00Z","Image":"sha256:image123","RestartCount":2,"Config":{"Env":["PUBLIC=value","API_TOKEN=secret","URL=https://user:pass@example.test"],"Labels":{"com.docker.compose.project":"stack","com.docker.compose.service":"web","secret.url":"https://user:pass@example.test"}},"Mounts":[{"Type":"volume","Name":"data","Source":"/var/lib/docker/volumes/data","Destination":"/data","Mode":"rw","RW":true}],"NetworkSettings":{"Networks":{"bridge":{"Gateway":"172.17.0.1","IPAddress":"172.17.0.2","MacAddress":"aa:bb"}}},"State":{"StartedAt":"2026-06-25T00:00:01Z","FinishedAt":"","Health":{"Status":"healthy"}}}] JSON @@ -2830,6 +2834,22 @@ describe("backend route and service behavior", () => { const response = await metricsRoutes["/api/metrics"].GET(); expect(response.status).toBe(200); await expect(response.json()).resolves.toMatchObject({ + polling: { + snapshots: expect.arrayContaining([ + expect.objectContaining({ + activeLoads: 1, + loads: 1, + name: "system.metrics", + requests: 1, + }), + ]), + }, + processes: expect.objectContaining({ + active: expect.any(Number), + failed: expect.any(Number), + started: expect.any(Number), + succeeded: expect.any(Number), + }), tokens: { byAgent: [ { @@ -3585,6 +3605,7 @@ describe("backend route and service behavior", () => { it("serves Docker inventory and safe mutations through a fake Docker CLI", async () => { rememberEnvironment("PATH"); rememberEnvironment("MIRA_DOCKER_COMPOSE_WRAPPER"); + rememberEnvironment("MIRA_TEST_DOCKER_INSPECT_FAILURE"); rememberEnvironment("MIRA_DOCKER_ROOT"); const fakeBin = createTemporaryRoot("mira-docker-route-bin-"); const dockerRoot = createTemporaryRoot("mira-docker-route-root-"); @@ -3621,7 +3642,23 @@ describe("backend route and service behavior", () => { .run(executionBaseline.rowId); }); - const containers = await dockerRoutes["/api/docker/containers"].GET(); + process.env.MIRA_TEST_DOCKER_INSPECT_FAILURE = "1"; + const containerStats = await dockerRoutes["/api/docker/containers/stats"].GET(); + await expect(containerStats.json()).resolves.toMatchObject({ + stats: [ + { + cpu: "1.00%", + id: "abc123def456", + memory: "10MiB / 1GiB", + }, + ], + }); + process.env.MIRA_TEST_DOCKER_INSPECT_FAILURE = "0"; + + const containers = await dockerRoutes["/api/docker/containers"].GET( + new Request("https://test.local/api/docker/containers") + ); + const containerEtag = containers.headers.get("etag"); await expect(containers.json()).resolves.toMatchObject({ containers: [ { @@ -3634,17 +3671,13 @@ describe("backend route and service behavior", () => { }, ], }); - - const containerStats = await dockerRoutes["/api/docker/containers/stats"].GET(); - await expect(containerStats.json()).resolves.toMatchObject({ - stats: [ - { - cpu: "1.00%", - id: "abc123def456", - memory: "10MiB / 1GiB", - }, - ], - }); + expect(containerEtag).toBeTruthy(); + const revalidatedContainers = await dockerRoutes["/api/docker/containers"].GET( + new Request("https://test.local/api/docker/containers", { + headers: { "If-None-Match": containerEtag! }, + }) + ); + expect(revalidatedContainers.status).toBe(304); const details = await dockerRoutes["/api/docker/containers/:containerId"].GET( requestWithParameters("/api/docker/containers/demo", { containerId: "demo" }) diff --git a/backend/test/setup.ts b/backend/test/setup.ts index c5059e54b..dc5fa8875 100644 --- a/backend/test/setup.ts +++ b/backend/test/setup.ts @@ -4,6 +4,8 @@ import path from "node:path"; import { afterAll, afterEach, jest } from "bun:test"; +import { resetCoalescedSnapshotsForTests } from "../src/lib/coalescedSnapshot.ts"; + const preloadDatabaseRoot = mkdtempSync( path.join(tmpdir(), "mira-dashboard-test-preload-") ); @@ -44,4 +46,5 @@ afterAll(() => { afterEach(() => { jest.restoreAllMocks(); + resetCoalescedSnapshotsForTests(); }); diff --git a/src/components/features/chat/useChatHistory.ts b/src/components/features/chat/useChatHistory.ts index 174aa3eca..802f7fa01 100644 --- a/src/components/features/chat/useChatHistory.ts +++ b/src/components/features/chat/useChatHistory.ts @@ -7,6 +7,7 @@ import { useState, } from "react"; +import { isBrowserPollingAllowed, refreshPolicy } from "../../../lib/refreshPolicy"; import { nextHistoryLoadSendError, nextRefreshedChatMessages, @@ -20,7 +21,7 @@ import { } from "./chatUtilities"; import type { ChatTransport } from "./transport/chatTransport"; -const LIVE_HISTORY_POLL_MS = 2000; +const LIVE_HISTORY_POLL_MS = refreshPolicy.live; interface ChatHistoryOptions { isConnected: boolean; @@ -246,7 +247,7 @@ export function useChatHistory({ const refreshVisibleHistory = async () => { if ( isRefreshInFlight || - document.visibilityState === "hidden" || + !isBrowserPollingAllowed() || !shouldStickToBottomReference.current ) { return; diff --git a/src/hooks/useAgents.ts b/src/hooks/useAgents.ts index ce8d4d537..1c23976be 100644 --- a/src/hooks/useAgents.ts +++ b/src/hooks/useAgents.ts @@ -1,5 +1,6 @@ import { useQuery } from "@tanstack/react-query"; +import { refreshPolicy } from "../lib/refreshPolicy"; import type { Agent, AgentTaskHistoryItem } from "../types/session"; import { apiFetchRequired } from "./useApi"; @@ -41,7 +42,7 @@ export function useAgentsStatus() { return useQuery({ queryKey: ["agents", "status"], queryFn: () => apiFetchRequired("/agents/status"), - refetchInterval: 2000, + refetchInterval: refreshPolicy.live, staleTime: 1000, }); } @@ -63,7 +64,7 @@ export function useAgentTaskHistory(limit = 8) { apiFetchRequired( `/agents/tasks/history?limit=${limit}` ), - refetchInterval: 5000, + refetchInterval: refreshPolicy.active, staleTime: 4000, }); } @@ -74,7 +75,7 @@ export function useAgentStatus(agentId: string) { queryKey: ["agents", "status", agentId], queryFn: () => apiFetchRequired(`/agents/${encodeURIComponent(agentId)}/status`), - refetchInterval: 2000, + refetchInterval: refreshPolicy.live, staleTime: 1000, }); } diff --git a/src/hooks/useBackups.ts b/src/hooks/useBackups.ts index 10f0a8029..46c02c3ac 100644 --- a/src/hooks/useBackups.ts +++ b/src/hooks/useBackups.ts @@ -1,5 +1,6 @@ import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { refreshPolicy } from "../lib/refreshPolicy"; import { apiFetchRequired, apiPostRequired } from "./useApi"; import { cacheKeys } from "./useCache"; import { @@ -39,7 +40,7 @@ export function useKopiaBackup() { queryFn: () => apiFetchRequired("/backups/kopia"), refetchInterval: (query) => { const status = query.state.data?.job?.status; - return status === "running" ? 1000 : 5000; + return status === "running" ? 1000 : refreshPolicy.active; }, staleTime: 1000, }); @@ -52,7 +53,7 @@ export function useWalgBackup() { queryFn: () => apiFetchRequired("/backups/walg"), refetchInterval: (query) => { const status = query.state.data?.job?.status; - return status === "running" ? 1000 : 5000; + return status === "running" ? 1000 : refreshPolicy.active; }, staleTime: 1000, }); diff --git a/src/hooks/useCron.ts b/src/hooks/useCron.ts index 77647d41f..d5c9caa28 100644 --- a/src/hooks/useCron.ts +++ b/src/hooks/useCron.ts @@ -1,5 +1,6 @@ import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { refreshPolicy } from "../lib/refreshPolicy"; import type { JobDisableIntent } from "../types/job"; import { apiFetchRequired, apiPostRequired } from "./useApi"; @@ -40,7 +41,7 @@ export function useCronJobs() { queryKey: cronKeys.jobs(), queryFn: () => apiFetchRequired("/cron/jobs"), select: (data) => data.jobs, - refetchInterval: 10_000, + refetchInterval: refreshPolicy.active * 2, }); } diff --git a/src/hooks/useDelivery.ts b/src/hooks/useDelivery.ts index 5d0a35030..f2e23177c 100644 --- a/src/hooks/useDelivery.ts +++ b/src/hooks/useDelivery.ts @@ -1,6 +1,7 @@ import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; import { AUTO_REFRESH_MS } from "../lib/queryClient"; +import { refreshPolicy } from "../lib/refreshPolicy"; import { apiFetchRequired, apiPostRequired } from "./useApi"; /** Represents pull request author. */ @@ -164,7 +165,7 @@ export const deliveryKeys = { releaseStatus: () => [...deliveryKeys.all, "releases"] as const, }; -export const DELIVERY_NAV_REFRESH_MS = 60_000; +export const DELIVERY_NAV_REFRESH_MS = refreshPolicy.static; export const DELIVERY_PAGE_REFRESH_MS = AUTO_REFRESH_MS; /** Fetches pull requests. */ @@ -286,7 +287,9 @@ async function stopPullRequestPreview(number: number): Promise) { return queryClient.invalidateQueries({ queryKey: cacheKeys.entry("docker.summary") }); @@ -281,7 +282,7 @@ export function useDockerContainer(containerId: string | undefined) { queryKey: dockerKeys.container(containerId || ""), queryFn: () => fetchContainer(containerId!), enabled: Boolean(containerId), - refetchInterval: 60_000, + refetchInterval: refreshPolicy.static, refetchOnWindowFocus: false, staleTime: 60_000, }); @@ -297,7 +298,7 @@ export function useDockerContainerLogs( queryKey: dockerKeys.containerLogs(containerId || "", tail), queryFn: () => fetchContainerLogs(containerId!, tail), enabled: isEnabled && Boolean(containerId), - refetchInterval: 5000, + refetchInterval: refreshPolicy.active, }); } diff --git a/src/hooks/useHealth.ts b/src/hooks/useHealth.ts index 0a0cdfd84..1e3098c0c 100644 --- a/src/hooks/useHealth.ts +++ b/src/hooks/useHealth.ts @@ -1,5 +1,6 @@ import { useQuery } from "@tanstack/react-query"; +import { refreshPolicy } from "../lib/refreshPolicy"; import { apiFetchRequired } from "./useApi"; /** Represents the health API response. */ @@ -35,7 +36,7 @@ export function useHealth() { return useQuery({ queryKey: ["health"], queryFn: fetchHealth, - refetchInterval: 10_000, + refetchInterval: refreshPolicy.active * 2, staleTime: 5000, }); } diff --git a/src/hooks/useJobExecutions.ts b/src/hooks/useJobExecutions.ts index 11b980c55..62c873610 100644 --- a/src/hooks/useJobExecutions.ts +++ b/src/hooks/useJobExecutions.ts @@ -5,6 +5,7 @@ import { useQueryClient, } from "@tanstack/react-query"; +import { refreshPolicy } from "../lib/refreshPolicy"; import { apiFetchRequired, apiPostRequired } from "./useApi"; export type JobResourceClass = @@ -54,7 +55,6 @@ export const jobExecutionKeys = { list: () => [...jobExecutionKeys.all, "list"] as const, }; -const JOB_EXECUTION_REFRESH_MS = 5000; const JOB_EXECUTION_ENQUEUE_REFRESH_DELAY_MS = 250; /** @@ -81,7 +81,7 @@ export function useJobExecutions() { return useQuery({ queryKey: jobExecutionKeys.list(), queryFn: () => apiFetchRequired("/job-executions"), - refetchInterval: JOB_EXECUTION_REFRESH_MS, + refetchInterval: refreshPolicy.active, refetchIntervalInBackground: false, staleTime: 500, }); diff --git a/src/hooks/useMetrics.ts b/src/hooks/useMetrics.ts index df08f6286..7c8029ac1 100644 --- a/src/hooks/useMetrics.ts +++ b/src/hooks/useMetrics.ts @@ -34,6 +34,26 @@ export interface Metrics { downloadMbps: number; uploadMbps: number; }; + polling?: { + snapshots: Array<{ + activeLoads: number; + averageLoadMs: number; + coalescedHits: number; + failures: number; + freshHits: number; + lastLoadMs: number; + loads: number; + name: string; + requests: number; + staleHits: number; + }>; + }; + processes?: { + active: number; + failed: number; + started: number; + succeeded: number; + }; tokens: { total: number; byModel: Record; diff --git a/src/hooks/useOpenClawSocket.ts b/src/hooks/useOpenClawSocket.ts index ccec349bf..f35f13503 100644 --- a/src/hooks/useOpenClawSocket.ts +++ b/src/hooks/useOpenClawSocket.ts @@ -15,6 +15,7 @@ import { type AuthSessionIdentity, isSignaledAuthSessionRotation, } from "../lib/authBoundary"; +import { isBrowserPollingAllowed, refreshPolicy } from "../lib/refreshPolicy"; import { createSocketClient, type SocketClient, @@ -55,6 +56,9 @@ export function OpenClawSocketProvider({ children }: { children: ReactNode }) { const previousAuthIdentityReference = useRef( undefined ); + const sessionListRefreshReference = useRef<{ promise: Promise } | undefined>( + undefined + ); const [isConnected, setIsConnected] = useState(false); const [hasConfirmedSessionList, setHasConfirmedSessionList] = useState(false); @@ -74,6 +78,29 @@ export function OpenClawSocketProvider({ children }: { children: ReactNode }) { setHasConfirmedSessionList(true); }; + /** Coalesces session resync triggers within this browser connection. */ + const refreshSessionList = async (client: SocketClient): Promise => { + const existing = sessionListRefreshReference.current; + if (existing) { + await existing.promise; + return; + } + + const load = async () => { + const payload = await client.request("sessions.list"); + applySessionsListResponse(client, payload); + }; + const refresh = { promise: load() }; + sessionListRefreshReference.current = refresh; + try { + await refresh.promise; + } finally { + if (sessionListRefreshReference.current === refresh) { + sessionListRefreshReference.current = undefined; + } + } + }; + /** Performs connect. */ const connect = () => { if (!isAuthenticated) { @@ -93,14 +120,9 @@ export function OpenClawSocketProvider({ children }: { children: ReactNode }) { if (!client) { return; } - void (async () => { - try { - const payload = await client.request("sessions.list"); - applySessionsListResponse(client, payload); - } catch { - // Best-effort socket resync. - } - })(); + void refreshSessionList(client).catch(() => { + // Best-effort socket resync. + }); }, onClose: () => { setIsConnected(false); @@ -148,6 +170,7 @@ export function OpenClawSocketProvider({ children }: { children: ReactNode }) { const disconnect = () => { clientReference.current?.disconnect(); clientReference.current = undefined; + sessionListRefreshReference.current = undefined; setIsConnected(false); setHasConfirmedSessionList(false); }; @@ -223,15 +246,14 @@ export function OpenClawSocketProvider({ children }: { children: ReactNode }) { } const interval = setInterval(() => { - if (!clientReference.current?.isOpen()) { + if (!isBrowserPollingAllowed() || !clientReference.current?.isOpen()) { return; } const client = clientReference.current; void (async () => { try { - const payload = await client.request("sessions.list"); - applySessionsListResponse(client, payload); + await refreshSessionList(client); } catch { if (clientReference.current !== client) { return; @@ -242,7 +264,7 @@ export function OpenClawSocketProvider({ children }: { children: ReactNode }) { client.connect(); } })(); - }, 10_000); + }, refreshPolicy.active * 2); return () => clearInterval(interval); }, [isConnected, connectionId]); @@ -254,7 +276,7 @@ export function OpenClawSocketProvider({ children }: { children: ReactNode }) { /** Performs resync visible socket. */ const resyncVisibleSocket = () => { - if (document.visibilityState === "hidden") { + if (!isBrowserPollingAllowed()) { return; } @@ -269,8 +291,7 @@ export function OpenClawSocketProvider({ children }: { children: ReactNode }) { if (!client) { return; } - const payload = await client.request("sessions.list"); - applySessionsListResponse(client, payload); + await refreshSessionList(client); } catch { // Best-effort socket resync. } @@ -292,6 +313,7 @@ export function OpenClawSocketProvider({ children }: { children: ReactNode }) { return () => { clientReference.current?.disconnect(); clientReference.current = undefined; + sessionListRefreshReference.current = undefined; }; }, []); diff --git a/src/hooks/useReports.ts b/src/hooks/useReports.ts index a97b6520c..124b23dc6 100644 --- a/src/hooks/useReports.ts +++ b/src/hooks/useReports.ts @@ -1,5 +1,6 @@ import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { refreshPolicy } from "../lib/refreshPolicy"; import { apiDeleteRequired, apiFetchRequired, apiPostRequired } from "./useApi"; export type ReportType = "daily_brief" | "daily_summary" | "heartbeat" | "custom"; @@ -48,7 +49,7 @@ interface ReportsFilters { type?: ReportType; } -const REPORTS_REFRESH_INTERVAL_MS = 30_000; +const REPORTS_REFRESH_INTERVAL_MS = refreshPolicy.background; export const reportKeys = { all: ["reports"] as const, diff --git a/src/hooks/useScheduledJobs.ts b/src/hooks/useScheduledJobs.ts index 2a6083901..cf581570f 100644 --- a/src/hooks/useScheduledJobs.ts +++ b/src/hooks/useScheduledJobs.ts @@ -1,5 +1,6 @@ import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { refreshPolicy } from "../lib/refreshPolicy"; import type { JobDisableIntent } from "../types/job"; import { apiFetchRequired, apiPatchRequired, apiPostRequired } from "./useApi"; import { @@ -120,8 +121,8 @@ export function useScheduledJobs() { select: (data) => data.jobs.map((job) => normalizeScheduledJob(job)), refetchInterval: (query) => query.state.data?.jobs.some((job) => job.isQueued || job.isRunning) - ? 2000 - : 30_000, + ? refreshPolicy.live + : refreshPolicy.background, }); } @@ -139,8 +140,8 @@ export function useScheduledJobRuns(id: string) { query.state.data?.runs.some( (run) => run.status === "queued" || run.status === "running" ) - ? 2000 - : 30_000, + ? refreshPolicy.live + : refreshPolicy.background, }); } diff --git a/src/lib/queryClient.ts b/src/lib/queryClient.ts index 0203c5f99..16112dc21 100644 --- a/src/lib/queryClient.ts +++ b/src/lib/queryClient.ts @@ -1,7 +1,9 @@ import { QueryClient } from "@tanstack/react-query"; +import { refreshPolicy } from "./refreshPolicy"; + /** Defines auto refresh milliseconds. */ -export const AUTO_REFRESH_MS = 5000; +export const AUTO_REFRESH_MS = refreshPolicy.active; /** Defines query client. */ export const queryClient = new QueryClient({ diff --git a/src/lib/refreshPolicy.ts b/src/lib/refreshPolicy.ts new file mode 100644 index 000000000..74538e8ec --- /dev/null +++ b/src/lib/refreshPolicy.ts @@ -0,0 +1,15 @@ +/** Named refresh tiers shared by recurring read-only UI queries. */ +export const refreshPolicy = { + active: 5000, + background: 30_000, + live: 2000, + static: 60_000, +} as const; + +/** Returns whether browser-driven polling or reconnect work should run now. */ +export function isBrowserPollingAllowed(): boolean { + const isVisible = + typeof document === "undefined" || document.visibilityState !== "hidden"; + const isOnline = typeof navigator === "undefined" || navigator.onLine !== false; + return isVisible && isOnline; +} diff --git a/src/lib/socket/socketClient.ts b/src/lib/socket/socketClient.ts index 5318698b7..7dcdb46c4 100644 --- a/src/lib/socket/socketClient.ts +++ b/src/lib/socket/socketClient.ts @@ -1,5 +1,6 @@ import type { SocketEnvelope } from "../../types/socket"; import { recoverOrHandleUnauthorizedSession } from "../authBoundary"; +import { isBrowserPollingAllowed, refreshPolicy } from "../refreshPolicy"; import { dispatchSecurityVerificationRequired, isSecurityVerificationCode, @@ -10,6 +11,27 @@ import { hasRecentUserActivity } from "../userActivity"; const DEFAULT_REQUEST_TIMEOUT_MS = 30_000; const MAX_TIMER_DELAY_MS = 2_147_483_647; +const MAX_RECONNECT_DELAY_MS = 30_000; +const RECONNECT_JITTER_RATIO = 0.2; + +/** Returns bounded exponential reconnect delay with symmetric jitter. */ +export function socketReconnectDelayMs( + attempt: number, + random: () => number = Math.random +): number { + const normalizedAttempt = Math.max(0, Math.min(10, Math.trunc(attempt))); + const exponentialDelay = Math.min( + refreshPolicy.live * 2 ** normalizedAttempt, + MAX_RECONNECT_DELAY_MS + ); + const randomValue = Math.min(1, Math.max(0, random())); + const jitterMultiplier = + 1 - RECONNECT_JITTER_RATIO + randomValue * RECONNECT_JITTER_RATIO * 2; + return Math.min( + MAX_RECONNECT_DELAY_MS, + Math.max(250, Math.round(exponentialDelay * jitterMultiplier)) + ); +} function normalizedRequestTimeoutMs(requestedTimeoutMs: number | undefined): number { return typeof requestedTimeoutMs === "number" && @@ -64,6 +86,8 @@ export function createSocketClient(options: SocketClientOptions): SocketClient { let shouldReconnect = true; let isRecoveringAuthorization = false; let requestId = 0; + let reconnectAttempt = 0; + let reconnectTimer: ReturnType | undefined; const pendingRequests = new Map(); const connectionWaiters = new Set<{ reject: (reason: Error) => void; @@ -91,6 +115,28 @@ export function createSocketClient(options: SocketClientOptions): SocketClient { connectionWaiters.clear(); }; + const clearReconnectTimer = () => { + if (reconnectTimer === undefined) return; + clearTimeout(reconnectTimer); + reconnectTimer = undefined; + }; + + const scheduleReconnect = () => { + if ( + !shouldReconnect || + reconnectTimer !== undefined || + !isBrowserPollingAllowed() + ) { + return; + } + const delayMs = socketReconnectDelayMs(reconnectAttempt); + reconnectAttempt += 1; + reconnectTimer = setTimeout(() => { + reconnectTimer = undefined; + if (shouldReconnect && isBrowserPollingAllowed()) connect(); + }, delayMs); + }; + const waitForOpenSocket = (requestOptions?: SocketRequestOptions): Promise => { if (ws?.readyState === WebSocket.OPEN) { return Promise.resolve(); @@ -165,6 +211,8 @@ export function createSocketClient(options: SocketClientOptions): SocketClient { shouldReconnect = true; isRecoveringAuthorization = false; + if (!isBrowserPollingAllowed()) return; + clearReconnectTimer(); const socket = new WebSocket(options.url); ws = socket; @@ -172,6 +220,8 @@ export function createSocketClient(options: SocketClientOptions): SocketClient { if (ws !== socket) { return; } + reconnectAttempt = 0; + clearReconnectTimer(); resolveConnectionWaiters(); options.onOpen?.(); }); @@ -260,13 +310,7 @@ export function createSocketClient(options: SocketClientOptions): SocketClient { })(); return; } - if (shouldReconnect) { - setTimeout(() => { - if (shouldReconnect) { - connect(); - } - }, 2000); - } + scheduleReconnect(); }); socket.addEventListener("error", () => { @@ -281,6 +325,8 @@ export function createSocketClient(options: SocketClientOptions): SocketClient { const disconnect = () => { shouldReconnect = false; isRecoveringAuthorization = false; + reconnectAttempt = 0; + clearReconnectTimer(); const socket = ws; ws = undefined; rejectPendingRequests(); diff --git a/src/test/frontendBehavior.test.tsx b/src/test/frontendBehavior.test.tsx index f49b596d0..ba00afa31 100644 --- a/src/test/frontendBehavior.test.tsx +++ b/src/test/frontendBehavior.test.tsx @@ -202,7 +202,7 @@ import { SecurityVerificationCancelledError, waitForSecurityVerification, } from "../lib/securityVerification"; -import { createSocketClient } from "../lib/socket/socketClient"; +import { createSocketClient, socketReconnectDelayMs } from "../lib/socket/socketClient"; import { handleSocketMessage } from "../lib/socket/socketMessageRouter"; import { hasRecentUserActivity, @@ -1747,6 +1747,69 @@ describe("Mira Dashboard frontend behavior", () => { } }); + it("uses bounded exponential WebSocket reconnect backoff with jitter", () => { + expect(socketReconnectDelayMs(0, () => 0.5)).toBe(2000); + expect(socketReconnectDelayMs(1, () => 0.5)).toBe(4000); + expect(socketReconnectDelayMs(2, () => 0)).toBe(6400); + expect(socketReconnectDelayMs(2, () => 1)).toBe(9600); + expect(socketReconnectDelayMs(20, () => 1)).toBe(30_000); + }); + + it("defers WebSocket connect and reconnect while the page is hidden", () => { + const originalWebSocket = WebSocket; + const visibilityDescriptor = Object.getOwnPropertyDescriptor( + document, + "visibilityState" + ); + FakeWebSocket.instances = []; + Object.defineProperty(globalThis, "WebSocket", { + configurable: true, + value: FakeWebSocket, + writable: true, + }); + Object.defineProperty(document, "visibilityState", { + configurable: true, + value: "hidden", + }); + const timeoutSpy = jest.spyOn(globalThis, "setTimeout"); + const client = createSocketClient({ url: "ws://dashboard.test/socket" }); + + try { + client.connect(); + expect(FakeWebSocket.instances).toHaveLength(0); + + Object.defineProperty(document, "visibilityState", { + configurable: true, + value: "visible", + }); + client.connect(); + const socket = FakeWebSocket.instances[0]!; + socket.open(); + + Object.defineProperty(document, "visibilityState", { + configurable: true, + value: "hidden", + }); + const timeoutCount = timeoutSpy.mock.calls.length; + socket.close(); + expect(timeoutSpy).toHaveBeenCalledTimes(timeoutCount); + } finally { + client.disconnect(); + timeoutSpy.mockRestore(); + Object.defineProperty(globalThis, "WebSocket", { + configurable: true, + value: originalWebSocket, + writable: true, + }); + if (visibilityDescriptor) { + Object.defineProperty(document, "visibilityState", visibilityDescriptor); + } else { + delete (document as unknown as { visibilityState?: string }) + .visibilityState; + } + } + }); + it("drives socket client request, response, error, and disconnect behavior", async () => { const originalWebSocket = WebSocket; FakeWebSocket.instances = [];